no explicit auth structures in python to enable easier parsing by bash and php

[monitor.git] / findbad.py

diff --git a/findbad.py b/findbad.py
index 141f9ac..2b8fe00 100755 (executable)
--- a/findbad.py
+++ b/findbad.py
@@ -8,9 +8,9 @@ import time
 
 # QUERY all nodes.
 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
-                                       "table=table_nodeview&" + \
-                                   "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
-                                   "formatcsv"
+                               "table=table_nodeview&" + \
+                               "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
+                               "formatcsv"
                                    #"formatcsv&" + \
                                        #"select='lastcotop!=0'"
 
@@ -21,19 +21,19 @@ externalState = {'round': round, 'nodes': {}}
 count = 0
 
 
-import soltesz
+import database
+import moncommands 
 import comon
 import threadpool
 import syncplcdb
 from nodequery import verify,query_to_dict,node_select
-
+import traceback
 import plc
-import auth
-api = plc.PLC(auth.auth, auth.plc)
+api = plc.getAuthAPI()
 
 def collectPingAndSSH(nodename, cohash):
        ### RUN PING ######################
-       ping = soltesz.CMD()
+       ping = moncommands.CMD()
        (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
 
        values = {}
@@ -46,7 +46,7 @@ def collectPingAndSSH(nodename, cohash):
 
        try:
                for port in [22, 806]: 
-                       ssh = soltesz.SSH('root', nodename, port)
+                       ssh = moncommands.SSH('root', nodename, port)
 
                        (oval, errval) = ssh.run_noexcept2(""" <<\EOF
                                echo "{"
@@ -72,12 +72,12 @@ EOF                 """)
                                '', 'princeton_comon' : '', 'princeton_comon_running' : '',
                                'princeton_comon_procs' : '', 'sshport' : None})
        except:
-               import traceback; print traceback.print_exc()
+               print traceback.print_exc()
                sys.exit(1)
 
        ### RUN SSH ######################
        b_getbootcd_id = True
-       #ssh = soltesz.SSH('root', nodename)
+       #ssh = moncommands.SSH('root', nodename)
        #oval = ""
        #errval = ""
        #(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
@@ -181,19 +181,19 @@ EOF                       """)
                values['comonstats'] = {'resptime':  '-1', 
                                                                'uptime':    '-1',
                                                                'sshstatus': '-1', 
-                                                               'lastcotop': '-1'}
+                                                               'lastcotop': '-1',
+                                                               'cpuspeed' : "null",
+                                                               'disksize' : 'null',
+                                                               'memsize'  : 'null'}
        # include output value
        ### GET PLC NODE ######################
        b_except = False
        plc_lock.acquire()
 
        try:
-               d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'last_contact', 'boot_state', 'nodegroup_ids'])
+               d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'date_created', 'last_updated', 'last_contact', 'boot_state', 'nodegroup_ids'])
        except:
                b_except = True
-               import traceback
-               b_except = True
-               import traceback
                traceback.print_exc()
 
        plc_lock.release()
@@ -208,13 +208,15 @@ EOF                       """)
                        values['pcu'] = "NOPCU"
                site_id = d_node[0]['site_id']
                last_contact = d_node[0]['last_contact']
-               nodegroups = d_node[0]['nodegroup_ids']
+               nodegroups = [ i['name'] for i in api.GetNodeGroups(d_node[0]['nodegroup_ids']) ]
                values['plcnode'] = {'status' : 'SUCCESS', 
                                                        'pcu_ids': pcu, 
                                                        'boot_state' : d_node[0]['boot_state'],
                                                        'site_id': site_id,
                                                        'nodegroups' : nodegroups,
-                                                       'last_contact': last_contact}
+                                                       'last_contact': last_contact,
+                                                       'date_created': d_node[0]['date_created'],
+                                                       'last_updated': d_node[0]['last_updated']}
        else:
                values['pcu']     = "UNKNOWN"
                values['plcnode'] = {'status' : "GN_FAILED"}
@@ -229,7 +231,6 @@ EOF                 """)
                                                        ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
        except:
                b_except = True
-               import traceback
                traceback.print_exc()
 
        plc_lock.release()
@@ -265,7 +266,7 @@ def recordPingAndSSH(request, result):
                count += 1
                print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
                if count % 20 == 0:
-                       soltesz.dbDump(config.dbname, externalState)
+                       database.dbDump(config.dbname, externalState)
 
 # this will be called when an exception occurs within a thread
 def handle_exception(request, result):
@@ -300,10 +301,16 @@ def checkAndRecordState(l_nodes, cohash):
                        pass
 
        # WAIT while all the work requests are processed.
+       begin = time.time()
        while 1:
                try:
                        time.sleep(1)
                        tp.poll()
+                       # if more than two hours
+                       if time.time() - begin > (60*60*1.5):
+                               print "findbad.py has run out of time!!!!!!"
+                               database.dbDump(config.dbname, externalState)
+                               os._exit(1)
                except KeyboardInterrupt:
                        print "Interrupted!"
                        break
@@ -311,14 +318,14 @@ def checkAndRecordState(l_nodes, cohash):
                        print "All results collected."
                        break
 
-       soltesz.dbDump(config.dbname, externalState)
+       database.dbDump(config.dbname, externalState)
 
 
 
 def main():
        global externalState
 
-       externalState = soltesz.if_cached_else(1, config.dbname, lambda : externalState) 
+       externalState = database.if_cached_else(1, config.dbname, lambda : externalState) 
 
        if config.increment:
                # update global round number to force refreshes across all nodes
@@ -389,9 +396,8 @@ if __name__ == '__main__':
        try:
                main()
        except Exception, err:
-               import traceback
                print traceback.print_exc()
                print "Exception: %s" % err
                print "Saving data... exitting."
-               soltesz.dbDump(config.dbname, externalState)
+               database.dbDump(config.dbname, externalState)
                sys.exit(0)