X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=findbad.py;h=6420f99c1efdafb498c67b68deb47c7193ba31a0;hb=refs%2Fheads%2F1.0;hp=5b04398fab45768da49332beebd965e944be10fc;hpb=d0652340b89d51c6115edb13d5c7c72b34dea66f;p=monitor.git diff --git a/findbad.py b/findbad.py index 5b04398..6420f99 100755 --- a/findbad.py +++ b/findbad.py @@ -4,6 +4,8 @@ import os import sys import string import time +import config +import util.file # QUERY all nodes. @@ -21,19 +23,19 @@ externalState = {'round': round, 'nodes': {}} count = 0 -import soltesz +import database +import moncommands import comon import threadpool import syncplcdb from nodequery import verify,query_to_dict,node_select import traceback import plc -import auth -api = plc.PLC(auth.auth, auth.plc) +api = plc.getAuthAPI() def collectPingAndSSH(nodename, cohash): ### RUN PING ###################### - ping = soltesz.CMD() + ping = moncommands.CMD() (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename) values = {} @@ -46,7 +48,7 @@ def collectPingAndSSH(nodename, cohash): try: for port in [22, 806]: - ssh = soltesz.SSH('root', nodename, port) + ssh = moncommands.SSH('root', nodename, port) (oval, errval) = ssh.run_noexcept2(""" <<\EOF echo "{" @@ -54,6 +56,8 @@ def collectPingAndSSH(nodename, cohash): echo ' "bmlog":"'`ls /tmp/bm.log`'",' echo ' "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",' echo ' "nm":"'`ps ax | grep nm.py | grep -v grep`'",' + echo ' "readonlyfs":"'`touch /var/log/monitor 2>&1 ; touch /vservers/monitor.log 2>&1`'",' + echo ' "dns":"'`host boot.planet-lab.org 2>&1`'",' echo ' "princeton_comon":"'`ls -d /vservers/princeton_comon`'",' ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'` @@ -68,16 +72,22 @@ EOF """) values['sshport'] = port break else: - values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', 'nm' : - '', 'princeton_comon' : '', 'princeton_comon_running' : '', - 'princeton_comon_procs' : '', 'sshport' : None}) + values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', + 'nm' : '', + 'readonlyfs' : '', + 'dns' : '', + 'princeton_comon' : '', + 'princeton_comon_running' : '', + 'princeton_comon_procs' : '', 'sshport' : None}) except: print traceback.print_exc() + from nodecommon import email_exception + email_exception() sys.exit(1) ### RUN SSH ###################### b_getbootcd_id = True - #ssh = soltesz.SSH('root', nodename) + #ssh = moncommands.SSH('root', nodename) #oval = "" #errval = "" #(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`') @@ -85,14 +95,14 @@ EOF """) oval = values['kernel'] if "2.6.17" in oval or "2.6.2" in oval: values['ssh'] = 'SSH' - values['category'] = 'ALPHA' + values['category'] = 'PROD' if "bm.log" in values['bmlog']: values['state'] = 'DEBUG' else: values['state'] = 'BOOT' elif "2.6.12" in oval or "2.6.10" in oval: values['ssh'] = 'SSH' - values['category'] = 'PROD' + values['category'] = 'OLDPROD' if "bm.log" in values['bmlog']: values['state'] = 'DEBUG' else: @@ -195,6 +205,8 @@ EOF """) except: b_except = True traceback.print_exc() + from nodecommon import email_exception + email_exception() plc_lock.release() if b_except: return (None, None) @@ -208,7 +220,7 @@ EOF """) values['pcu'] = "NOPCU" site_id = d_node[0]['site_id'] last_contact = d_node[0]['last_contact'] - nodegroups = [ i['name'] for i in api.GetNodeGroups(d_node[0]['nodegroup_ids']) ] + nodegroups = [ i['groupname'] for i in api.GetNodeGroups(d_node[0]['nodegroup_ids']) ] values['plcnode'] = {'status' : 'SUCCESS', 'pcu_ids': pcu, 'boot_state' : d_node[0]['boot_state'], @@ -232,6 +244,8 @@ EOF """) except: b_except = True traceback.print_exc() + from nodecommon import email_exception + email_exception() plc_lock.release() if b_except: return (None, None) @@ -266,7 +280,7 @@ def recordPingAndSSH(request, result): count += 1 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values']) if count % 20 == 0: - soltesz.dbDump(config.dbname, externalState) + database.dbDump(config.dbname, externalState) # this will be called when an exception occurs within a thread def handle_exception(request, result): @@ -301,10 +315,16 @@ def checkAndRecordState(l_nodes, cohash): pass # WAIT while all the work requests are processed. + begin = time.time() while 1: try: time.sleep(1) tp.poll() + # if more than two hours + if time.time() - begin > (60*60*1.5): + print "findbad.py has run out of time!!!!!!" + database.dbDump(config.dbname, externalState) + os._exit(1) except KeyboardInterrupt: print "Interrupted!" break @@ -312,14 +332,14 @@ def checkAndRecordState(l_nodes, cohash): print "All results collected." break - soltesz.dbDump(config.dbname, externalState) + database.dbDump(config.dbname, externalState) def main(): global externalState - externalState = soltesz.if_cached_else(1, config.dbname, lambda : externalState) + externalState = database.if_cached_else(1, config.dbname, lambda : externalState) if config.increment: # update global round number to force refreshes across all nodes @@ -334,14 +354,14 @@ def main(): #cohash = {} cohash = cotop.coget(cotop_url) l_nodes = syncplcdb.create_plcdb() - if config.filename: - f_nodes = config.getListFromFile(config.filename) + if config.nodelist: + f_nodes = util.file.getListFromFile(config.nodelist) l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes) elif config.node: f_nodes = [config.node] l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes) elif config.nodegroup: - ng = api.GetNodeGroups({'name' : config.nodegroup}) + ng = api.GetNodeGroups({'groupname' : config.nodegroup}) l_nodes = api.GetNodes(ng[0]['node_ids']) elif config.site: site = api.GetSites(config.site) @@ -352,7 +372,8 @@ def main(): # perform this query after the above options, so that the filter above # does not break. if config.nodeselect: - l_nodes = node_select(config.nodeselect) + fb = database.dbLoad("findbad") + l_nodes = node_select(config.nodeselect, fb['nodes'].keys(), fb) print "fetching %s hosts" % len(l_nodes) @@ -362,36 +383,29 @@ def main(): if __name__ == '__main__': - from config import config - from optparse import OptionParser - parser = OptionParser() - parser.set_defaults(filename=None, node=None, site=None, nodeselect=False, nodegroup=None, - increment=False, dbname="findbadnodes", cachenodes=False) - parser.add_option("", "--node", dest="node", metavar="hostname", - help="Provide a single node to operate on") - parser.add_option("-f", "--nodelist", dest="filename", metavar="FILE", - help="Provide the input file for the node list") - parser.add_option("", "--nodeselect", dest="nodeselect", metavar="query string", - help="Provide a selection string to return a node list.") - parser.add_option("", "--nodegroup", dest="nodegroup", metavar="FILE", - help="Provide the nodegroup for the list of nodes.") - parser.add_option("", "--site", dest="site", metavar="site name", - help="Specify a site to view node status") + import parser as parsermodule + + parser = parsermodule.getParser(['nodesets']) + parser.set_defaults( increment=False, dbname="findbad", cachenodes=False) parser.add_option("", "--cachenodes", action="store_true", help="Cache node lookup from PLC") parser.add_option("", "--dbname", dest="dbname", metavar="FILE", help="Specify the name of the database to which the information is saved") parser.add_option("-i", "--increment", action="store_true", dest="increment", help="Increment round number to force refresh or retry") - config = config(parser) - config.parse_args() + + parser = parsermodule.getParser(['defaults'], parser) + + cfg = parsermodule.parse_args(parser) try: main() except Exception, err: print traceback.print_exc() + from nodecommon import email_exception + email_exception() print "Exception: %s" % err print "Saving data... exitting." - soltesz.dbDump(config.dbname, externalState) + database.dbDump(config.dbname, externalState) sys.exit(0)