X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=grouprins.py;h=cfefc6a2b9662cba7c2137fb2b29f775ca559c99;hb=e8d7b6fcbb8ef3bc1e185f5cf7294961bb9d380d;hp=92a745ced27d8bfcbaddae7afafe1d2d1faa4d86;hpb=c3f2afdc81c6711c3825c82e2cd4970671575438;p=monitor.git diff --git a/grouprins.py b/grouprins.py index 92a745c..cfefc6a 100755 --- a/grouprins.py +++ b/grouprins.py @@ -15,12 +15,12 @@ import plc api = plc.getAuthAPI() -import policy import traceback -from config import config as cfg +import config import util.file from optparse import OptionParser +import const from nodecommon import * from nodequery import verify,query_to_dict,node_select import database @@ -32,8 +32,8 @@ import parser as parsermodule from model import * import bootman # debug nodes -import monitor # down nodes with pcu import reboot # down nodes without pcu +import mailmonitor # down nodes with pcu from emailTxt import mailtxt #reboot.verbose = 0 import sys @@ -54,20 +54,45 @@ class Reboot(object): mailtxt.pcudown_one[1] % args, True, db='pcu_persistmessages') loginbase = plc.siteId(host) - m.send([policy.TECHEMAIL % loginbase]) + m.send([const.TECHEMAIL % loginbase]) def pcu(self, host): # TODO: It should be possible to diagnose the various conditions of # the PCU here, and send different messages as appropriate. - if self.fbnode['pcu'] == "PCU": + print "'%s'" % self.fbnode['pcu'] + if self.fbnode['pcu'] == "PCU" or "PCUOK" in self.fbnode['pcu']: self.action = "reboot.reboot('%s')" % host pflags = PersistFlags(host, 2*60*60*24, db='pcu_persistflags') + #pflags.resetRecentFlag('pcutried') if not pflags.getRecentFlag('pcutried'): - pflags.setRecentFlag('pcutried') try: + print "CALLING REBOOT!!!" ret = reboot.reboot(host) + pflags.setRecentFlag('pcutried') + pflags.save() + return ret + + except Exception,e: + print traceback.print_exc(); print e + + # NOTE: this failure could be an implementation issue on + # our end. So, extra notices are confusing... + # self._send_pcunotice(host) + + pflags.setRecentFlag('pcufailed') + pflags.save() + return False + + elif not pflags.getRecentFlag('pcu_rins_tried'): + try: + # set node to 'rins' boot state. + print "CALLING REBOOT +++ RINS" + plc.nodeBootState(host, 'rins') + ret = reboot.reboot(host) + + pflags.setRecentFlag('pcu_rins_tried') pflags.save() return ret @@ -90,12 +115,14 @@ class Reboot(object): pflags.setRecentFlag('pcumessagesent') pflags.save() - # NOTE: this will result in just one message sent at a time. - return True - else: - return False + # This will result in mail() being called next, to try to + # engage the technical contact to take care of it also. + print "RETURNING FALSE" + return False + else: + print "NO PCUOK" self.action = "None" return False @@ -108,10 +135,10 @@ class Reboot(object): pflags.setRecentFlag('endrecord') pflags.save() - # Then in either case, run monitor.reboot() - self.action = "monitor.reboot('%s')" % host + # Then in either case, run mailmonitor.reboot() + self.action = "mailmonitor.reboot('%s')" % host try: - return monitor.reboot(host) + return mailmonitor.reboot(host) except Exception, e: print traceback.print_exc(); print e return False @@ -169,8 +196,6 @@ parser.set_defaults( timewait=0, force=False, nosetup=False, verbose=False, - stopkey=None, - stopvalue=None, quiet=False, ) @@ -203,26 +228,36 @@ if config.nodegroup: nodelist = api.GetNodes(ng[0]['node_ids']) hostnames = [ n['hostname'] for n in nodelist ] +if config.site: + site = api.GetSites(config.site) + l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname']) + hostnames = [ n['hostname'] for n in l_nodes ] + if config.node or config.nodelist: if config.node: hostnames = [ config.node ] - else: hostnames = config.getListFromFile(config.nodelist) + else: hostnames = util.file.getListFromFile(config.nodelist) + +fb = database.dbLoad("findbad") if config.nodeselect: - hostnames = node_select(config.nodeselect) + hostnames = node_select(config.nodeselect, fb['nodes'].keys(), fb) if config.findbad: # rerun findbad with the nodes in the given nodes. file = "findbad.txt" util.file.setFileFromList(file, hostnames) - os.system("./findbad.py --cachenodes --debug=0 --dbname=findbad --increment --nodelist %s" % file) + os.system("./findbad.py --cachenodes --increment --nodelist %s" % file) + # TODO: shouldn't we reload the node list now? -fb = database.dbLoad("findbad") +l_blacklist = database.if_cached_else(1, "l_blacklist", lambda : []) # commands: i = 1 count = 1 +#print "hosts: %s" % hostnames for host in hostnames: #if 'echo' in host or 'hptest-1' in host: continue + try: try: node = api.GetNodes(host)[0] @@ -233,7 +268,10 @@ for host in hostnames: print "%-2d" % i, nodegroup_display(node, fb) i += 1 - if i < int(config.skip): continue + if i-1 <= int(config.skip): continue + if host in l_blacklist: + print "%s is blacklisted. Skipping." % host + continue if config.stopselect: dict_query = query_to_dict(config.stopselect) @@ -243,20 +281,17 @@ for host in hostnames: if verify(dict_query, fbnode) and observed_state != "dbg ": # evaluates to true, therefore skip. print "%s evaluates true for %s ; skipping..." % ( config.stopselect, host ) - continue - - if config.stopkey and config.stopvalue: - fbnode = fb['nodes'][host]['values'] - observed_state = get_current_state(fbnode) + try: + # todo: clean up act_all record here. + # todo: send thank you, etc. + mailmonitor.reboot(host) + except Exception, e: + print traceback.print_exc(); print e - if config.stopkey in fbnode: - if config.stopvalue in fbnode[config.stopkey] and observed_state != "dbg ": - print "%s has stopvalue; skipping..." % host - continue - else: - print "stopkey %s not in fbnode record for %s; skipping..." % (config.stopkey, host) - print fbnode continue + #else: + #print "%s failed to match %s: -%s-" % ( host, dict_query, observed_state ) + #sys.exit(1) if not config.force and rebootlog.find(host, {'action' : ".*reboot"}, 60*60*2): print "recently rebooted %s. skipping... " % host @@ -309,10 +344,10 @@ for host in hostnames: print "ALL METHODS OF RESTARTING %s FAILED" % host args = {} args['hostname'] = host - m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args, - "CANNOT CONTACT", False, db='suspect_persistmessages') - m.reset() - m.send(['monitor-list@lists.planet-lab.org']) + #m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args, + # "CANNOT CONTACT", False, db='suspect_persistmessages') + #m.reset() + #m.send(['monitor-list@lists.planet-lab.org']) l = Log(host, record) print l