#pflags.resetRecentFlag('pcutried')
if not pflags.getRecentFlag('pcutried'):
try:
- print "CALLING REBOOT!!!"
- ret = reboot.reboot(host)
+ node_pf = PersistFlags(host, 1, db='node_persistflags')
+ if node_pf.checkattr('last_change') and \
+ node_pf.last_change < time.time() - 60*60*24 and \
+ node_pf.checkattr('status') and \
+ node_pf.status != "good":
- pflags.setRecentFlag('pcutried')
- pflags.save()
- return ret
+ print "CALLING REBOOT!!!"
+ ret = reboot.reboot(host)
+
+ pflags.setRecentFlag('pcutried')
+ pflags.save()
+ return ret
+ else:
+ return True
except Exception,e:
+ email_exception()
print traceback.print_exc(); print e
# NOTE: this failure could be an implementation issue on
elif not pflags.getRecentFlag('pcu_rins_tried'):
try:
- # set node to 'rins' boot state.
- print "CALLING REBOOT +++ RINS"
- plc.nodeBootState(host, 'rins')
- ret = reboot.reboot(host)
-
- pflags.setRecentFlag('pcu_rins_tried')
- pflags.save()
- return ret
+ # NOTE: check that the node has been down for at least a
+ # day before rebooting it. this avoids false-reboots/rins
+ # from failed node detections. circa 03-12-09
+ node_pf = PersistFlags(host, 1, db='node_persistflags')
+ if node_pf.checkattr('last_change') and \
+ node_pf.last_change < time.time() - 60*60*24 and \
+ node_pf.checkattr('status') and \
+ node_pf.status != "good":
+
+ # set node to 'rins' boot state.
+ print "CALLING REBOOT +++ RINS"
+ plc.nodeBootState(host, 'reinstall')
+ ret = reboot.reboot(host)
+
+ pflags.setRecentFlag('pcu_rins_tried')
+ pflags.save()
+ return ret
+
+ else:
+ return True
except Exception,e:
+ email_exception()
print traceback.print_exc(); print e
# NOTE: this failure could be an implementation issue on
try:
return mailmonitor.reboot(host)
except Exception, e:
+ email_exception(host)
print traceback.print_exc(); print e
return False
node = api.GetNodes(host, ['boot_state', 'last_contact', 'last_updated', 'date_created'])
record = {'observation' : node[0],
'model' : 'USER_REQUEST',
- 'action' : 'api.UpdateNode(%s, {"boot_state" : "rins"})' % host,
+ 'action' : 'api.UpdateNode(%s, {"boot_state" : "reinstall"})' % host,
'time' : time.time()}
l = Log(host, record)
- ret = api.UpdateNode(host, {'boot_state' : 'rins'})
+ ret = api.UpdateNode(host, {'boot_state' : 'reinstall'})
if ret:
# it's nice to see the current status rather than the previous status on the console
node = api.GetNodes(host)[0]
# COLLECT nodegroups, nodes and node lists
if config.nodegroup:
- ng = api.GetNodeGroups({'name' : config.nodegroup})
+ ng = api.GetNodeGroups({'groupname' : config.nodegroup})
nodelist = api.GetNodes(ng[0]['node_ids'])
hostnames = [ n['hostname'] for n in nodelist ]
+if config.site:
+ site = api.GetSites(config.site)
+ l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
+ hostnames = [ n['hostname'] for n in l_nodes ]
+
if config.node or config.nodelist:
if config.node: hostnames = [ config.node ]
else: hostnames = util.file.getListFromFile(config.nodelist)
try:
node = api.GetNodes(host)[0]
except:
+ email_exception()
print traceback.print_exc();
print "FAILED GETNODES for host: %s" % host
continue
# todo: send thank you, etc.
mailmonitor.reboot(host)
except Exception, e:
+ email_exception()
print traceback.print_exc(); print e
continue
print "ALL METHODS OF RESTARTING %s FAILED" % host
args = {}
args['hostname'] = host
- m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
- "CANNOT CONTACT", False, db='suspect_persistmessages')
- m.reset()
- m.send(['monitor-list@lists.planet-lab.org'])
+ #m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
+ # "CANNOT CONTACT", False, db='suspect_persistmessages')
+ #m.reset()
+ #m.send(['monitor-list@lists.planet-lab.org'])
l = Log(host, record)
print l
print "Killed by interrupt"
sys.exit(0)
except:
+ email_exception()
print traceback.print_exc();
print "Continuing..."