api = plc.PLC(auth.auth, auth.plc)
import policy
-
+import traceback
from config import config as cfg
+import config as configmodule
from optparse import OptionParser
from nodecommon import *
from nodequery import verify,query_to_dict,node_select
import soltesz
from unified_model import *
+import os
import time
import bootman # debug nodes
import monitor # down nodes with pcu
import reboot # down nodes without pcu
-reboot.verbose = 0
+from emailTxt import mailtxt
+#reboot.verbose = 0
import sys
class Reboot(object):
m = PersistMessage(host, mailtxt.pcudown_one[0] % args,
mailtxt.pcudown_one[1] % args, True, db='pcu_persistmessages')
- loginbase = plc.siteId(hostname)
+ loginbase = plc.siteId(host)
m.send([policy.TECHEMAIL % loginbase])
def pcu(self, host):
if self.fbnode['pcu'] == "PCU":
self.action = "reboot.reboot('%s')" % host
- pflags = PersistFlags(host, 1*60*60*24, db='pcu_persistflags')
- if not pflags.getRecentFlag('pcutried'): # or not pflags.getFlag('pcufailed'):
+ pflags = PersistFlags(host, 2*60*60*24, db='pcu_persistflags')
+ if not pflags.getRecentFlag('pcutried'):
pflags.setRecentFlag('pcutried')
try:
ret = reboot.reboot(host)
return ret
except Exception,e:
- import traceback; print traceback.print_exc(); print e
+ print traceback.print_exc(); print e
# NOTE: this failure could be an implementation issue on
# our end. So, extra notices are confusing...
pflags.setRecentFlag('pcumessagesent')
pflags.save()
+ # NOTE: this will result in just one message sent at a time.
+ return True
else:
- pass # just skip it?
-
+ return False
else:
self.action = "None"
return False
try:
return monitor.reboot(host)
except Exception, e:
- import traceback; print traceback.print_exc(); print e
+ print traceback.print_exc(); print e
return False
class RebootDebug(Reboot):
self.action = "None"
return False # this always fails, since the node will be down.
+def set_node_to_rins(host, fb):
+
+ node = api.GetNodes(host, ['boot_state', 'last_contact', 'last_updated', 'date_created'])
+ record = {'observation' : node[0],
+ 'model' : 'USER_REQUEST',
+ 'action' : 'api.UpdateNode(%s, {"boot_state" : "rins"})' % host,
+ 'time' : time.time()}
+ l = Log(host, record)
+
+ ret = api.UpdateNode(host, {'boot_state' : 'rins'})
+ if ret:
+ # it's nice to see the current status rather than the previous status on the console
+ node = api.GetNodes(host)[0]
+ print l
+ print "%-2d" % (i-1), nodegroup_display(node, fb)
+ return l
+ else:
+ print "FAILED TO UPDATE NODE BOOT STATE : %s" % host
+ return None
+
try:
rebootlog = soltesz.dbLoad("rebootlog")
node=None,
nodelist=None,
nodeselect=None,
- timewait=30,
+ timewait=0,
skip=0,
rins=False,
reboot=False,
if config.findbad:
# rerun findbad with the nodes in the given nodes.
- import os
file = "findbad.txt"
- config.setFileFromList(file, hostnames)
+ configmodule.setFileFromList(file, hostnames)
os.system("./findbad.py --cachenodes --debug=0 --dbname=findbad --increment --nodelist %s" % file)
fb = soltesz.dbLoad("findbad")
for host in hostnames:
#if 'echo' in host or 'hptest-1' in host: continue
-
try:
try:
node = api.GetNodes(host)[0]
except:
- import traceback; print traceback.print_exc();
+ print traceback.print_exc();
print "FAILED GETNODES for host: %s" % host
continue
print "recently rebooted %s. skipping... " % host
continue
- if config.rins:
- # reset the boot_state to 'rins'
- node = api.GetNodes(host, ['boot_state', 'last_contact', 'last_updated', 'date_created'])
- record = {'observation' : node[0],
- 'model' : 'USER_REQUEST',
- 'action' : 'api.UpdateNode(%s, {"boot_state" : "rins"})' % host,
- 'time' : time.time()}
- l = Log(host, record)
-
- ret = api.UpdateNode(host, {'boot_state' : 'rins'})
- if ret:
- # it's nice to see the current status rather than the previous status on the console
- node = api.GetNodes(host)[0]
- print l
- print "%-2d" % (i-1), nodegroup_display(node, fb)
- rebootlog.add(l)
- else:
- print "FAILED TO UPDATE NODE BOOT STATE : %s" % host
-
-
if config.reboot:
fbnode = fb['nodes'][host]['values']
o = RebootDebug(fbnode)
elif observed_state == "boot" :
+ if config.rins:
+ l = set_node_to_rins(host, fb)
+ if l: rebootlog.add(l)
+
o = RebootBoot(fbnode)
elif observed_state == "down":
+ if config.rins:
+ l = set_node_to_rins(host, fb)
+ if l: rebootlog.add(l)
+
o = RebootDown(fbnode)
'time' : time.time()}
print "ALL METHODS OF RESTARTING %s FAILED" % host
+ args = {}
+ args['hostname'] = host
+ m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
+ "CANNOT CONTACT", False, db='suspect_persistmessages')
+ m.reset()
+ m.send(['monitor-list@lists.planet-lab.org'])
l = Log(host, record)
print l
print "Killed by interrupt"
sys.exit(0)
except:
- import traceback; print traceback.print_exc();
+ print traceback.print_exc();
print "Continuing..."
time.sleep(1)
git://git.onelab.eu / monitor.git / blobdiff