# * do something else to them all.
#
-import plc
+from monitor import config
+from monitor import util
+from monitor import const
+from monitor import database
+from monitor import parser as parsermodule
+from monitor.pcu import reboot
+from monitor.wrapper import plc
api = plc.getAuthAPI()
import traceback
-import config
-import util.file
from optparse import OptionParser
-import const
from nodecommon import *
from nodequery import verify,query_to_dict,node_select
-import database
-from unified_model import *
+from monitor.model import *
import os
import time
-import parser as parsermodule
-from model import *
import bootman # debug nodes
-import reboot # down nodes without pcu
-import mailmonitor # down nodes with pcu
+import mailmonitor # down nodes without pcu
from emailTxt import mailtxt
-#reboot.verbose = 0
import sys
class Reboot(object):
self.action = "reboot.reboot('%s')" % host
pflags = PersistFlags(host, 2*60*60*24, db='pcu_persistflags')
- pflags.resetRecentFlag('pcutried')
+ #pflags.resetRecentFlag('pcutried')
if not pflags.getRecentFlag('pcutried'):
- pflags.setRecentFlag('pcutried')
try:
print "CALLING REBOOT!!!"
ret = reboot.reboot(host)
+ pflags.setRecentFlag('pcutried')
+ pflags.save()
+ return ret
+
+ except Exception,e:
+ print traceback.print_exc(); print e
+
+ # NOTE: this failure could be an implementation issue on
+ # our end. So, extra notices are confusing...
+ # self._send_pcunotice(host)
+
+ pflags.setRecentFlag('pcufailed')
+ pflags.save()
+ return False
+
+ elif not pflags.getRecentFlag('pcu_rins_tried'):
+ try:
+ # set node to 'rins' boot state.
+ print "CALLING REBOOT +++ RINS"
+ plc.nodeBootState(host, 'rins')
+ ret = reboot.reboot(host)
+
+ pflags.setRecentFlag('pcu_rins_tried')
pflags.save()
return ret
pflags.setRecentFlag('pcumessagesent')
pflags.save()
- # NOTE: this will result in just one message sent at a time.
- return True
- else:
- print "GetRecentFlag()"
- return False
+ # This will result in mail() being called next, to try to
+ # engage the technical contact to take care of it also.
+ print "RETURNING FALSE"
+ return False
+
else:
print "NO PCUOK"
self.action = "None"
force=False,
nosetup=False,
verbose=False,
- stopkey=None,
- stopvalue=None,
quiet=False,
)
nodelist = api.GetNodes(ng[0]['node_ids'])
hostnames = [ n['hostname'] for n in nodelist ]
+if config.site:
+ site = api.GetSites(config.site)
+ l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
+ hostnames = [ n['hostname'] for n in l_nodes ]
+
if config.node or config.nodelist:
if config.node: hostnames = [ config.node ]
- else: hostnames = config.getListFromFile(config.nodelist)
+ else: hostnames = util.file.getListFromFile(config.nodelist)
-fb = database.dbLoad("findbad")
+fbquery = FindbadNodeRecord.get_all_latest()
+fb_nodelist = [ n.hostname for n in fbquery ]
if config.nodeselect:
- hostnames = node_select(config.nodeselect, fb['nodes'].keys(), fb)
+ hostnames = node_select(config.nodeselect, fb_nodelist)
if config.findbad:
# rerun findbad with the nodes in the given nodes.
file = "findbad.txt"
util.file.setFileFromList(file, hostnames)
- os.system("./findbad.py --cachenodes --debug=0 --dbname=findbad --increment --nodelist %s" % file)
+ os.system("./findbad.py --cachenodes --increment --nodelist %s" % file)
+ # TODO: shouldn't we reload the node list now?
+l_blacklist = database.if_cached_else(1, "l_blacklist", lambda : [])
# commands:
i = 1
count = 1
+#print "hosts: %s" % hostnames
for host in hostnames:
#if 'echo' in host or 'hptest-1' in host: continue
+
try:
try:
node = api.GetNodes(host)[0]
print "%-2d" % i, nodegroup_display(node, fb)
i += 1
if i-1 <= int(config.skip): continue
+ if host in l_blacklist:
+ print "%s is blacklisted. Skipping." % host
+ continue
if config.stopselect:
dict_query = query_to_dict(config.stopselect)
if verify(dict_query, fbnode) and observed_state != "dbg ":
# evaluates to true, therefore skip.
print "%s evaluates true for %s ; skipping..." % ( config.stopselect, host )
- continue
-
- if config.stopkey and config.stopvalue:
- fbnode = fb['nodes'][host]['values']
- observed_state = get_current_state(fbnode)
+ try:
+ # todo: clean up act_all record here.
+ # todo: send thank you, etc.
+ mailmonitor.reboot(host)
+ except Exception, e:
+ print traceback.print_exc(); print e
- if config.stopkey in fbnode:
- if config.stopvalue in fbnode[config.stopkey] and observed_state != "dbg ":
- print "%s has stopvalue; skipping..." % host
- continue
- else:
- print "stopkey %s not in fbnode record for %s; skipping..." % (config.stopkey, host)
- print fbnode
continue
+ #else:
+ #print "%s failed to match %s: -%s-" % ( host, dict_query, observed_state )
+ #sys.exit(1)
if not config.force and rebootlog.find(host, {'action' : ".*reboot"}, 60*60*2):
print "recently rebooted %s. skipping... " % host
print "ALL METHODS OF RESTARTING %s FAILED" % host
args = {}
args['hostname'] = host
- m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
- "CANNOT CONTACT", False, db='suspect_persistmessages')
- m.reset()
- m.send(['monitor-list@lists.planet-lab.org'])
+ #m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
+ # "CANNOT CONTACT", False, db='suspect_persistmessages')
+ #m.reset()
+ #m.send(['monitor-list@lists.planet-lab.org'])
l = Log(host, record)
print l
git://git.onelab.eu / monitor.git / blobdiff