X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=monitor%2Fbootman.py;h=975421871593f2677ff2317fe24916c268eda8a8;hb=32e47491837a321e684ea167ab6aa430145562f1;hp=25bd9f4282894afd25d2906e0e36ea02127bd079;hpb=45fea95bb898f254ea7e987d7417e9091885fbee;p=monitor.git diff --git a/monitor/bootman.py b/monitor/bootman.py index 25bd9f4..9754218 100755 --- a/monitor/bootman.py +++ b/monitor/bootman.py @@ -11,7 +11,7 @@ import traceback import subprocess from sets import Set -from monitor.getsshkeys import SSHKnownHosts +from monitor.util.sshknownhosts import SSHKnownHosts from monitor.Rpyc import SocketConnection, Async from monitor.Rpyc.Utils import * @@ -57,7 +57,6 @@ def bootmanager_log_action(hostname, short_log_path, logtype="bm.log"): action_type=logtype, log_path=short_log_path, error_string=err) - session.flush(); session.clear() return @@ -65,11 +64,13 @@ class ExceptionDoubleSSHError(Exception): pass class NodeConnection: def __init__(self, connection, node, config): + print "init nodeconnection" self.node = node self.c = connection self.config = config def get_boot_state(self): + print "get_boot_state(self)" try: if self.c.modules.os.path.exists('/tmp/source'): return "debug" @@ -97,25 +98,12 @@ class NodeConnection: def get_bootmanager_log(self): bm_name = bootmanager_log_name(self.node) download(self.c, "/tmp/bm.log", "%s/%s" % (config.MONITOR_BOOTMANAGER_LOG, bm_name)) + #email_exception(self.node, "collected BM log for %s" % self.node) bootmanager_log_action(self.node, bm_name, "collected_bm.log") os.system("cp %s/%s %s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, bm_name, config.MONITOR_BOOTMANAGER_LOG, self.node)) log = open("%s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r') return log - -# def get_dmesg(self): -# self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log") -# download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node) -# log = open("log/dmesg.%s.log" % self.node, 'r') -# return log -# -# def get_bootmanager_log(self): -# download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node) -# #os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node)) -# os.system("cp log/bm.%s.log.gz log/bm.%s.log" % (self.node, self.node)) -# log = open("log/bm.%s.log" % self.node, 'r') -# return log - def dump_plconf_file(self): c = self.c self.c.modules.sys.path.append("/tmp/source/") @@ -141,6 +129,21 @@ class NodeConnection: print key, " == ", bm.VARS[key] else: print " Unable to read Node Configuration" + + def fprobe_repair_node(self): + # When fprobe data gets too much, it fills the root partition and + # fails to boot + c = self.c + self.c.modules.sys.path.append("/tmp/source/") + + # NOTE: assume that the root fs is already mounted... + if self.c.modules.os.path.exists('/tmp/mnt/sysimg/var/local/fprobe'): + print "CLEARING FPROBE DATA on %s" % self.node + self.c.modules.os.chdir('/tmp/mnt/sysimg/var/local/fprobe') + cmd = """ ls -lrt . | awk '{if (i> out.log + netstat -ap | grep python | grep 18812 | awk '{print $7}' | awk -F / '{print $1}' | xargs kill ps ax | grep Rpyc | grep -v grep | awk '{print $1}' | xargs kill 2> /dev/null ; echo "export" >> out.log export PYTHONPATH=$HOME ; @@ -414,22 +421,27 @@ class DebugInterface: email_exception(msg) return False + print "Getting connection: 1st try" try: conn = self.session.get_connection(config) except EOFError: # NOTE: sometimes the wait in setup_host() is not long enough. # So, here we try to wait a little longer before giving up entirely. try: + print "Getting connection: 2nd try" time.sleep(self.session.timeout*5) conn = self.session.get_connection(config) except EOFError: # failed twice... no need to report this really, it's just in a # weird state... + print "Getting connection: failed" + email_exception(self.hostname, "failed twice to get connection") return False except: traceback.print_exc() email_exception(self.hostname) return False + print "Getting connection: ok" #print "trying to use conn before returning it." #print conn.c.modules.sys.path #print conn.c.modules.os.path.exists('/tmp/source') @@ -440,176 +452,19 @@ class DebugInterface: def getSequences(self): - # TODO: This can be replaced with a DB definition at a future time. - # This would make it possible for an admin to introduce new - # patterns without touching code. - + # NOTE: The DB is now the autoritative record for all BM sequences. + # An admin can introduce new patterns and actions without touching code. sequences = {} - # restart_bootmanager_boot - for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-protoerror-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-implementerror-bootupdatefail-update-debug-done", - - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-exception-protoerror-update-protoerror-debug-done", - - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-exception-chrootfail-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-exception-protoerror-protoerror-debug-validate-done", - "bminit-cfg-auth-protoerror-exception-update-debug-validate-exception-done", - "bminit-cfg-auth-getplc-update-debug-done", - "bminit-cfg-auth-protoerror2-debug-done", - "bminit-cfg-auth-getplc-exception-protoerror-update-protoerror-debug-done", - "bminit-cfg-auth-protoerror-exception-update-protoerror-debug-done", - "bminit-cfg-auth-protoerror-exception-update-bootupdatefail-authfail-debug-done", - "bminit-cfg-auth-protoerror-exception-update-debug-done", - "bminit-cfg-auth-getplc-exception-protoerror-update-debug-done", - "bminit-cfg-auth-getplc-implementerror-update-debug-done", - "bminit-cfg-auth-authfail2-protoerror2-debug-done", - ]: - sequences.update({n : "restart_bootmanager_boot"}) - - # conn.restart_bootmanager('reinstall') - for n in [ "bminit-cfg-auth-getplc-installinit-validate-exception-modulefail-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-modulefail-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-done", - "bminit-cfg-auth-getplc-hardware-installinit-installdisk-bziperror-exception-update-debug-done", - "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done", - "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done", - "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-exception-mkfsfail-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-exception-chrootfail-update-debug-done", - "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-installcfg-installstop-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-kernelcopyfail-exception-update-debug-done", - "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-installcfg-installstop-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-kernelcopyfail-exception-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-noinstall-update-debug-done", - # actual solution appears to involve removing the bad files, and - # continually trying to boot the node. - "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-implementerror-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-bmexceptmount-exception-noinstall-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-bmexceptmount-exception-noinstall-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-validate-bmexceptvgscan-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-validate-done", - "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-validate-bmexceptvgscan-done", - "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-debug-validate-bmexceptvgscan-done", - "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-debug-validate-bmexceptvgscan-done", - ]: - sequences.update({n : "restart_bootmanager_rins"}) - - # repair_node_keys - for n in ["bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-validate-exception-done", - "bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-done", - "bminit-cfg-auth-bootcheckfail-authfail-exception-update-debug-validate-exception-done", - "bminit-cfg-auth-bootcheckfail-authfail-exception-authfail-debug-validate-exception-done", - "bminit-cfg-auth-authfail-debug-done", - "bminit-cfg-auth-authfail2-authfail-debug-done", - ]: - sequences.update({n: "repair_node_keys"}) - - # conn.restart_node('reinstall') - for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-exception-chrootfail-update-debug-done", - "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-installcfg-exception-chrootfail-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-writeerror-exception-chrootfail-update-debug-done", - "bminit-cfg-auth-getplc-update-hardware-installinit-exception-bmexceptrmfail-update-debug-done", - "bminit-cfg-auth-getplc-hardware-installinit-exception-bmexceptrmfail-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-implementerror-bootupdatefail-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-readonlyfs-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-nospace-exception-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done", - "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-update-debug-done", - "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done", - ]: - sequences.update({n : "restart_node_rins"}) - - # restart_node_boot - for n in ["bminit-cfg-auth-getplc-implementerror-bootupdatefail-update-debug-done", - "bminit-cfg-auth-implementerror-bootcheckfail-update-debug-done", - "bminit-cfg-auth-implementerror-bootcheckfail-update-implementerror-bootupdatefail-done", - "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done", - "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done", - "bminit-cfg-auth-getplc-update-installinit-validate-implementerror-update-debug-done", - "bminit-cfg-auth-getplc-exception-update-bootupdatefail-debug-done", - ]: - sequences.update({n: "restart_node_boot"}) - - # fsck_repair - for n in ["bminit-cfg-auth-getplc-update-installinit-validate-fsckabort-exception-fsckfail-bmexceptmount-exception-noinstall-update-debug-validate-fsckabort-exception-fsckfail-bmexceptmount-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-noinstall-update-debug-validate-exception-fsckfail-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail-exception-noinstall-update-debug-validate-exception-fsckfail-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail2-exception-noinstall-update-debug-validate-exception-fsckfail2-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail2-exception-debug-validate-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail2-exception-debug-validate-exception-fsckfail2-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail2-exception-debug-validate-exception-fsckfail-done", - "bminit-cfg-auth-getplc-update-installinit-validate-fsckabort-exception-fsckfail-exception-debug-validate-fsckabort-exception-fsckfail-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail2-exception-debug-validate-exception-fsckfail2-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-debug-validate-exception-fsckfail2-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-debug-validate-exception-fsckfail-done", - "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-debug-validate-done", - "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail-exception-debug-validate-exception-fsckfail-done", - "bminit-cfg-auth-getplc-update-debug-validate-exception-fsckfail-done", - ]: - sequences.update({n : "fsck_repair"}) - - # nodeconfig_notice - for n in ["bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-done", - "bminit-cfg-exception-update-bootupdatefail-nonode-debug-done", - "bminit-cfg-exception-update-bootupdatefail-nonode-debug-validate-exception-done", - "bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-validate-exception-done", - "bminit-cfg-auth-bootcheckfail-nonode-exception-update-bootupdatefail-nonode-debug-done", - "bminit-cfg-exception-noconfig-nonode-debug-validate-exception-done", - "bminit-cfg-exception-noconfig-update-debug-validate-exception-done", - ]: - sequences.update({n : "nodeconfig_notice"}) - - for n in [ "bminit-cfg-exception-nodehostname-update-debug-done", - "bminit-cfg-update-exception-nodehostname-update-debug-validate-exception-done", - "bminit-cfg-update-exception-nodehostname-update-debug-done", - "bminit-cfg-exception-nodehostname-debug-validate-exception-done", - ]: - sequences.update({n : "nodenetwork_email"}) - - # noblockdevice_notice - for n in ["bminit-cfg-auth-getplc-update-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done", - "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-update-debug-validate-bmexceptvgscan-done", - "bminit-cfg-auth-getplc-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done", - "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done", - "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done", - "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-debug-validate-bmexceptvgscan-done", - "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-debug-validate-bmexceptvgscan-done", - ]: - sequences.update({n : "noblockdevice_notice"}) - - # update_bootcd_email - for n in [ "bminit-cfg-auth-getplc-hardware-exception-hardwarerequirefail-update-debug-done", - ]: - sequences.update({n : "update_bootcd_email"}) - - for n in [ "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done", - ]: - sequences.update({n: "unknownsequence_notice"}) - - # minimalhardware_notice - sequences.update({"bminit-cfg-auth-getplc-hardware-exception-disktoosmall-hardwarerequirefail-update-debug-done" : "minimalhardware_notice"}) - sequences.update({"bminit-cfg-auth-getplc-hardware-disktoosmall-exception-hardwarerequirefail-update-debug-done" : "minimalhardware_notice"}) - - # baddisk_notice - sequences.update({"bminit-cfg-auth-getplc-update-hardware-exception-hardwarerequirefail-update-debug-done" : "baddisk_notice"}) - - # baddns_notice - for n in [ - "bminit-cfg-update-implementerror-bootupdatefail-dnserror-update-implementerror-bootupdatefail-dnserror-done", - "bminit-cfg-auth-implementerror-bootcheckfail-dnserror-update-implementerror-bootupdatefail-dnserror-done", - ]: - sequences.update( { n : "baddns_notice"}) + bms = BootmanSequenceRecord.query.all() + for s in bms: + sequences[s.sequence] = s.action + return sequences def getDiskSteps(self): steps = [ + ('scsierror2' , 'sd \d:\d:\d:\d: ioctl_internal_command return code = \d+'), ('scsierror' , 'SCSI error : <\d+ \d+ \d+ \d+> return code = 0x\d+'), ('ioerror' , 'end_request: I/O error, dev sd\w+, sector \d+'), ('ccisserror' , 'cciss: cmd \w+ has CHECK CONDITION'), @@ -695,7 +550,9 @@ class DebugInterface: ('noinstall' , 'notinstalled'), ('bziperror' , 'bzip2: Data integrity error when decompressing.'), ('noblockdev' , "No block devices detected."), + ('missingkernel', "missingkernel"), ('dnserror' , 'Name or service not known'), + ('noparseconfig', "Found configuration file plnode.txt on floppy, but was unable to parse it"), ('noconfig' , "Unable to find and read a node configuration file"), ('downloadfail' , 'Unable to download main tarball /boot/bootstrapfs-planetlab-i386.tar.bz2 from server.'), ('disktoosmall' , 'The total usable disk size of all disks is insufficient to be usable as a PlanetLab node.'), @@ -796,12 +653,16 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): log=conn.get_dmesg().read() sitehist.sendMessage('baddisk_notice', hostname=hostname, log=log) - #conn.set_nodestate('disabled') + return "skipping_baddisk" + else: + # NOTE: "" does not add a new action record + return "" - return "skipping_baddisk" print "...Downloading bm.log from %s" %hostname log = conn.get_bootmanager_log() + bm_log_data = log.read() # get data + log.seek(0) # reset fd pointer for fdspawn child = fdpexpect.fdspawn(log) if hasattr(config, 'collect') and config.collect: return "collect" @@ -831,15 +692,18 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): args = {} args['hostname'] = hostname args['sequence'] = s - args['bmlog'] = conn.get_bootmanager_log().read() + args['bmlog'] = bm_log_data args['viart'] = False args['saveact'] = True args['ccemail'] = True - sitehist.sendMessage('unknownsequence_notice', **args) + if 'nospace' in s: + # NOTE: sequence is unknown and contains nospace, so try the + # fprobe repair trick first. + conn.fprobe_repair_node() + sitehist.sendMessage('unknownsequence_notice', **args) conn.restart_bootmanager('boot') - bootman_action = "restart_bootmanager" # NOTE: Do not set the pflags value for this sequence if it's unknown. @@ -882,7 +746,7 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): args = {} args['hostname'] = hostname args['sequence'] = s - args['bmlog'] = conn.get_bootmanager_log().read() + args['bmlog'] = bm_log_data args['viart'] = False args['saveact'] = True args['ccemail'] = True @@ -906,7 +770,7 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): if not found_within(recent_actions, 'nodeconfig_notice', 3.5): args = {} args['hostname'] = hostname - args['bmlog'] = conn.get_bootmanager_log().read() + args['bmlog'] = bm_log_data sitehist.sendMessage('nodeconfig_notice', **args) conn.dump_plconf_file() else: @@ -947,7 +811,7 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): print "...NOTIFYING OWNERS OF MINIMAL HARDWARE FAILURE on %s!!!" % hostname args = {} args['hostname'] = hostname - args['bmlog'] = conn.get_bootmanager_log().read() + args['bmlog'] = bm_log_data sitehist.sendMessage('minimalhardware_notice', **args) else: # NOTE: do not add a new action record @@ -980,46 +844,5 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): return bootman_action -# MAIN ------------------------------------------------------------------- - -def main(): - from monitor import parser as parsermodule - parser = parsermodule.getParser() - - parser.set_defaults(child=False, collect=False, nosetup=False, verbose=False, - force=None, quiet=False) - parser.add_option("", "--child", dest="child", action="store_true", - help="This is the child mode of this process.") - parser.add_option("", "--force", dest="force", metavar="boot_state", - help="Force a boot state passed to BootManager.py.") - parser.add_option("", "--quiet", dest="quiet", action="store_true", - help="Extra quiet output messages.") - parser.add_option("", "--verbose", dest="verbose", action="store_true", - help="Extra debug output messages.") - parser.add_option("", "--nonet", dest="nonet", action="store_true", - help="Do not setup the network, use existing log files to re-run a test pass.") - parser.add_option("", "--collect", dest="collect", action="store_true", - help="No action, just collect dmesg, and bm.log") - parser.add_option("", "--nosetup", dest="nosetup", action="store_true", - help="Do not perform the orginary setup phase.") - - parser = parsermodule.getParser(['nodesets', 'defaults'], parser) - config = parsermodule.parse_args(parser) - - if config.nodelist: - nodes = config.getListFromFile(config.nodelist) - elif config.node: - nodes = [ config.node ] - else: - parser.print_help() - sys.exit(1) - - for node in nodes: - # get sitehist - lb = plccache.plcdb_hn2lb[node] - sitehist = SiteInterface.get_or_make(loginbase=lb) - #reboot(node, config) - restore(sitehist, node, config=None, forced_action=None) - if __name__ == "__main__": - main() + print "ERROR: Can not execute module as a command! Please use commands/%s.py" % os.path.splitext(__file__)[0]