clearer names for actions, and infer actions better
[monitor.git] / monitor / bootman.py
index 25bd9f4..9754218 100755 (executable)
@@ -11,7 +11,7 @@ import traceback
 import subprocess
 from sets import Set
 
-from monitor.getsshkeys import SSHKnownHosts
+from monitor.util.sshknownhosts import SSHKnownHosts
 from monitor.Rpyc import SocketConnection, Async
 from monitor.Rpyc.Utils import *
 
@@ -57,7 +57,6 @@ def bootmanager_log_action(hostname, short_log_path, logtype="bm.log"):
                                                action_type=logtype,
                                                log_path=short_log_path,
                                                error_string=err)
-       session.flush(); session.clear()
        return
        
 
@@ -65,11 +64,13 @@ class ExceptionDoubleSSHError(Exception): pass
 
 class NodeConnection:
        def __init__(self, connection, node, config):
+               print "init nodeconnection"
                self.node = node
                self.c = connection
                self.config = config
 
        def get_boot_state(self):
+               print "get_boot_state(self)"
                try:
                        if self.c.modules.os.path.exists('/tmp/source'):
                                return "debug"
@@ -97,25 +98,12 @@ class NodeConnection:
        def get_bootmanager_log(self):
                bm_name = bootmanager_log_name(self.node)
                download(self.c, "/tmp/bm.log", "%s/%s" % (config.MONITOR_BOOTMANAGER_LOG, bm_name))
+               #email_exception(self.node, "collected BM log for %s" % self.node)
                bootmanager_log_action(self.node, bm_name, "collected_bm.log")
                os.system("cp %s/%s %s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, bm_name, config.MONITOR_BOOTMANAGER_LOG, self.node))
                log = open("%s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r')
                return log
 
-
-#      def get_dmesg(self):
-#              self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
-#              download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node)
-#              log = open("log/dmesg.%s.log" % self.node, 'r')
-#              return log
-#
-#      def get_bootmanager_log(self):
-#              download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node)
-#              #os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node))
-#              os.system("cp log/bm.%s.log.gz log/bm.%s.log" % (self.node, self.node))
-#              log = open("log/bm.%s.log" % self.node, 'r')
-#              return log
-
        def dump_plconf_file(self):
                c = self.c
                self.c.modules.sys.path.append("/tmp/source/")
@@ -141,6 +129,21 @@ class NodeConnection:
                                print key, " == ", bm.VARS[key]
                else:
                        print "   Unable to read Node Configuration"
+
+       def fprobe_repair_node(self):
+               # When fprobe data gets too much, it fills the root partition and
+               # fails to boot
+               c = self.c
+               self.c.modules.sys.path.append("/tmp/source/")
+
+               # NOTE: assume that the root fs is already mounted...
+               if self.c.modules.os.path.exists('/tmp/mnt/sysimg/var/local/fprobe'):
+                       print "CLEARING FPROBE DATA on %s" % self.node
+                       self.c.modules.os.chdir('/tmp/mnt/sysimg/var/local/fprobe')
+                       cmd = """ ls -lrt . | awk '{if (i<NR/2 && $9) {print "rm "$9;i=i+1;}}' | sh """
+                       self.c.modules.os.system(cmd)
+               else:
+                       print "COULD NOT CLEAR FPROBE DATA on %s" % self.node
                
        def fsck_repair_node(self):
                c = self.c
@@ -276,7 +279,10 @@ class PlanetLabSession:
 
        def get_connection(self, config):
                try:
-                       conn = NodeConnection(SocketConnection("localhost", self.port), self.node, config)
+                       print "SocketConnection(localhost, %s" % self.port
+                       sc = SocketConnection("localhost", self.port)
+                       print "NodeConnection(%s, %s)" % (sc, self.node)
+                       conn = NodeConnection(sc, self.node, config)
                except:
                        # NOTE: try twice since this can sometimes fail the first time. If
                        #               it fails again, let it go.
@@ -300,7 +306,7 @@ class PlanetLabSession:
 
                # COPY Rpyc files to host
                #cmd = "rsync -vvv -az -e ssh %(monitordir)s/Rpyc/ %(user)s@%(hostname)s:Rpyc 2> /dev/null" % args
-               cmd = """rsync -vvv -az -e "ssh -o BatchMode=yes" %(monitordir)s/Rpyc/ %(user)s@%(hostname)s:Rpyc""" % args
+               cmd = """rsync -vvv -az -e "ssh -o BatchMode=yes" %(monitordir)s/monitor/Rpyc/ %(user)s@%(hostname)s:Rpyc""" % args
                if self.verbose: print cmd
                print cmd
                # TODO: Add timeout
@@ -328,6 +334,7 @@ class PlanetLabSession:
                (ov,ev) = ssh.run_noexcept2("""<<\EOF
             rm -f out.log
             echo "kill server" >> out.log
+                       netstat -ap | grep python | grep 18812 | awk '{print $7}' | awk -F / '{print $1}' | xargs kill
             ps ax | grep Rpyc | grep -v grep | awk '{print $1}' | xargs kill 2> /dev/null ; 
             echo "export" >> out.log
             export PYTHONPATH=$HOME  ;
@@ -414,22 +421,27 @@ class DebugInterface:
                        email_exception(msg)
                        return False
 
+               print "Getting connection: 1st try"
                try:
                        conn = self.session.get_connection(config)
                except EOFError:
                        # NOTE: sometimes the wait in setup_host() is not long enough.  
                        # So, here we try to wait a little longer before giving up entirely.
                        try:
+                               print "Getting connection: 2nd try"
                                time.sleep(self.session.timeout*5)
                                conn = self.session.get_connection(config)
                        except EOFError:
                                # failed twice... no need to report this really, it's just in a
                                # weird state...
+                               print "Getting connection: failed"
+                               email_exception(self.hostname, "failed twice to get connection")
                                return False
                        except:
                                traceback.print_exc()
                                email_exception(self.hostname)
                                return False
+               print "Getting connection: ok"
                #print "trying to use conn before returning it."
                #print conn.c.modules.sys.path
                #print conn.c.modules.os.path.exists('/tmp/source')
@@ -440,176 +452,19 @@ class DebugInterface:
 
        def getSequences(self):
 
-               # TODO: This can be replaced with a DB definition at a future time.
-               #               This would make it possible for an admin to introduce new
-               #               patterns without touching code.
-               
+               # NOTE: The DB is now the autoritative record for all BM sequences. 
+               #               An admin can introduce new patterns and actions without touching code.
                sequences = {}
-               # restart_bootmanager_boot
-               for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-protoerror-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-implementerror-bootupdatefail-update-debug-done",
-
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-exception-protoerror-update-protoerror-debug-done",
-
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-exception-chrootfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-exception-protoerror-protoerror-debug-validate-done",
-                               "bminit-cfg-auth-protoerror-exception-update-debug-validate-exception-done",
-                               "bminit-cfg-auth-getplc-update-debug-done",
-                               "bminit-cfg-auth-protoerror2-debug-done",
-                               "bminit-cfg-auth-getplc-exception-protoerror-update-protoerror-debug-done",
-                               "bminit-cfg-auth-protoerror-exception-update-protoerror-debug-done",
-                               "bminit-cfg-auth-protoerror-exception-update-bootupdatefail-authfail-debug-done",
-                               "bminit-cfg-auth-protoerror-exception-update-debug-done",
-                               "bminit-cfg-auth-getplc-exception-protoerror-update-debug-done",
-                               "bminit-cfg-auth-getplc-implementerror-update-debug-done",
-                               "bminit-cfg-auth-authfail2-protoerror2-debug-done",
-                               ]:
-                       sequences.update({n : "restart_bootmanager_boot"})
-
-               #       conn.restart_bootmanager('reinstall')
-               for n in [ "bminit-cfg-auth-getplc-installinit-validate-exception-modulefail-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-exception-modulefail-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-done",
-                               "bminit-cfg-auth-getplc-hardware-installinit-installdisk-bziperror-exception-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done",
-                               "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-exception-mkfsfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-exception-chrootfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-installcfg-installstop-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-kernelcopyfail-exception-update-debug-done",
-                               "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-installcfg-installstop-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-kernelcopyfail-exception-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-exception-noinstall-update-debug-done",
-                               # actual solution appears to involve removing the bad files, and
-                               # continually trying to boot the node.
-                               "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-implementerror-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-exception-bmexceptmount-exception-noinstall-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-exception-bmexceptmount-exception-noinstall-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-validate-bmexceptvgscan-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-validate-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-validate-bmexceptvgscan-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-debug-validate-bmexceptvgscan-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-debug-validate-bmexceptvgscan-done",
-                               ]:
-                       sequences.update({n : "restart_bootmanager_rins"})
-
-               # repair_node_keys
-               for n in ["bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-validate-exception-done",
-                                       "bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-done",
-                                       "bminit-cfg-auth-bootcheckfail-authfail-exception-update-debug-validate-exception-done",
-                                       "bminit-cfg-auth-bootcheckfail-authfail-exception-authfail-debug-validate-exception-done",
-                                       "bminit-cfg-auth-authfail-debug-done",
-                                       "bminit-cfg-auth-authfail2-authfail-debug-done",
-                               ]:
-                       sequences.update({n: "repair_node_keys"})
-
-               #   conn.restart_node('reinstall')
-               for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-exception-chrootfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-installcfg-exception-chrootfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-writeerror-exception-chrootfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-hardware-installinit-exception-bmexceptrmfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-hardware-installinit-exception-bmexceptrmfail-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-implementerror-bootupdatefail-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-readonlyfs-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-nospace-exception-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done",
-                               "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done",
-                               ]:
-                       sequences.update({n : "restart_node_rins"})
-
-               #       restart_node_boot
-               for n in ["bminit-cfg-auth-getplc-implementerror-bootupdatefail-update-debug-done",
-                                "bminit-cfg-auth-implementerror-bootcheckfail-update-debug-done",
-                                "bminit-cfg-auth-implementerror-bootcheckfail-update-implementerror-bootupdatefail-done",
-                                "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done",
-                                "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done",
-                                "bminit-cfg-auth-getplc-update-installinit-validate-implementerror-update-debug-done",
-                                "bminit-cfg-auth-getplc-exception-update-bootupdatefail-debug-done",
-                                ]:
-                       sequences.update({n: "restart_node_boot"})
-
-               # fsck_repair
-               for n in ["bminit-cfg-auth-getplc-update-installinit-validate-fsckabort-exception-fsckfail-bmexceptmount-exception-noinstall-update-debug-validate-fsckabort-exception-fsckfail-bmexceptmount-done",
-                                 "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-noinstall-update-debug-validate-exception-fsckfail-done",
-                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail-exception-noinstall-update-debug-validate-exception-fsckfail-done",
-                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail2-exception-noinstall-update-debug-validate-exception-fsckfail2-done",
-                                 "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail2-exception-debug-validate-done",
-                                 "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail2-exception-debug-validate-exception-fsckfail2-done",
-                                 "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail2-exception-debug-validate-exception-fsckfail-done",
-                                 "bminit-cfg-auth-getplc-update-installinit-validate-fsckabort-exception-fsckfail-exception-debug-validate-fsckabort-exception-fsckfail-done",
-                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail2-exception-debug-validate-exception-fsckfail2-done",
-                                 "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-debug-validate-exception-fsckfail2-done",
-                                 "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-debug-validate-exception-fsckfail-done",
-                                 "bminit-cfg-auth-getplc-installinit-validate-exception-fsckfail-exception-debug-validate-done",
-                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-fsckfail-exception-debug-validate-exception-fsckfail-done",
-                                 "bminit-cfg-auth-getplc-update-debug-validate-exception-fsckfail-done",
-                               ]:
-                       sequences.update({n : "fsck_repair"})
-
-               # nodeconfig_notice
-               for n in ["bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-done",
-                                 "bminit-cfg-exception-update-bootupdatefail-nonode-debug-done",
-                                 "bminit-cfg-exception-update-bootupdatefail-nonode-debug-validate-exception-done",
-                                 "bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-validate-exception-done",
-                                 "bminit-cfg-auth-bootcheckfail-nonode-exception-update-bootupdatefail-nonode-debug-done",
-                                 "bminit-cfg-exception-noconfig-nonode-debug-validate-exception-done",
-                                 "bminit-cfg-exception-noconfig-update-debug-validate-exception-done",
-                               ]:
-                       sequences.update({n : "nodeconfig_notice"})
-
-               for n in [ "bminit-cfg-exception-nodehostname-update-debug-done", 
-                                  "bminit-cfg-update-exception-nodehostname-update-debug-validate-exception-done",
-                                  "bminit-cfg-update-exception-nodehostname-update-debug-done", 
-                                  "bminit-cfg-exception-nodehostname-debug-validate-exception-done",
-                               ]:
-                       sequences.update({n : "nodenetwork_email"})
-
-               # noblockdevice_notice
-               for n in ["bminit-cfg-auth-getplc-update-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-update-debug-validate-bmexceptvgscan-done",
-                               "bminit-cfg-auth-getplc-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done",
-                               "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done",
-                               "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done",
-                               "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-debug-validate-bmexceptvgscan-done",
-                               "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-debug-validate-bmexceptvgscan-done",
-                               ]:
-                       sequences.update({n : "noblockdevice_notice"})
-
-               # update_bootcd_email
-               for n in [ "bminit-cfg-auth-getplc-hardware-exception-hardwarerequirefail-update-debug-done",
-                               ]:
-                       sequences.update({n : "update_bootcd_email"})
-
-               for n in [ "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done",
-                               ]:
-                       sequences.update({n: "unknownsequence_notice"})
-
-               # minimalhardware_notice
-               sequences.update({"bminit-cfg-auth-getplc-hardware-exception-disktoosmall-hardwarerequirefail-update-debug-done" : "minimalhardware_notice"})
-               sequences.update({"bminit-cfg-auth-getplc-hardware-disktoosmall-exception-hardwarerequirefail-update-debug-done" : "minimalhardware_notice"})
-
-               # baddisk_notice
-               sequences.update({"bminit-cfg-auth-getplc-update-hardware-exception-hardwarerequirefail-update-debug-done" : "baddisk_notice"})
-
-               # baddns_notice
-               for n in [ 
-                "bminit-cfg-update-implementerror-bootupdatefail-dnserror-update-implementerror-bootupdatefail-dnserror-done",
-                       "bminit-cfg-auth-implementerror-bootcheckfail-dnserror-update-implementerror-bootupdatefail-dnserror-done",
-                       ]:
-                       sequences.update( { n : "baddns_notice"})
 
+               bms = BootmanSequenceRecord.query.all()
+               for s in bms:
+                       sequences[s.sequence] = s.action
+               
                return sequences
 
        def getDiskSteps(self):
                steps = [
+                       ('scsierror2' , 'sd \d:\d:\d:\d: ioctl_internal_command return code = \d+'),
                        ('scsierror'  , 'SCSI error : <\d+ \d+ \d+ \d+> return code = 0x\d+'),
                        ('ioerror'    , 'end_request: I/O error, dev sd\w+, sector \d+'),
                        ('ccisserror' , 'cciss: cmd \w+ has CHECK CONDITION'),
@@ -695,7 +550,9 @@ class DebugInterface:
                        ('noinstall'    , 'notinstalled'),
                        ('bziperror'    , 'bzip2: Data integrity error when decompressing.'),
                        ('noblockdev'   , "No block devices detected."),
+                       ('missingkernel', "missingkernel"),
                        ('dnserror'     , 'Name or service not known'),
+                       ('noparseconfig', "Found configuration file plnode.txt on floppy, but was unable to parse it"),
                        ('noconfig'             , "Unable to find and read a node configuration file"),
                        ('downloadfail' , 'Unable to download main tarball /boot/bootstrapfs-planetlab-i386.tar.bz2 from server.'),
                        ('disktoosmall' , 'The total usable disk size of all disks is insufficient to be usable as a PlanetLab node.'),
@@ -796,12 +653,16 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None):
 
                                log=conn.get_dmesg().read()
                                sitehist.sendMessage('baddisk_notice', hostname=hostname, log=log)
-                               #conn.set_nodestate('disabled')
+                               return "skipping_baddisk"
+                       else:
+                               # NOTE: "" does not add a new action record
+                               return ""
 
-                       return "skipping_baddisk"
 
        print "...Downloading bm.log from %s" %hostname 
        log = conn.get_bootmanager_log()
+       bm_log_data = log.read() # get data
+       log.seek(0)     # reset fd pointer for fdspawn
        child = fdpexpect.fdspawn(log)
 
        if hasattr(config, 'collect') and config.collect: return "collect"
@@ -831,15 +692,18 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None):
                args = {}
                args['hostname'] = hostname
                args['sequence'] = s
-               args['bmlog'] = conn.get_bootmanager_log().read()
+               args['bmlog'] = bm_log_data
                args['viart'] = False
                args['saveact'] = True
                args['ccemail'] = True
 
-               sitehist.sendMessage('unknownsequence_notice', **args)
+               if 'nospace' in s:
+                       # NOTE: sequence is unknown and contains nospace, so try the
+                       # fprobe repair trick first.
+                       conn.fprobe_repair_node()
 
+               sitehist.sendMessage('unknownsequence_notice', **args)
                conn.restart_bootmanager('boot')
-
                bootman_action = "restart_bootmanager"
 
                # NOTE: Do not set the pflags value for this sequence if it's unknown.
@@ -882,7 +746,7 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None):
                        args = {}
                        args['hostname'] = hostname
                        args['sequence'] = s
-                       args['bmlog'] = conn.get_bootmanager_log().read()
+                       args['bmlog'] = bm_log_data
                        args['viart'] = False
                        args['saveact'] = True
                        args['ccemail'] = True
@@ -906,7 +770,7 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None):
                        if not found_within(recent_actions, 'nodeconfig_notice', 3.5):
                                args = {}
                                args['hostname'] = hostname
-                               args['bmlog'] = conn.get_bootmanager_log().read()
+                               args['bmlog'] = bm_log_data
                                sitehist.sendMessage('nodeconfig_notice', **args)
                                conn.dump_plconf_file()
                        else:
@@ -947,7 +811,7 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None):
                                print "...NOTIFYING OWNERS OF MINIMAL HARDWARE FAILURE on %s!!!" % hostname
                                args = {}
                                args['hostname'] = hostname
-                               args['bmlog'] = conn.get_bootmanager_log().read()
+                               args['bmlog'] = bm_log_data
                                sitehist.sendMessage('minimalhardware_notice', **args)
                        else:
                                # NOTE: do not add a new action record
@@ -980,46 +844,5 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None):
        return bootman_action
        
 
-# MAIN -------------------------------------------------------------------
-
-def main():
-       from monitor import parser as parsermodule
-       parser = parsermodule.getParser()
-
-       parser.set_defaults(child=False, collect=False, nosetup=False, verbose=False, 
-                                               force=None, quiet=False)
-       parser.add_option("", "--child", dest="child", action="store_true", 
-                                               help="This is the child mode of this process.")
-       parser.add_option("", "--force", dest="force", metavar="boot_state",
-                                               help="Force a boot state passed to BootManager.py.")
-       parser.add_option("", "--quiet", dest="quiet", action="store_true", 
-                                               help="Extra quiet output messages.")
-       parser.add_option("", "--verbose", dest="verbose", action="store_true", 
-                                               help="Extra debug output messages.")
-       parser.add_option("", "--nonet", dest="nonet", action="store_true", 
-                                               help="Do not setup the network, use existing log files to re-run a test pass.")
-       parser.add_option("", "--collect", dest="collect", action="store_true", 
-                                               help="No action, just collect dmesg, and bm.log")
-       parser.add_option("", "--nosetup", dest="nosetup", action="store_true", 
-                                               help="Do not perform the orginary setup phase.")
-
-       parser = parsermodule.getParser(['nodesets', 'defaults'], parser)
-       config = parsermodule.parse_args(parser)
-
-       if config.nodelist:
-               nodes = config.getListFromFile(config.nodelist)
-       elif config.node:
-               nodes = [ config.node ]
-       else:
-               parser.print_help()
-               sys.exit(1)
-
-       for node in nodes:
-               # get sitehist
-               lb = plccache.plcdb_hn2lb[node]
-               sitehist = SiteInterface.get_or_make(loginbase=lb)
-               #reboot(node, config)
-               restore(sitehist, node, config=None, forced_action=None)
-
 if __name__ == "__main__":
-       main()
+       print "ERROR: Can not execute module as a command! Please use commands/%s.py" % os.path.splitext(__file__)[0]