monitor/bootman.py

   1 #!/usr/bin/python
   2
   3 # Attempt to reboot a node in debug state.
   4
   5
   6
   7 import os
   8 import sys
   9 import time
  10 import random
  11 import signal
  12 import traceback
  13 import subprocess
  14 from sets import Set
  15
  16 from monitor.getsshkeys import SSHKnownHosts
  17
  18 from monitor.Rpyc import SocketConnection, Async
  19 from monitor.Rpyc.Utils import *
  20
  21 from monitor import getconf
  22 from monitor import config
  23 from monitor import const
  24 from monitor.model import *
  25 from monitor.common import email_exception, found_within
  26 from monitor.database.info.model import *
  27 from monitor.database.info.interface import *
  28 from monitor.wrapper import plc
  29 from monitor.wrapper import plccache
  30 from monitor.wrapper.emailTxt import mailtxt
  31 from monitor.nodeconfig import network_config_to_str
  32
  33 from pcucontrol.util import command as moncommands
  34 from pcucontrol.util.command import Sopen
  35 from pcucontrol.transports.ssh import pxssh as pxssh
  36 from pcucontrol.transports.ssh import fdpexpect as fdpexpect
  37 from pcucontrol.transports.ssh import pexpect as pexpect
  38
  39
  40
  41 api = plc.getAuthAPI()
  42 fb = None
  43
  44
  45 class ExceptionDoubleSSHError(Exception): pass
  46
  47 class NodeConnection:
  48         def __init__(self, connection, node, config):
  49                 self.node = node
  50                 self.c = connection
  51                 self.config = config
  52
  53         def get_boot_state(self):
  54                 try:
  55                         if self.c.modules.os.path.exists('/tmp/source'):
  56                                 return "debug"
  57                         elif self.c.modules.os.path.exists('/vservers'):
  58                                 return "boot"
  59                         else:
  60                                 return "unknown"
  61                 except EOFError:
  62                         traceback.print_exc()
  63                         print self.c.modules.sys.path
  64                 except:
  65                         email_exception()
  66                         traceback.print_exc()
  67
  68                 return "unknown"
  69
  70         def get_dmesg(self):
  71                 t_stamp = time.strftime("%Y-%m-%d-%H:%M")
  72                 self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
  73                 download(self.c, "/var/log/dmesg.bm.log", "%s/history/%s-dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node))
  74                 os.system("cp %s/history/%s-dmesg.%s.log %s/dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node, config.MONITOR_BOOTMANAGER_LOG, self.node))
  75                 log = open("%s/dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r')
  76                 return log
  77
  78         def get_bootmanager_log(self):
  79                 t_stamp = time.strftime("%Y-%m-%d-%H:%M")
  80                 download(self.c, "/tmp/bm.log", "%s/history/%s-bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node))
  81                 os.system("cp %s/history/%s-bm.%s.log %s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node, config.MONITOR_BOOTMANAGER_LOG, self.node))
  82                 log = open("%s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r')
  83                 return log
  84
  85
  86 #       def get_dmesg(self):
  87 #               self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
  88 #               download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node)
  89 #               log = open("log/dmesg.%s.log" % self.node, 'r')
  90 #               return log
  91 #
  92 #       def get_bootmanager_log(self):
  93 #               download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node)
  94 #               #os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node))
  95 #               os.system("cp log/bm.%s.log.gz log/bm.%s.log" % (self.node, self.node))
  96 #               log = open("log/bm.%s.log" % self.node, 'r')
  97 #               return log
  98
  99         def dump_plconf_file(self):
 100                 c = self.c
 101                 self.c.modules.sys.path.append("/tmp/source/")
 102                 self.c.modules.os.chdir('/tmp/source')
 103
 104                 log = c.modules.BootManager.log('/tmp/new.log')
 105                 bm = c.modules.BootManager.BootManager(log,'boot')
 106
 107                 BootManagerException = c.modules.Exceptions.BootManagerException
 108                 InitializeBootManager = c.modules.BootManager.InitializeBootManager
 109                 ReadNodeConfiguration = c.modules.BootManager.ReadNodeConfiguration
 110                 bm_continue = True
 111
 112                 InitializeBootManager.Run(bm.VARS, bm.LOG)
 113                 try: ReadNodeConfiguration.Run(bm.VARS, bm.LOG)
 114                 except Exception, x:
 115                         bm_continue = False
 116                         print "   ERROR:", x
 117                         print "   Possibly, unable to find valid configuration file"
 118
 119                 if bm_continue:
 120                         for key in bm.VARS.keys():
 121                                 print key, " == ", bm.VARS[key]
 122                 else:
 123                         print "   Unable to read Node Configuration"
 124
 125
 126         def compare_and_repair_nodekeys(self):
 127                 c = self.c
 128                 self.c.modules.sys.path.append("/tmp/source/")
 129                 self.c.modules.os.chdir('/tmp/source')
 130
 131                 log = c.modules.BootManager.log('/tmp/new.log')
 132                 bm = c.modules.BootManager.BootManager(log,'boot')
 133
 134                 BootManagerException = c.modules.Exceptions.BootManagerException
 135                 InitializeBootManager = c.modules.BootManager.InitializeBootManager
 136                 ReadNodeConfiguration = c.modules.BootManager.ReadNodeConfiguration
 137                 bm_continue = True
 138
 139                 plcnode = plccache.GetNodeByName(self.node)
 140
 141                 InitializeBootManager.Run(bm.VARS, bm.LOG)
 142                 try: ReadNodeConfiguration.Run(bm.VARS, bm.LOG)
 143                 except Exception, x:
 144                         bm_continue = False
 145                         print "exception"
 146                         print x
 147                         print "   Possibly, unable to find valid configuration file"
 148
 149                 if bm_continue:
 150                         print "   NODE: %s" % bm.VARS['NODE_KEY']
 151                         print "   PLC : %s" % plcnode['key']
 152
 153                         if bm.VARS['NODE_KEY'] == plcnode['key']:
 154                                 return True
 155                         else:
 156                                 if api.UpdateNode(self.node, {'key': bm.VARS['NODE_KEY']}):
 157                                         print "   Successfully updated NODE_KEY with PLC"
 158                                         return True
 159                                 else:
 160                                         return False
 161
 162                         #for key in bm.VARS.keys():
 163                         #       print key, " == ", bm.VARS[key]
 164                 else:
 165                         print "   Unable to retrieve NODE_KEY"
 166
 167         def bootmanager_running(self):
 168                 if self.c.modules.os.path.exists('/tmp/BM_RUNNING'):
 169                         return True
 170                 else:
 171                         return False
 172
 173         def set_nodestate(self, state='boot'):
 174                 return api.UpdateNode(self.node, {'boot_state' : state})
 175
 176         def restart_node(self, state='boot'):
 177                 api.UpdateNode(self.node, {'boot_state' : state})
 178
 179                 pflags = PersistFlags(self.node, 1*60*60*24, db='restart_persistflags')
 180                 if not pflags.getRecentFlag('gentlekill'):
 181                         print "   Killing all slice processes... : %s" %  self.node
 182                         cmd_slicekill = "ls -d /proc/virtual/[0-9]* | awk -F '/' '{print $4}' | xargs -I{} /usr/sbin/vkill -s 9 --xid {} -- 0"
 183                         self.c.modules.os.system(cmd_slicekill)
 184                         cmd = """ shutdown -r +1 & """
 185                         print "   Restarting %s : %s" % ( self.node, cmd)
 186                         self.c.modules.os.system(cmd)
 187
 188                         pflags.setRecentFlag('gentlekill')
 189                         pflags.save()
 190                 else:
 191                         print "   Restarting with sysrq 'sub' %s" % self.node
 192                         cmd = """ (sleep 5; echo 's' > /proc/sysrq-trigger; echo 'u' > /proc/sysrq-trigger; echo 'b' > /proc/sysrq-trigger ) & """
 193                         self.c.modules.os.system(cmd)
 194
 195                 return
 196
 197         def restart_bootmanager(self, forceState):
 198
 199                 self.c.modules.os.chdir('/tmp/source')
 200                 if self.c.modules.os.path.exists('/tmp/BM_RUNNING'):
 201                         print "   BootManager is already running: try again soon..."
 202                 else:
 203                         print "   Starting 'BootManager.py %s' on %s " % (forceState, self.node)
 204                         cmd = "( touch /tmp/BM_RUNNING ;  " + \
 205                               "  python ./BootManager.py %s &> server.log < /dev/null ; " + \
 206                                   "  rm -f /tmp/BM_RUNNING " + \
 207                                   ") &"
 208                         cmd = cmd % forceState
 209                         self.c.modules.os.system(cmd)
 210
 211                 return
 212
 213
 214 class PlanetLabSession:
 215         globalport = 22000 + int(random.random()*1000)
 216
 217         def __init__(self, node, nosetup, verbose):
 218                 self.verbose = verbose
 219                 self.node = node
 220                 self.port = None
 221                 self.nosetup = nosetup
 222                 self.command = None
 223                 self.setup_host()
 224
 225         def get_connection(self, config):
 226                 conn = NodeConnection(SocketConnection("localhost", self.port), self.node, config)
 227                 #i = 0
 228                 #while i < 3:
 229                 #       print i, conn.c.modules.sys.path
 230                 #       print conn.c.modules.os.path.exists('/tmp/source')
 231                 #       i+=1
 232                 #       time.sleep(1)
 233                 return conn
 234
 235         def setup_host(self):
 236                 self.port = PlanetLabSession.globalport
 237                 PlanetLabSession.globalport = PlanetLabSession.globalport + 1
 238
 239                 args = {}
 240                 args['port'] = self.port
 241                 args['user'] = 'root'
 242                 args['hostname'] = self.node
 243                 args['monitordir'] = config.MONITOR_SCRIPT_ROOT
 244                 ssh_port = 22
 245
 246                 if self.nosetup:
 247                         print "Skipping setup"
 248                         return
 249
 250                 # COPY Rpyc files to host
 251                 cmd = "rsync -qv -az -e ssh %(monitordir)s/Rpyc/ %(user)s@%(hostname)s:Rpyc 2> /dev/null" % args
 252                 if self.verbose: print cmd
 253                 print cmd
 254                 # TODO: Add timeout
 255                 timeout = 120
 256                 localos = moncommands.CMD()
 257
 258                 ret = localos.system(cmd, timeout)
 259                 print ret
 260                 if ret != 0:
 261                         print "\tUNKNOWN SSH KEY FOR %s; making an exception" % self.node
 262                         #print "MAKE EXPLICIT EXCEPTION FOR %s" % self.node
 263                         k = SSHKnownHosts(); k.updateDirect(self.node); k.write(); del k
 264                         ret = localos.system(cmd, timeout)
 265                         print ret
 266                         if ret != 0:
 267                                 print "\tFAILED TWICE"
 268                                 #sys.exit(1)
 269                                 raise ExceptionDoubleSSHError("Failed twice trying to login with updated ssh host key")
 270
 271                 t1 = time.time()
 272                 # KILL any already running servers.
 273                 ssh = moncommands.SSH(args['user'], args['hostname'], ssh_port)
 274                 (ov,ev) = ssh.run_noexcept2("""<<\EOF
 275             rm -f out.log
 276             echo "kill server" >> out.log
 277             ps ax | grep Rpyc | grep -v grep | awk '{print $1}' | xargs kill 2> /dev/null ;
 278             echo "export" >> out.log
 279             export PYTHONPATH=$HOME  ;
 280             echo "start server" >> out.log
 281             python Rpyc/Servers/forking_server.py &> server.log &
 282             echo "done" >> out.log
 283 EOF""")
 284                 #cmd = """ssh %(user)s@%(hostname)s """ + \
 285                 #        """'ps ax | grep Rpyc | grep -v grep | awk "{print \$1}" | xargs kill 2> /dev/null' """
 286                 #cmd = cmd % args
 287                 #if self.verbose: print cmd
 288                 ## TODO: Add timeout
 289                 #print localos.system(cmd,timeout)
 290
 291                 ## START a new rpyc server.
 292                 #cmd = """ssh -n %(user)s@%(hostname)s "export PYTHONPATH=\$HOME; """ + \
 293                 #        """python Rpyc/Servers/forking_server.py &> server.log < /dev/null &" """
 294                 #cmd = cmd % args
 295                 #if self.verbose: print cmd
 296                 #print localos.system(cmd,timeout)
 297                 print "setup rpyc server over ssh"
 298                 print ssh.ret
 299
 300                 # TODO: Add timeout
 301                 # This was tricky to make synchronous.  The combination of ssh-clients-4.7p1,
 302                 # and the following options seems to work well.
 303                 cmd = """ssh -o ExitOnForwardFailure=yes -o BatchMode=yes """ + \
 304                           """-o PermitLocalCommand=yes -o LocalCommand='echo "READY"' """ + \
 305                           """-o ConnectTimeout=120 """ + \
 306                           """-n -N -L %(port)s:localhost:18812 """ + \
 307                           """%(user)s@%(hostname)s"""
 308                 cmd = cmd % args
 309                 if self.verbose: print cmd
 310                 print cmd
 311                 self.command = Sopen(cmd, shell=True, stdout=subprocess.PIPE)
 312                 # TODO: the read() here may block indefinitely.  Need a better
 313                 # approach therefore, that includes a timeout.
 314                 #ret = self.command.stdout.read(5)
 315                 ret = moncommands.read_t(self.command.stdout, 5)
 316
 317                 t2 = time.time()
 318                 if 'READY' in ret:
 319                         # NOTE: There is still a slight race for machines that are slow...
 320                         self.timeout = 2*(t2-t1)
 321                         print "Sleeping for %s sec" % self.timeout
 322                         time.sleep(self.timeout)
 323                         return
 324
 325                 if self.command.returncode is not None:
 326                         print "Failed to establish tunnel!"
 327                         raise Exception("SSH Tunnel exception : %s %s" % (self.node, self.command.returncode))
 328
 329                 raise Exception("Unknown SSH Tunnel Exception: still running, but did not report 'READY'")
 330
 331         def __del__(self):
 332                 if self.command:
 333                         if self.verbose: print "Killing SSH session %s" % self.port
 334                         print "Killing SSH session %s" % self.port
 335                         self.command.kill()
 336
 337
 338 def steps_to_list(steps, index=1):
 339         return map(lambda x: x[index], steps)
 340
 341 def index_to_id(steps,index):
 342         if index < len(steps):
 343                 return steps[index][0]
 344         else:
 345                 return "done"
 346
 347 class DebugInterface:
 348         def __init__(self, hostname):
 349                 self.hostname = hostname
 350                 self.session = None
 351
 352         def getConnection(self):
 353                 print "Creating session for %s" % self.hostname
 354                 # update known_hosts file (in case the node has rebooted since last run)
 355                 try:
 356                         k = SSHKnownHosts(); k.update(self.hostname); k.write(); del k
 357                 except:
 358                         email_exception()
 359                         print traceback.print_exc()
 360                         return False
 361
 362                 try:
 363                         if config == None:
 364                                 self.session = PlanetLabSession(self.hostname, False, True)
 365                         else:
 366                                 self.session = PlanetLabSession(self.hostname, config.nosetup, config.verbose)
 367                 except ExceptionDoubleSSHError, e:
 368                         msg = "ERROR setting up session for %s" % self.hostname
 369                         print msg
 370                         return False
 371                 except Exception, e:
 372                         traceback.print_exc()
 373                         email_exception(msg)
 374                         return False
 375
 376                 try:
 377                         conn = self.session.get_connection(config)
 378                 except EOFError:
 379                         # NOTE: sometimes the wait in setup_host() is not long enough.
 380                         # So, here we try to wait a little longer before giving up entirely.
 381                         try:
 382                                 time.sleep(self.session.timeout*5)
 383                                 conn = self.session.get_connection(config)
 384                         except EOFError:
 385                                 # failed twice... no need to report this really, it's just in a
 386                                 # weird state...
 387                                 return False
 388                         except:
 389                                 traceback.print_exc()
 390                                 email_exception(self.hostname)
 391                                 return False
 392                 #print "trying to use conn before returning it."
 393                 #print conn.c.modules.sys.path
 394                 #print conn.c.modules.os.path.exists('/tmp/source')
 395                 #time.sleep(1)
 396
 397                 #print "conn: %s" % conn
 398                 return conn
 399
 400         def getSequences(self):
 401
 402                 # TODO: This can be replaced with a DB definition at a future time.
 403                 #               This would make it possible for an admin to introduce new
 404                 #               patterns without touching code.
 405
 406                 sequences = {}
 407                 # restart_bootmanager_boot
 408                 for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-done",
 409                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-protoerror-debug-done",
 410                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-implementerror-bootupdatefail-update-debug-done",
 411
 412                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-exception-protoerror-update-protoerror-debug-done",
 413
 414                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-debug-done",
 415                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-exception-chrootfail-update-debug-done",
 416                                 "bminit-cfg-auth-getplc-update-debug-done",
 417                                 "bminit-cfg-auth-getplc-exception-protoerror-update-protoerror-debug-done",
 418                                 "bminit-cfg-auth-protoerror-exception-update-protoerror-debug-done",
 419                                 "bminit-cfg-auth-protoerror-exception-update-bootupdatefail-authfail-debug-done",
 420                                 "bminit-cfg-auth-protoerror-exception-update-debug-done",
 421                                 "bminit-cfg-auth-getplc-exception-protoerror-update-debug-done",
 422                                 "bminit-cfg-auth-getplc-implementerror-update-debug-done",
 423                                 ]:
 424                         sequences.update({n : "restart_bootmanager_boot"})
 425
 426                 #       conn.restart_bootmanager('reinstall')
 427                 for n in [ "bminit-cfg-auth-getplc-installinit-validate-exception-modulefail-update-debug-done",
 428                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-modulefail-update-debug-done",
 429                                 "bminit-cfg-auth-getplc-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done",
 430                                 "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done",
 431                                 "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done",
 432                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-done",
 433                                 "bminit-cfg-auth-getplc-hardware-installinit-installdisk-bziperror-exception-update-debug-done",
 434                                 "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-update-debug-done",
 435                                 "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done",
 436                                 "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-update-debug-done",
 437                                 "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done",
 438                                 "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-exception-mkfsfail-update-debug-done",
 439                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done",
 440                                 "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-exception-chrootfail-update-debug-done",
 441                                 "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-installcfg-installstop-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-kernelcopyfail-exception-update-debug-done",
 442                                 "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-installcfg-installstop-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-kernelcopyfail-exception-update-debug-done",
 443                                 "bminit-cfg-auth-getplc-installinit-validate-exception-noinstall-update-debug-done",
 444                                 # actual solution appears to involve removing the bad files, and
 445                                 # continually trying to boot the node.
 446                                 "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-implementerror-update-debug-done",
 447                                 "bminit-cfg-auth-getplc-installinit-validate-exception-bmexceptmount-exception-noinstall-update-debug-done",
 448                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-bmexceptmount-exception-noinstall-update-debug-done",
 449                                 "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-validate-bmexceptvgscan-done",
 450                                 "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-validate-done",
 451                                 ]:
 452                         sequences.update({n : "restart_bootmanager_rins"})
 453
 454                 # repair_node_keys
 455                 for n in ["bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-validate-exception-done",
 456                                         "bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-done",
 457                                 ]:
 458                         sequences.update({n: "repair_node_keys"})
 459
 460                 #   conn.restart_node('reinstall')
 461                 for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done",
 462                                 "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-exception-chrootfail-update-debug-done",
 463                                 "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-installcfg-exception-chrootfail-update-debug-done",
 464                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-writeerror-exception-chrootfail-update-debug-done",
 465                                 "bminit-cfg-auth-getplc-update-hardware-installinit-exception-bmexceptrmfail-update-debug-done",
 466                                 "bminit-cfg-auth-getplc-hardware-installinit-exception-bmexceptrmfail-update-debug-done",
 467                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-implementerror-bootupdatefail-update-debug-done",
 468                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-readonlyfs-update-debug-done",
 469                                 "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-nospace-exception-update-debug-done",
 470                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done",
 471                                 "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-update-debug-done",
 472                                 "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done",
 473                                 ]:
 474                         sequences.update({n : "restart_node_rins"})
 475
 476                 #       restart_node_boot
 477                 for n in ["bminit-cfg-auth-getplc-implementerror-bootupdatefail-update-debug-done",
 478                                  "bminit-cfg-auth-implementerror-bootcheckfail-update-debug-done",
 479                                  "bminit-cfg-auth-implementerror-bootcheckfail-update-implementerror-bootupdatefail-done",
 480                                  "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done",
 481                                  "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done",
 482                                  "bminit-cfg-auth-getplc-update-installinit-validate-implementerror-update-debug-done",
 483                                  "bminit-cfg-auth-getplc-exception-update-bootupdatefail-debug-done",
 484                                  ]:
 485                         sequences.update({n: "restart_node_boot"})
 486
 487                 # update_node_config_email
 488                 for n in ["bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-done",
 489                                   "bminit-cfg-exception-update-bootupdatefail-nonode-debug-done",
 490                                   "bminit-cfg-exception-update-bootupdatefail-nonode-debug-validate-exception-done",
 491                                   "bminit-cfg-auth-bootcheckfail-nonode-exception-update-bootupdatefail-nonode-debug-done",
 492                                 ]:
 493                         sequences.update({n : "update_node_config_email"})
 494
 495                 for n in [ "bminit-cfg-exception-nodehostname-update-debug-done",
 496                                    "bminit-cfg-update-exception-nodehostname-update-debug-done",
 497                                 ]:
 498                         sequences.update({n : "nodenetwork_email"})
 499
 500                 # update_bootcd_email
 501                 for n in ["bminit-cfg-auth-getplc-update-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done",
 502                                 "bminit-cfg-auth-getplc-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done",
 503                                 "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done",
 504                                 "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done",
 505                                 "bminit-cfg-auth-getplc-hardware-exception-hardwarerequirefail-update-debug-done",
 506                                 ]:
 507                         sequences.update({n : "update_bootcd_email"})
 508
 509                 for n in [ "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done",
 510                                 ]:
 511                         sequences.update({n: "suspect_error_email"})
 512
 513                 # update_hardware_email
 514                 sequences.update({"bminit-cfg-auth-getplc-hardware-exception-disktoosmall-hardwarerequirefail-update-debug-done" : "update_hardware_email"})
 515                 sequences.update({"bminit-cfg-auth-getplc-hardware-disktoosmall-exception-hardwarerequirefail-update-debug-done" : "update_hardware_email"})
 516
 517                 # broken_hardware_email
 518                 sequences.update({"bminit-cfg-auth-getplc-update-hardware-exception-hardwarerequirefail-update-debug-done" : "broken_hardware_email"})
 519
 520                 # bad_dns_email
 521                 for n in [
 522                  "bminit-cfg-update-implementerror-bootupdatefail-dnserror-update-implementerror-bootupdatefail-dnserror-done",
 523                         "bminit-cfg-auth-implementerror-bootcheckfail-dnserror-update-implementerror-bootupdatefail-dnserror-done",
 524                         ]:
 525                         sequences.update( { n : "bad_dns_email"})
 526
 527                 return sequences
 528
 529         def getDiskSteps(self):
 530                 steps = [
 531                         ('scsierror'  , 'SCSI error : <\d+ \d+ \d+ \d+> return code = 0x\d+'),
 532                         ('ioerror'    , 'end_request: I/O error, dev sd\w+, sector \d+'),
 533                         ('ccisserror' , 'cciss: cmd \w+ has CHECK CONDITION  byte \w+ = \w+'),
 534
 535                         ('buffererror', 'Buffer I/O error on device dm-\d, logical block \d+'),
 536
 537                         ('hdaseekerror', 'hda: dma_intr: status=0x\d+ { DriveReady SeekComplete Error }'),
 538                         ('hdacorrecterror', 'hda: dma_intr: error=0x\d+ { UncorrectableError }, LBAsect=\d+, sector=\d+'),
 539
 540                         ('atareadyerror'   , 'ata\d+: status=0x\d+ { DriveReady SeekComplete Error }'),
 541                         ('atacorrecterror' , 'ata\d+: error=0x\d+ { UncorrectableError }'),
 542
 543                         ('sdXerror'   , 'sd\w: Current: sense key: Medium Error'),
 544                         ('ext3error'   , 'EXT3-fs error (device dm-\d+): ext3_find_entry: reading directory #\d+ offset \d+'),
 545
 546                         ('floppytimeout','floppy0: floppy timeout called'),
 547                         ('floppyerror',  'end_request: I/O error, dev fd\w+, sector \d+'),
 548
 549                         # hda: dma_intr: status=0x51 { DriveReady SeekComplete Error }
 550                         # hda: dma_intr: error=0x40 { UncorrectableError }, LBAsect=23331263, sector=23331263
 551
 552                         # floppy0: floppy timeout called
 553                         # end_request: I/O error, dev fd0, sector 0
 554
 555                         # Buffer I/O error on device dm-2, logical block 8888896
 556                         # ata1: status=0x51 { DriveReady SeekComplete Error }
 557                         # ata1: error=0x40 { UncorrectableError }
 558                         # SCSI error : <0 0 0 0> return code = 0x8000002
 559                         # sda: Current: sense key: Medium Error
 560                         #       Additional sense: Unrecovered read error - auto reallocate failed
 561
 562                         # SCSI error : <0 2 0 0> return code = 0x40001
 563                         # end_request: I/O error, dev sda, sector 572489600
 564                 ]
 565                 return steps
 566
 567         def getDiskSequence(self, steps, child):
 568                 sequence = []
 569                 while True:
 570                         id = index_to_id(steps, child.expect( steps_to_list(steps) + [ pexpect.EOF ]))
 571                         sequence.append(id)
 572
 573                         if id == "done":
 574                                 break
 575                 return sequence
 576
 577         def getBootManagerStepPatterns(self):
 578                 steps = [
 579                         ('bminit'               , 'Initializing the BootManager.'),
 580                         ('cfg'                  , 'Reading node configuration file.'),
 581                         ('auth'                 , 'Authenticating node with PLC.'),
 582                         ('getplc'               , 'Retrieving details of node from PLC.'),
 583                         ('update'               , 'Updating node boot state at PLC.'),
 584                         ('hardware'             , 'Checking if hardware requirements met.'),
 585                         ('installinit'  , 'Install: Initializing.'),
 586                         ('installdisk'  , 'Install: partitioning disks.'),
 587                         ('installbootfs', 'Install: bootstrapfs tarball.'),
 588                         ('installcfg'   , 'Install: Writing configuration files.'),
 589                         ('installstop'  , 'Install: Shutting down installer.'),
 590                         ('update2'              , 'Updating node boot state at PLC.'),
 591                         ('installinit2' , 'Install: Initializing.'),
 592                         ('validate'             , 'Validating node installation.'),
 593                         ('rebuildinitrd', 'Rebuilding initrd'),
 594                         ('netcfg'               , 'Install: Writing Network Configuration files.'),
 595                         ('update3'              , 'Updating node configuration.'),
 596                         ('disk'                 , 'Checking for unused disks to add to LVM.'),
 597                         ('update4'              , 'Sending hardware configuration to PLC.'),
 598                         ('debug'                , 'Starting debug mode'),
 599                         ('bmexceptmount', 'BootManagerException during mount'),
 600                         ('bmexceptvgscan', 'BootManagerException during vgscan/vgchange'),
 601                         ('bmexceptrmfail', 'Unable to remove directory tree: /tmp/mnt'),
 602                         ('exception'    , 'Exception'),
 603                         ('nocfg'        , 'Found configuration file planet.cnf on floppy, but was unable to parse it.'),
 604                         ('protoerror'   , 'XML RPC protocol error'),
 605                         ('nodehostname' , 'Configured node hostname does not resolve'),
 606                         ('implementerror', 'Implementation Error'),
 607                         ('readonlyfs'   , '[Errno 30] Read-only file system'),
 608                         ('baddisk'      , "IOError: [Errno 13] Permission denied: '/tmp/mnt/sysimg//vservers/\w+/etc/hosts'"),
 609                         ('noinstall'    , 'notinstalled'),
 610                         ('bziperror'    , 'bzip2: Data integrity error when decompressing.'),
 611                         ('noblockdev'   , "No block devices detected."),
 612                         ('dnserror'     , 'Name or service not known'),
 613                         ('downloadfail' , 'Unable to download main tarball /boot/bootstrapfs-planetlab-i386.tar.bz2 from server.'),
 614                         ('disktoosmall' , 'The total usable disk size of all disks is insufficient to be usable as a PlanetLab node.'),
 615                         ('hardwarerequirefail' , 'Hardware requirements not met'),
 616                         ('mkfsfail'         , 'while running: Running mkfs.ext2 -q  -m 0 -j /dev/planetlab/vservers failed'),
 617                         ('nofilereference', "No such file or directory: '/tmp/mnt/sysimg//vservers/.vref/planetlab-f8-i386/etc/hosts'"),
 618                         ('kernelcopyfail', "cp: cannot stat `/tmp/mnt/sysimg/boot/kernel-boot': No such file or directory"),
 619                         ('chrootfail'   , 'Running chroot /tmp/mnt/sysimg'),
 620                         ('modulefail'   , 'Unable to get list of system modules'),
 621                         ('writeerror'   , 'write error: No space left on device'),
 622                         ('nospace'      , "No space left on device"),
 623                         ('nonode'       , 'Failed to authenticate call: No such node'),
 624                         ('authfail'     , 'Failed to authenticate call: Call could not be authenticated'),
 625                         ('bootcheckfail'     , 'BootCheckAuthentication'),
 626                         ('bootupdatefail'   , 'BootUpdateNode'),
 627                 ]
 628                 return steps
 629
 630         def getBootManagerSequenceFromLog(self, steps, child):
 631                 sequence = []
 632                 while True:
 633
 634                         index = child.expect( steps_to_list(steps) + [ pexpect.EOF ])
 635                         id = index_to_id(steps,index)
 636                         sequence.append(id)
 637
 638                         if id == "exception":
 639                                 print "...Found An Exception!!!"
 640                         elif id == "done": #index == len(steps_to_list(steps)):
 641                                 #print "Reached EOF"
 642                                 break
 643
 644                 return sequence
 645
 646 def restore(sitehist, hostname, config=None, forced_action=None):
 647         ret = restore_basic(sitehist, hostname, config, forced_action)
 648         session.flush()
 649         return ret
 650
 651 def restore_basic(sitehist, hostname, config=None, forced_action=None):
 652
 653         # NOTE: Nothing works if the bootcd is REALLY old.
 654         #       So, this is the first step.
 655
 656         fbnode = FindbadNodeRecord.get_latest_by(hostname=hostname).to_dict()
 657         recent_actions = sitehist.getRecentActions(hostname=hostname)
 658
 659         if fbnode['observed_category'] == "OLDBOOTCD":
 660                 print "\t...Notify owner to update BootImage!!!"
 661
 662                 if not found_within(recent_actions, 'newbootcd_notice', 3.5):
 663                         sitehist.sendMessage('newbootcd_notice', hostname=hostname)
 664
 665                         print "\tDisabling %s due to out-of-date BootImage" % hostname
 666                         api.UpdateNode(hostname, {'boot_state' : 'disabled'})
 667
 668                 # NOTE: nothing else is possible.
 669                 return True
 670
 671         debugnode = DebugInterface(hostname)
 672         conn = debugnode.getConnection()
 673         if type(conn) == type(False): return False
 674
 675         boot_state = conn.get_boot_state()
 676         if boot_state != "debug":
 677                 print "... %s in %s state: skipping..." % (hostname , boot_state)
 678                 return boot_state == "boot"
 679
 680         if conn.bootmanager_running():
 681                 print "...BootManager is currently running.  Skipping host %s" %hostname
 682                 return True
 683
 684         # Read persistent flags, tagged on one week intervals.
 685
 686         if config and not config.quiet: print "...downloading dmesg from %s" %hostname
 687         dmesg = conn.get_dmesg()
 688         child = fdpexpect.fdspawn(dmesg)
 689
 690         steps = debugnode.getDiskSteps()
 691         sequence = debugnode.getDiskSequence(steps, child)
 692
 693         s = Set(sequence)
 694         if config and not config.quiet: print "\tSET: ", s
 695
 696         if len(s) > 1:
 697                 print "...Potential drive errors on %s" % hostname
 698                 if len(s) == 2 and 'floppyerror' in s:
 699                         print "...Should investigate.  Continuing with node."
 700                 else:
 701                         print "...Should investigate.  Skipping node."
 702                         # TODO: send message related to these errors.
 703
 704                         if not found_within(recent_actions, 'baddisk_notice', 7):
 705                                 print "baddisk_notice not found recently"
 706
 707                                 log=conn.get_dmesg().read()
 708                                 sitehist.sendMessage('baddisk_notice', hostname=hostname, log=log)
 709                                 conn.set_nodestate('disabled')
 710
 711                         return False
 712
 713         print "...Downloading bm.log from %s" %hostname
 714         log = conn.get_bootmanager_log()
 715         child = fdpexpect.fdspawn(log)
 716
 717         if hasattr(config, 'collect') and config.collect: return True
 718
 719         if config and not config.quiet: print "...Scanning bm.log for errors"
 720
 721         time.sleep(1)
 722
 723         steps = debugnode.getBootManagerStepPatterns()
 724         sequence = debugnode.getBootManagerSequenceFromLog(steps, child)
 725
 726         s = "-".join(sequence)
 727         print "   FOUND SEQUENCE: ", s
 728
 729         # NOTE: We get or set the flag based on the current sequence identifier.
 730         #  By using the sequence identifier, we guarantee that there will be no
 731         #  frequent loops.  I'm guessing there is a better way to track loops,
 732         #  though.
 733
 734         sequences = debugnode.getSequences()
 735         flag_set = True
 736
 737         if s not in sequences:
 738                 print "   HOST %s" % hostname
 739                 print "   UNKNOWN SEQUENCE: %s" % s
 740
 741                 args = {}
 742                 args['hostname'] = hostname
 743                 args['sequence'] = s
 744                 args['bmlog'] = conn.get_bootmanager_log().read()
 745                 args['viart'] = False
 746                 args['saveact'] = True
 747
 748                 sitehist.sendMessage('unknownsequence_notice', **args)
 749
 750                 conn.restart_bootmanager('boot')
 751
 752                 # NOTE: Do not set the pflags value for this sequence if it's unknown.
 753                 # This way, we can check it again after we've fixed it.
 754                 flag_set = False
 755
 756         else:
 757
 758                 if   sequences[s] == "restart_bootmanager_boot":
 759                         print "...Restarting BootManager.py on %s "%hostname
 760                         conn.restart_bootmanager('boot')
 761                 elif sequences[s] == "restart_bootmanager_rins":
 762                         print "...Restarting BootManager.py on %s "%hostname
 763                         conn.restart_bootmanager('reinstall')
 764                 elif sequences[s] == "restart_node_rins":
 765                         conn.restart_node('reinstall')
 766                 elif sequences[s] == "restart_node_boot":
 767                         conn.restart_node('boot')
 768                 elif sequences[s] == "repair_node_keys":
 769                         if conn.compare_and_repair_nodekeys():
 770                                 # the keys either are in sync or were forced in sync.
 771                                 # so try to reboot the node again.
 772                                 # TODO: why was this originally 'reinstall' instead of 'boot'??
 773                                 conn.restart_bootmanager('boot')
 774                                 pass
 775                         else:
 776                                 # there was some failure to synchronize the keys.
 777                                 print "...Unable to repair node keys on %s" %hostname
 778
 779                 elif sequences[s] == "suspect_error_email":
 780                         args = {}
 781                         args['hostname'] = hostname
 782                         args['sequence'] = s
 783                         args['bmlog'] = conn.get_bootmanager_log().read()
 784                         args['viart'] = False
 785                         args['saveact'] = True
 786
 787                         sitehist.sendMessage('unknownsequence_notice', **args)
 788                         conn.restart_bootmanager('boot')
 789
 790                 # TODO: differentiate this and the 'nodenetwork_email' actions.
 791                 elif sequences[s] == "update_node_config_email":
 792
 793                         if not found_within(recent_actions, 'nodeconfig_notice', 3.5):
 794                                 args = {}
 795                                 args['hostname'] = hostname
 796                                 sitehist.sendMessage('nodeconfig_notice', **args)
 797                                 conn.dump_plconf_file()
 798
 799                 elif sequences[s] == "nodenetwork_email":
 800
 801                         if not found_within(recent_actions, 'nodeconfig_notice', 3.5):
 802                                 args = {}
 803                                 args['hostname'] = hostname
 804                                 args['bmlog'] = conn.get_bootmanager_log().read()
 805                                 sitehist.sendMessage('nodeconfig_notice', **args)
 806                                 conn.dump_plconf_file()
 807
 808                 elif sequences[s] == "update_bootcd_email":
 809
 810                         if not found_within(recent_actions, 'newalphacd_notice', 3.5):
 811                                 args = {}
 812                                 args.update(getconf.getconf(hostname)) # NOTE: Generates boot images for the user:
 813                                 args['hostname'] = hostname
 814
 815                                 sitehist.sendMessage('newalphacd_notice', **args)
 816
 817                                 print "\tDisabling %s due to out-of-date BOOTCD" % hostname
 818
 819                 elif sequences[s] == "broken_hardware_email":
 820                         # MAKE An ACTION record that this host has failed hardware.  May
 821                         # require either an exception "/minhw" or other manual intervention.
 822                         # Definitely need to send out some more EMAIL.
 823                         # TODO: email notice of broken hardware
 824                         if not found_within(recent_actions, 'baddisk_notice', 7):
 825                                 print "...NOTIFYING OWNERS OF BROKEN HARDWARE on %s!!!" % hostname
 826                                 args = {}
 827                                 args['hostname'] = hostname
 828                                 args['log'] = conn.get_dmesg().read()
 829
 830                                 sitehist.sendMessage('baddisk_notice', **args)
 831                                 conn.set_nodestate('disabled')
 832
 833                 elif sequences[s] == "update_hardware_email":
 834                         if not found_within(recent_actions, 'minimalhardware_notice', 7):
 835                                 print "...NOTIFYING OWNERS OF MINIMAL HARDWARE FAILURE on %s!!!" % hostname
 836                                 args = {}
 837                                 args['hostname'] = hostname
 838                                 args['bmlog'] = conn.get_bootmanager_log().read()
 839                                 sitehist.sendMessage('minimalhardware_notice', **args)
 840
 841                 elif sequences[s] == "bad_dns_email":
 842                         if not found_within(recent_actions, 'baddns_notice', 1):
 843                                 print "...NOTIFYING OWNERS OF DNS FAILURE on %s!!!" % hostname
 844                                 args = {}
 845                                 try:
 846                                         node = plccache.GetNodeByName(hostname)
 847                                         net = api.GetInterfaces(node['interface_ids'])[0]
 848                                 except:
 849                                         email_exception()
 850                                         print traceback.print_exc()
 851                                         # TODO: api error. skip email, b/c all info is not available,
 852                                         # flag_set will not be recorded.
 853                                         return False
 854                                 nodenet_str = network_config_to_str(net)
 855
 856                                 args['hostname'] = hostname
 857                                 args['network_config'] = nodenet_str
 858                                 args['interface_id'] = net['interface_id']
 859
 860                                 sitehist.sendMessage('baddns_notice', **args)
 861
 862         return True
 863
 864
 865 # MAIN -------------------------------------------------------------------
 866
 867 def main():
 868         from monitor import parser as parsermodule
 869         parser = parsermodule.getParser()
 870
 871         parser.set_defaults(child=False, collect=False, nosetup=False, verbose=False,
 872                                                 force=None, quiet=False)
 873         parser.add_option("", "--child", dest="child", action="store_true",
 874                                                 help="This is the child mode of this process.")
 875         parser.add_option("", "--force", dest="force", metavar="boot_state",
 876                                                 help="Force a boot state passed to BootManager.py.")
 877         parser.add_option("", "--quiet", dest="quiet", action="store_true",
 878                                                 help="Extra quiet output messages.")
 879         parser.add_option("", "--verbose", dest="verbose", action="store_true",
 880                                                 help="Extra debug output messages.")
 881         parser.add_option("", "--nonet", dest="nonet", action="store_true",
 882                                                 help="Do not setup the network, use existing log files to re-run a test pass.")
 883         parser.add_option("", "--collect", dest="collect", action="store_true",
 884                                                 help="No action, just collect dmesg, and bm.log")
 885         parser.add_option("", "--nosetup", dest="nosetup", action="store_true",
 886                                                 help="Do not perform the orginary setup phase.")
 887
 888         parser = parsermodule.getParser(['nodesets', 'defaults'], parser)
 889         config = parsermodule.parse_args(parser)
 890
 891         if config.nodelist:
 892                 nodes = config.getListFromFile(config.nodelist)
 893         elif config.node:
 894                 nodes = [ config.node ]
 895         else:
 896                 parser.print_help()
 897                 sys.exit(1)
 898
 899         for node in nodes:
 900                 # get sitehist
 901                 lb = plccache.plcdb_hn2lb[node]
 902                 sitehist = SiteInterface.get_or_make(loginbase=lb)
 903                 #reboot(node, config)
 904                 restore(sitehist, node, config=None, forced_action=None)
 905
 906 if __name__ == "__main__":
 907         main()