bootman.py

   1 #!/usr/bin/python
   2
   3 # Attempt to reboot a node in debug state.
   4
   5 import plc
   6 api = plc.getAuthAPI()
   7
   8 import sys
   9 import os
  10 import policy
  11
  12 from getsshkeys import SSHKnownHosts
  13
  14 import subprocess
  15 import time
  16 import database
  17 import moncommands
  18 from sets import Set
  19
  20 import ssh.pxssh as pxssh
  21 import ssh.fdpexpect as fdpexpect
  22 import ssh.pexpect as pexpect
  23 from unified_model import *
  24 from emailTxt import mailtxt
  25 from nodeconfig import network_config_to_str
  26 import traceback
  27 import monitorconfig
  28
  29 import signal
  30 class Sopen(subprocess.Popen):
  31         def kill(self, signal = signal.SIGTERM):
  32                 os.kill(self.pid, signal)
  33
  34 #from Rpyc import SocketConnection, Async
  35 from Rpyc import SocketConnection, Async
  36 from Rpyc.Utils import *
  37
  38 def get_fbnode(node):
  39         fb = database.dbLoad("findbad")
  40         fbnode = fb['nodes'][node]['values']
  41         return fbnode
  42
  43 class NodeConnection:
  44         def __init__(self, connection, node, config):
  45                 self.node = node
  46                 self.c = connection
  47                 self.config = config
  48
  49         def get_boot_state(self):
  50                 if self.c.modules.os.path.exists('/tmp/source'):
  51                         return "dbg"
  52                 elif self.c.modules.os.path.exists('/vservers'):
  53                         return "boot"
  54                 else:
  55                         return "unknown"
  56
  57         def get_dmesg(self):
  58                 self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
  59                 download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node)
  60                 log = open("log/dmesg.%s.log" % self.node, 'r')
  61                 return log
  62
  63         def get_bootmanager_log(self):
  64                 download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node)
  65                 os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node))
  66                 log = open("log/bm.%s.log" % self.node, 'r')
  67                 return log
  68
  69         def dump_plconf_file(self):
  70                 c = self.c
  71                 self.c.modules.sys.path.append("/tmp/source/")
  72                 self.c.modules.os.chdir('/tmp/source')
  73
  74                 log = c.modules.BootManager.log('/tmp/new.log')
  75                 bm = c.modules.BootManager.BootManager(log,'boot')
  76
  77                 BootManagerException = c.modules.Exceptions.BootManagerException
  78                 InitializeBootManager = c.modules.BootManager.InitializeBootManager
  79                 ReadNodeConfiguration = c.modules.BootManager.ReadNodeConfiguration
  80                 bm_continue = True
  81
  82                 InitializeBootManager.Run(bm.VARS, bm.LOG)
  83                 try: ReadNodeConfiguration.Run(bm.VARS, bm.LOG)
  84                 except Exception, x:
  85                         bm_continue = False
  86                         print "   ERROR:", x
  87                         print "   Possibly, unable to find valid configuration file"
  88
  89                 if bm_continue and self.config and not self.config.quiet:
  90                         for key in bm.VARS.keys():
  91                                 print key, " == ", bm.VARS[key]
  92                 else:
  93                         if self.config and not self.config.quiet: print "   Unable to read Node Configuration"
  94
  95
  96         def compare_and_repair_nodekeys(self):
  97                 c = self.c
  98                 self.c.modules.sys.path.append("/tmp/source/")
  99                 self.c.modules.os.chdir('/tmp/source')
 100
 101                 log = c.modules.BootManager.log('/tmp/new.log')
 102                 bm = c.modules.BootManager.BootManager(log,'boot')
 103
 104                 BootManagerException = c.modules.Exceptions.BootManagerException
 105                 InitializeBootManager = c.modules.BootManager.InitializeBootManager
 106                 ReadNodeConfiguration = c.modules.BootManager.ReadNodeConfiguration
 107                 bm_continue = True
 108
 109                 plcnode = api.GetNodes({'hostname': self.node}, None)[0]
 110
 111                 InitializeBootManager.Run(bm.VARS, bm.LOG)
 112                 try: ReadNodeConfiguration.Run(bm.VARS, bm.LOG)
 113                 except Exception, x:
 114                         bm_continue = False
 115                         print "exception"
 116                         print x
 117                         print "   Possibly, unable to find valid configuration file"
 118
 119                 if bm_continue:
 120                         print "   NODE: %s" % bm.VARS['NODE_KEY']
 121                         print "   PLC : %s" % plcnode['key']
 122
 123                         if bm.VARS['NODE_KEY'] == plcnode['key']:
 124                                 return True
 125                         else:
 126                                 if api.UpdateNode(self.node, {'key': bm.VARS['NODE_KEY']}):
 127                                         print "   Successfully updated NODE_KEY with PLC"
 128                                         return True
 129                                 else:
 130                                         return False
 131
 132                         #for key in bm.VARS.keys():
 133                         #       print key, " == ", bm.VARS[key]
 134                 else:
 135                         print "   Unable to retrieve NODE_KEY"
 136
 137         def bootmanager_running(self):
 138                 if self.c.modules.os.path.exists('/tmp/BM_RUNNING'):
 139                         return True
 140                 else:
 141                         return False
 142
 143         def set_nodestate(self, state='boot'):
 144                 return api.UpdateNode(self.node, {'boot_state' : state})
 145
 146         def restart_node(self, state='boot'):
 147                 api.UpdateNode(self.node, {'boot_state' : state})
 148
 149                 pflags = PersistFlags(self.node, 1*60*60*24, db='restart_persistflags')
 150                 if not pflags.getRecentFlag('gentlekill'):
 151                         print "   Killing all slice processes... : %s" %  self.node
 152                         cmd_slicekill = "ls -d /proc/virtual/[0-9]* | awk -F '/' '{print $4}' | xargs -I{} /usr/sbin/vkill -s 9 --xid {} -- 0"
 153                         self.c.modules.os.system(cmd_slicekill)
 154                         cmd = """ shutdown -r +1 & """
 155                         print "   Restarting %s : %s" % ( self.node, cmd)
 156                         self.c.modules.os.system(cmd)
 157
 158                         pflags.setRecentFlag('gentlekill')
 159                         pflags.save()
 160                 else:
 161                         print "   Restarting with sysrq 'sub' %s" % self.node
 162                         cmd = """ (sleep 5; echo 's' > /proc/sysrq-trigger; echo 'u' > /proc/sysrq-trigger; echo 'b' > /proc/sysrq-trigger ) & """
 163                         self.c.modules.os.system(cmd)
 164
 165                 return
 166
 167         def restart_bootmanager(self, forceState):
 168
 169                 self.c.modules.os.chdir('/tmp/source')
 170                 if self.c.modules.os.path.exists('/tmp/BM_RUNNING'):
 171                         print "   BootManager is already running: try again soon..."
 172                 else:
 173                         print "   Starting 'BootManager.py %s' on %s " % (forceState, self.node)
 174                         cmd = "( touch /tmp/BM_RUNNING ;  " + \
 175                               "  python ./BootManager.py %s &> server.log < /dev/null ; " + \
 176                                   "  rm -f /tmp/BM_RUNNING " + \
 177                                   ") &"
 178                         cmd = cmd % forceState
 179                         self.c.modules.os.system(cmd)
 180
 181                 return
 182
 183
 184 import random
 185 class PlanetLabSession:
 186         globalport = 22000 + int(random.random()*1000)
 187
 188         def __init__(self, node, nosetup, verbose):
 189                 self.verbose = verbose
 190                 self.node = node
 191                 self.port = None
 192                 self.nosetup = nosetup
 193                 self.command = None
 194                 self.setup_host()
 195
 196         def get_connection(self, config):
 197                 return NodeConnection(SocketConnection("localhost", self.port), self.node, config)
 198
 199         def setup_host(self):
 200                 self.port = PlanetLabSession.globalport
 201                 PlanetLabSession.globalport = PlanetLabSession.globalport + 1
 202
 203                 args = {}
 204                 args['port'] = self.port
 205                 args['user'] = 'root'
 206                 args['hostname'] = self.node
 207                 args['monitordir'] = monitorconfig.MONITOR_SCRIPT_ROOT
 208                 ssh_port = 22
 209
 210                 if self.nosetup:
 211                         print "Skipping setup"
 212                         return
 213
 214                 # COPY Rpyc files to host
 215                 cmd = "rsync -qv -az -e ssh %(monitordir)s/Rpyc/ %(user)s@%(hostname)s:Rpyc 2> /dev/null" % args
 216                 if self.verbose: print cmd
 217                 # TODO: Add timeout
 218                 timeout = 120
 219                 localos = moncommands.CMD()
 220
 221                 ret = localos.system(cmd, timeout)
 222                 print ret
 223                 if ret != 0:
 224                         print "\tUNKNOWN SSH KEY FOR %s; making an exception" % self.node
 225                         #print "MAKE EXPLICIT EXCEPTION FOR %s" % self.node
 226                         k = SSHKnownHosts(); k.updateDirect(self.node); k.write(); del k
 227                         ret = localos.system(cmd, timeout)
 228                         print ret
 229                         if ret != 0:
 230                                 print "\tFAILED TWICE"
 231                                 #sys.exit(1)
 232                                 raise Exception("Failed twice trying to login with updated ssh host key")
 233
 234                 t1 = time.time()
 235                 # KILL any already running servers.
 236                 ssh = moncommands.SSH(args['user'], args['hostname'], ssh_port)
 237                 (ov,ev) = ssh.run_noexcept2("""<<\EOF
 238             rm -f out.log
 239             echo "kill server" >> out.log
 240             ps ax | grep Rpyc | grep -v grep | awk '{print $1}' | xargs kill 2> /dev/null ;
 241             echo "export" >> out.log
 242             export PYTHONPATH=$HOME  ;
 243             echo "start server" >> out.log
 244             python Rpyc/Servers/forking_server.py &> server.log &
 245             echo "done" >> out.log
 246 EOF""")
 247                 #cmd = """ssh %(user)s@%(hostname)s """ + \
 248                 #        """'ps ax | grep Rpyc | grep -v grep | awk "{print \$1}" | xargs kill 2> /dev/null' """
 249                 #cmd = cmd % args
 250                 #if self.verbose: print cmd
 251                 ## TODO: Add timeout
 252                 #print localos.system(cmd,timeout)
 253
 254                 ## START a new rpyc server.
 255                 #cmd = """ssh -n %(user)s@%(hostname)s "export PYTHONPATH=\$HOME; """ + \
 256                 #        """python Rpyc/Servers/forking_server.py &> server.log < /dev/null &" """
 257                 #cmd = cmd % args
 258                 #if self.verbose: print cmd
 259                 #print localos.system(cmd,timeout)
 260                 print ssh.ret
 261
 262                 # TODO: Add timeout
 263                 # This was tricky to make synchronous.  The combination of ssh-clients-4.7p1,
 264                 # and the following options seems to work well.
 265                 cmd = """ssh -o ExitOnForwardFailure=yes -o BatchMode=yes """ + \
 266                           """-o PermitLocalCommand=yes -o LocalCommand='echo "READY"' """ + \
 267                           """-o ConnectTimeout=120 """ + \
 268                           """-n -N -L %(port)s:localhost:18812 """ + \
 269                           """%(user)s@%(hostname)s"""
 270                 cmd = cmd % args
 271                 if self.verbose: print cmd
 272                 self.command = Sopen(cmd, shell=True, stdout=subprocess.PIPE)
 273                 # TODO: the read() here may block indefinitely.  Need a better
 274                 # approach therefore, that includes a timeout.
 275                 #ret = self.command.stdout.read(5)
 276                 ret = moncommands.read_t(self.command.stdout, 5)
 277
 278                 t2 = time.time()
 279                 if 'READY' in ret:
 280                         # NOTE: There is still a slight race for machines that are slow...
 281                         self.timeout = 2*(t2-t1)
 282                         print "Sleeping for %s sec" % self.timeout
 283                         time.sleep(self.timeout)
 284                         return
 285
 286                 if self.command.returncode is not None:
 287                         print "Failed to establish tunnel!"
 288                         raise Exception("SSH Tunnel exception : %s %s" % (self.node, self.command.returncode))
 289
 290                 raise Exception("Unknown SSH Tunnel Exception: still running, but did not report 'READY'")
 291
 292         def __del__(self):
 293                 if self.command:
 294                         if self.verbose: print "Killing SSH session %s" % self.port
 295                         self.command.kill()
 296
 297
 298 def steps_to_list(steps):
 299         ret_list = []
 300         for (id,label) in steps:
 301                 ret_list.append(label)
 302         return ret_list
 303
 304 def index_to_id(steps,index):
 305         if index < len(steps):
 306                 return steps[index][0]
 307         else:
 308                 return "done"
 309
 310 def reboot(hostname, config=None, forced_action=None):
 311
 312         # NOTE: Nothing works if the bootcd is REALLY old.
 313         #       So, this is the first step.
 314         fbnode = get_fbnode(hostname)
 315         if fbnode['category'] == "OLDBOOTCD":
 316                 print "...NOTIFY OWNER TO UPDATE BOOTCD!!!"
 317                 args = {}
 318                 args['hostname_list'] = "    %s" % hostname
 319
 320                 m = PersistMessage(hostname, "Please Update Boot Image for %s" % hostname,
 321                                                         mailtxt.newbootcd_one[1] % args, True, db='bootcd_persistmessages')
 322
 323                 loginbase = plc.siteId(hostname)
 324                 m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 325
 326                 print "\tDisabling %s due to out-of-date BOOTCD" % hostname
 327                 api.UpdateNode(hostname, {'boot_state' : 'disable'})
 328                 return True
 329
 330         node = hostname
 331         print "Creating session for %s" % node
 332         # update known_hosts file (in case the node has rebooted since last run)
 333         if config and not config.quiet: print "...updating known_hosts ssh-rsa key for %s" % node
 334         try:
 335                 k = SSHKnownHosts(); k.update(node); k.write(); del k
 336         except:
 337                 print traceback.print_exc()
 338                 return False
 339
 340         try:
 341                 if config == None:
 342                         session = PlanetLabSession(node, False, True)
 343                 else:
 344                         session = PlanetLabSession(node, config.nosetup, config.verbose)
 345         except Exception, e:
 346                 print "ERROR setting up session for %s" % hostname
 347                 print traceback.print_exc()
 348                 print e
 349                 return False
 350
 351         try:
 352                 conn = session.get_connection(config)
 353         except EOFError:
 354                 # NOTE: sometimes the wait in setup_host() is not long enough.
 355                 # So, here we try to wait a little longer before giving up entirely.
 356                 try:
 357                         time.sleep(session.timeout*4)
 358                         conn = session.get_connection(config)
 359                 except:
 360                         print traceback.print_exc()
 361                         return False
 362
 363
 364         if forced_action == "reboot":
 365                 conn.restart_node('rins')
 366                 return True
 367
 368         boot_state = conn.get_boot_state()
 369         if boot_state == "boot":
 370                 print "...Boot state of %s already completed : skipping..." % node
 371                 return True
 372         elif boot_state == "unknown":
 373                 print "...Unknown bootstate for %s : skipping..."% node
 374                 return False
 375         else:
 376                 pass
 377
 378         if conn.bootmanager_running():
 379                 print "...BootManager is currently running.  Skipping host %s" % node
 380                 return True
 381
 382         #if config != None:
 383         #       if config.force:
 384         #               conn.restart_bootmanager(config.force)
 385         #               return True
 386
 387         # Read persistent flags, tagged on one week intervals.
 388         pflags = PersistFlags(hostname, 3*60*60*24, db='debug_persistflags')
 389
 390
 391         if config and not config.quiet: print "...downloading dmesg from %s" % node
 392         dmesg = conn.get_dmesg()
 393         child = fdpexpect.fdspawn(dmesg)
 394
 395         sequence = []
 396         while True:
 397                 steps = [
 398                         ('scsierror'  , 'SCSI error : <\d+ \d+ \d+ \d+> return code = 0x\d+'),
 399                         ('ioerror'    , 'end_request: I/O error, dev sd\w+, sector \d+'),
 400                         ('ccisserror' , 'cciss: cmd \w+ has CHECK CONDITION  byte \w+ = \w+'),
 401
 402                         ('buffererror', 'Buffer I/O error on device dm-\d, logical block \d+'),
 403                         ('atareadyerror'   , 'ata\d+: status=0x\d+ { DriveReady SeekComplete Error }'),
 404                         ('atacorrecterror' , 'ata\d+: error=0x\d+ { UncorrectableError }'),
 405                         ('sdXerror'   , 'sd\w: Current: sense key: Medium Error'),
 406                         ('ext3error'   , 'EXT3-fs error (device dm-\d+): ext3_find_entry: reading directory #\d+ offset \d+'),
 407                         ('floppytimeout','floppy0: floppy timeout called'),
 408                         ('floppyerror',  'end_request: I/O error, dev fd\w+, sector \d+'),
 409
 410                         # floppy0: floppy timeout called
 411                         # end_request: I/O error, dev fd0, sector 0
 412
 413                         #Buffer I/O error on device dm-2, logical block 8888896
 414                         #ata1: status=0x51 { DriveReady SeekComplete Error }
 415                         #ata1: error=0x40 { UncorrectableError }
 416                         #SCSI error : <0 0 0 0> return code = 0x8000002
 417                         #sda: Current: sense key: Medium Error
 418                         #       Additional sense: Unrecovered read error - auto reallocate failed
 419
 420                         #SCSI error : <0 2 0 0> return code = 0x40001
 421                         #end_request: I/O error, dev sda, sector 572489600
 422                 ]
 423                 id = index_to_id(steps, child.expect( steps_to_list(steps) + [ pexpect.EOF ]))
 424                 sequence.append(id)
 425
 426                 if id == "done":
 427                         break
 428
 429         s = Set(sequence)
 430         if config and not config.quiet: print "\tSET: ", s
 431
 432         if len(s) > 1:
 433                 print "...Potential drive errors on %s" % node
 434                 if len(s) == 2 and 'floppyerror' in s:
 435                         print "...Should investigate.  Continuing with node."
 436                 else:
 437                         print "...Should investigate.  Skipping node."
 438                         # TODO: send message related to these errors.
 439                         args = {}
 440                         args['hostname'] = hostname
 441                         args['log'] = conn.get_dmesg().read()
 442
 443                         m = PersistMessage(hostname, mailtxt.baddisk[0] % args,
 444                                                                                  mailtxt.baddisk[1] % args, True, db='hardware_persistmessages')
 445
 446                         loginbase = plc.siteId(hostname)
 447                         m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 448                         conn.set_nodestate('disable')
 449                         return False
 450
 451         print "...Downloading bm.log from %s" % node
 452         log = conn.get_bootmanager_log()
 453         child = fdpexpect.fdspawn(log)
 454
 455         try:
 456                 if config.collect: return True
 457         except:
 458                 pass
 459
 460         time.sleep(1)
 461
 462         if config and not config.quiet: print "...Scanning bm.log for errors"
 463         action_id = "dbg"
 464         sequence = []
 465         while True:
 466
 467                 steps = [
 468                         ('bminit'               , 'Initializing the BootManager.'),
 469                         ('cfg'                  , 'Reading node configuration file.'),
 470                         ('auth'                 , 'Authenticating node with PLC.'),
 471                         ('getplc'               , 'Retrieving details of node from PLC.'),
 472                         ('update'               , 'Updating node boot state at PLC.'),
 473                         ('hardware'             , 'Checking if hardware requirements met.'),
 474                         ('installinit'  , 'Install: Initializing.'),
 475                         ('installdisk'  , 'Install: partitioning disks.'),
 476                         ('installbootfs', 'Install: bootstrapfs tarball.'),
 477                         ('installcfg'   , 'Install: Writing configuration files.'),
 478                         ('installstop'  , 'Install: Shutting down installer.'),
 479                         ('update2'              , 'Updating node boot state at PLC.'),
 480                         ('installinit2' , 'Install: Initializing.'),
 481                         ('validate'             , 'Validating node installation.'),
 482                         ('rebuildinitrd', 'Rebuilding initrd'),
 483                         ('netcfg'               , 'Install: Writing Network Configuration files.'),
 484                         ('update3'              , 'Updating node configuration.'),
 485                         ('disk'                 , 'Checking for unused disks to add to LVM.'),
 486                         ('update4'              , 'Sending hardware configuration to PLC.'),
 487                         ('debug'                , 'Starting debug mode'),
 488                         ('bmexceptmount', 'BootManagerException during mount'),
 489                         ('bmexceptvgscan', 'BootManagerException during vgscan/vgchange'),
 490                         ('bmexceptrmfail', 'Unable to remove directory tree: /tmp/mnt'),
 491                         ('exception'    , 'Exception'),
 492                         ('nocfg'        , 'Found configuration file planet.cnf on floppy, but was unable to parse it.'),
 493                         ('protoerror'   , 'XML RPC protocol error'),
 494                         ('nodehostname' , 'Configured node hostname does not resolve'),
 495                         ('implementerror', 'Implementation Error'),
 496                         ('readonlyfs'   , '[Errno 30] Read-only file system'),
 497                         ('noinstall'    , 'notinstalled'),
 498                         ('bziperror'    , 'bzip2: Data integrity error when decompressing.'),
 499                         ('noblockdev'   , "No block devices detected."),
 500                         ('dnserror'     , 'Name or service not known'),
 501                         ('downloadfail' , 'Unable to download main tarball /boot/bootstrapfs-planetlab-i386.tar.bz2 from server.'),
 502                         ('disktoosmall' , 'The total usable disk size of all disks is insufficient to be usable as a PlanetLab node.'),
 503                         ('hardwarerequirefail' , 'Hardware requirements not met'),
 504                         ('mkfsfail'         , 'while running: Running mkfs.ext2 -q  -m 0 -j /dev/planetlab/vservers failed'),
 505                         ('nofilereference', "No such file or directory: '/tmp/mnt/sysimg//vservers/.vref/planetlab-f8-i386/etc/hosts'"),
 506                         ('chrootfail'   , 'Running chroot /tmp/mnt/sysimg'),
 507                         ('modulefail'   , 'Unable to get list of system modules'),
 508                         ('writeerror'   , 'write error: No space left on device'),
 509                         ('nospace'      , "No space left on device"),
 510                         ('nonode'       , 'Failed to authenticate call: No such node'),
 511                         ('authfail'     , 'Failed to authenticate call: Call could not be authenticated'),
 512                         ('bootcheckfail'     , 'BootCheckAuthentication'),
 513                         ('bootupdatefail'   , 'BootUpdateNode'),
 514                 ]
 515                 list = steps_to_list(steps)
 516                 index = child.expect( list + [ pexpect.EOF ])
 517                 id = index_to_id(steps,index)
 518                 sequence.append(id)
 519
 520                 if id == "exception":
 521                         if config and not config.quiet: print "...Found An Exception!!!"
 522                 elif index == len(list):
 523                         #print "Reached EOF"
 524                         break
 525
 526         s = "-".join(sequence)
 527         print "   FOUND SEQUENCE: ", s
 528
 529         # NOTE: We get or set the flag based on the current sequence identifier.
 530         #  By using the sequence identifier, we guarantee that there will be no
 531         #  frequent loops.  I'm guessing there is a better way to track loops,
 532         #  though.
 533         if not config.force and pflags.getRecentFlag(s):
 534                 pflags.setRecentFlag(s)
 535                 pflags.save()
 536                 print "... flag is set or it has already run recently. Skipping %s" % node
 537                 return True
 538
 539         sequences = {}
 540
 541
 542         # restart_bootmanager_boot
 543         for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-done",
 544                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-protoerror-debug-done",
 545                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-implementerror-bootupdatefail-update-debug-done",
 546
 547                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-exception-protoerror-update-protoerror-debug-done",
 548
 549                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-exception-protoerror-update-debug-done",
 550                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-exception-chrootfail-update-debug-done",
 551                         "bminit-cfg-auth-getplc-update-debug-done",
 552                         "bminit-cfg-auth-getplc-exception-protoerror-update-protoerror-debug-done",
 553                         "bminit-cfg-auth-protoerror-exception-update-protoerror-debug-done",
 554                         "bminit-cfg-auth-protoerror-exception-update-bootupdatefail-authfail-debug-done",
 555                         "bminit-cfg-auth-protoerror-exception-update-debug-done",
 556                         "bminit-cfg-auth-getplc-exception-protoerror-update-debug-done",
 557                         "bminit-cfg-auth-getplc-implementerror-update-debug-done",
 558                         ]:
 559                 sequences.update({n : "restart_bootmanager_boot"})
 560
 561         #       conn.restart_bootmanager('rins')
 562         for n in [ "bminit-cfg-auth-getplc-installinit-validate-exception-modulefail-update-debug-done",
 563                         "bminit-cfg-auth-getplc-update-installinit-validate-exception-modulefail-update-debug-done",
 564                         "bminit-cfg-auth-getplc-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done",
 565                         "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptmount-exception-noinstall-update-debug-done",
 566                         "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done",
 567                         "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-done",
 568                         "bminit-cfg-auth-getplc-hardware-installinit-installdisk-bziperror-exception-update-debug-done",
 569                         "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-update-debug-done",
 570                         "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-done",
 571                         "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-update-debug-done",
 572                         "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done",
 573                         "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-exception-mkfsfail-update-debug-done",
 574                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done",
 575                         "bminit-cfg-auth-getplc-installinit-validate-exception-noinstall-update-debug-done",
 576                         ]:
 577                 sequences.update({n : "restart_bootmanager_rins"})
 578
 579         # repair_node_keys
 580         sequences.update({"bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-done": "repair_node_keys"})
 581
 582         #   conn.restart_node('rins')
 583         for n in ["bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-exception-chrootfail-update-debug-done",
 584                         "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-exception-chrootfail-update-debug-done",
 585                         "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-installcfg-exception-chrootfail-update-debug-done",
 586                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-writeerror-exception-chrootfail-update-debug-done",
 587                         "bminit-cfg-auth-getplc-update-hardware-installinit-exception-bmexceptrmfail-update-debug-done",
 588                         "bminit-cfg-auth-getplc-hardware-installinit-exception-bmexceptrmfail-update-debug-done",
 589                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-disk-update4-update3-implementerror-bootupdatefail-update-debug-done",
 590                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-readonlyfs-update-debug-done",
 591                         "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-nospace-exception-update-debug-done",
 592                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done",
 593                         "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-update-debug-done",
 594                         "bminit-cfg-auth-getplc-update-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done",
 595                         ]:
 596                 sequences.update({n : "restart_node_rins"})
 597
 598         #       restart_node_boot
 599         for n in ["bminit-cfg-auth-getplc-implementerror-bootupdatefail-update-debug-done",
 600                          "bminit-cfg-auth-implementerror-bootcheckfail-update-debug-done",
 601                          "bminit-cfg-auth-implementerror-bootcheckfail-update-implementerror-bootupdatefail-done",
 602                          "bminit-cfg-auth-getplc-update-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nospace-update-debug-done",
 603                          "bminit-cfg-auth-getplc-hardware-installinit-installdisk-installbootfs-exception-downloadfail-update-debug-done",
 604                          ]:
 605                 sequences.update({n: "restart_node_boot"})
 606
 607         # update_node_config_email
 608         for n in ["bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-done",
 609                         "bminit-cfg-exception-update-bootupdatefail-nonode-debug-done",
 610                         ]:
 611                 sequences.update({n : "update_node_config_email"})
 612
 613         for n in [ "bminit-cfg-exception-nodehostname-update-debug-done", ]:
 614                 sequences.update({n : "nodenetwork_email"})
 615
 616         # update_bootcd_email
 617         for n in ["bminit-cfg-auth-getplc-update-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done",
 618                         "bminit-cfg-auth-getplc-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done",
 619                         "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done",
 620                         "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done",
 621                         "bminit-cfg-auth-getplc-hardware-exception-hardwarerequirefail-update-debug-done",
 622                         ]:
 623                 sequences.update({n : "update_bootcd_email"})
 624
 625         for n in [ "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-update3-implementerror-nofilereference-update-debug-done",
 626                         ]:
 627                 sequences.update({n: "suspect_error_email"})
 628
 629         # update_hardware_email
 630         sequences.update({"bminit-cfg-auth-getplc-hardware-exception-disktoosmall-hardwarerequirefail-update-debug-done" : "update_hardware_email"})
 631         sequences.update({"bminit-cfg-auth-getplc-hardware-disktoosmall-exception-hardwarerequirefail-update-debug-done" : "update_hardware_email"})
 632
 633         # broken_hardware_email
 634         sequences.update({"bminit-cfg-auth-getplc-update-hardware-exception-hardwarerequirefail-update-debug-done" : "broken_hardware_email"})
 635
 636         # bad_dns_email
 637         sequences.update({"bminit-cfg-update-implementerror-bootupdatefail-dnserror-update-implementerror-bootupdatefail-dnserror-done" : "bad_dns_email"})
 638
 639         flag_set = True
 640
 641
 642         if s not in sequences:
 643                 print "   HOST %s" % hostname
 644                 print "   UNKNOWN SEQUENCE: %s" % s
 645
 646                 args = {}
 647                 args['hostname'] = hostname
 648                 args['sequence'] = s
 649                 args['bmlog'] = conn.get_bootmanager_log().read()
 650                 m = PersistMessage(hostname, mailtxt.unknownsequence[0] % args,
 651                                                                          mailtxt.unknownsequence[1] % args, False, db='unknown_persistmessages')
 652                 m.reset()
 653                 m.send(['monitor-list@lists.planet-lab.org'])
 654
 655                 conn.restart_bootmanager('boot')
 656
 657                 # NOTE: Do not set the pflags value for this sequence if it's unknown.
 658                 # This way, we can check it again after we've fixed it.
 659                 flag_set = False
 660
 661         else:
 662
 663                 if   sequences[s] == "restart_bootmanager_boot":
 664                         if config and not config.quiet: print "...Restarting BootManager.py on %s "% node
 665                         conn.restart_bootmanager('boot')
 666                 elif sequences[s] == "restart_bootmanager_rins":
 667                         if config and not config.quiet: print "...Restarting BootManager.py on %s "% node
 668                         conn.restart_bootmanager('rins')
 669                 elif sequences[s] == "restart_node_rins":
 670                         conn.restart_node('rins')
 671                 elif sequences[s] == "restart_node_boot":
 672                         conn.restart_node('boot')
 673                 elif sequences[s] == "repair_node_keys":
 674                         if conn.compare_and_repair_nodekeys():
 675                                 # the keys either are in sync or were forced in sync.
 676                                 # so try to reboot the node again.
 677                                 conn.restart_bootmanager('rins')
 678                                 pass
 679                         else:
 680                                 # there was some failure to synchronize the keys.
 681                                 print "...Unable to repair node keys on %s" % node
 682
 683                 elif sequences[s] == "suspect_error_email":
 684                         args = {}
 685                         args['hostname'] = hostname
 686                         args['sequence'] = s
 687                         args['bmlog'] = conn.get_bootmanager_log().read()
 688                         m = PersistMessage(hostname, "Suspicous error from BootManager on %s" % args,
 689                                                                                  mailtxt.unknownsequence[1] % args, False, db='suspect_persistmessages')
 690                         m.reset()
 691                         m.send(['monitor-list@lists.planet-lab.org'])
 692
 693                         conn.restart_bootmanager('boot')
 694
 695                 elif sequences[s] == "update_node_config_email":
 696                         print "...Sending message to UPDATE NODE CONFIG"
 697                         args = {}
 698                         args['hostname'] = hostname
 699                         m = PersistMessage(hostname,  mailtxt.plnode_cfg[0] % args,  mailtxt.plnode_cfg[1] % args,
 700                                                                 True, db='nodeid_persistmessages')
 701                         loginbase = plc.siteId(hostname)
 702                         m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 703                         conn.dump_plconf_file()
 704                         conn.set_nodestate('disable')
 705
 706                 elif sequences[s] == "nodenetwork_email":
 707                         print "...Sending message to LOOK AT NODE NETWORK"
 708                         args = {}
 709                         args['hostname'] = hostname
 710                         args['bmlog'] = conn.get_bootmanager_log().read()
 711                         m = PersistMessage(hostname,  mailtxt.plnode_network[0] % args,  mailtxt.plnode_cfg[1] % args,
 712                                                                 True, db='nodenet_persistmessages')
 713                         loginbase = plc.siteId(hostname)
 714                         m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 715                         conn.dump_plconf_file()
 716                         conn.set_nodestate('disable')
 717
 718                 elif sequences[s] == "update_bootcd_email":
 719                         print "...NOTIFY OWNER TO UPDATE BOOTCD!!!"
 720                         import getconf
 721                         args = {}
 722                         args.update(getconf.getconf(hostname)) # NOTE: Generates boot images for the user:
 723                         args['hostname_list'] = "%s" % hostname
 724
 725                         m = PersistMessage(hostname, "Please Update Boot Image for %s" % hostname,
 726                                                                 mailtxt.newalphacd_one[1] % args, True, db='bootcd_persistmessages')
 727
 728                         loginbase = plc.siteId(hostname)
 729                         m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 730
 731                         print "\tDisabling %s due to out-of-date BOOTCD" % hostname
 732                         conn.set_nodestate('disable')
 733
 734                 elif sequences[s] == "broken_hardware_email":
 735                         # MAKE An ACTION record that this host has failed hardware.  May
 736                         # require either an exception "/minhw" or other manual intervention.
 737                         # Definitely need to send out some more EMAIL.
 738                         print "...NOTIFYING OWNERS OF BROKEN HARDWARE on %s!!!" % hostname
 739                         # TODO: email notice of broken hardware
 740                         args = {}
 741                         args['hostname'] = hostname
 742                         args['log'] = conn.get_dmesg().read()
 743                         m = PersistMessage(hostname, mailtxt.baddisk[0] % args,
 744                                                                                  mailtxt.baddisk[1] % args, True, db='hardware_persistmessages')
 745
 746                         loginbase = plc.siteId(hostname)
 747                         m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 748                         conn.set_nodestate('disable')
 749
 750                 elif sequences[s] == "update_hardware_email":
 751                         print "...NOTIFYING OWNERS OF MINIMAL HARDWARE FAILURE on %s!!!" % hostname
 752                         args = {}
 753                         args['hostname'] = hostname
 754                         args['bmlog'] = conn.get_bootmanager_log().read()
 755                         m = PersistMessage(hostname, mailtxt.minimalhardware[0] % args,
 756                                                                                  mailtxt.minimalhardware[1] % args, True, db='minhardware_persistmessages')
 757
 758                         loginbase = plc.siteId(hostname)
 759                         m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 760                         conn.set_nodestate('disable')
 761
 762                 elif sequences[s] == "bad_dns_email":
 763                         print "...NOTIFYING OWNERS OF DNS FAILURE on %s!!!" % hostname
 764                         args = {}
 765                         try:
 766                                 node = api.GetNodes(hostname)[0]
 767                                 net = api.GetNodeNetworks(node['nodenetwork_ids'])[0]
 768                         except:
 769                                 print traceback.print_exc()
 770                                 # TODO: api error. skip email, b/c all info is not available,
 771                                 # flag_set will not be recorded.
 772                                 return False
 773                         nodenet_str = network_config_to_str(net)
 774
 775                         args['hostname'] = hostname
 776                         args['network_config'] = nodenet_str
 777                         args['nodenetwork_id'] = net['nodenetwork_id']
 778                         m = PersistMessage(hostname, mailtxt.baddns[0] % args,
 779                                                                                  mailtxt.baddns[1] % args, True, db='baddns_persistmessages')
 780
 781                         loginbase = plc.siteId(hostname)
 782                         m.send([policy.PIEMAIL % loginbase, policy.TECHEMAIL % loginbase])
 783                         conn.set_nodestate('disable')
 784
 785         if flag_set:
 786                 pflags.setRecentFlag(s)
 787                 pflags.save()
 788
 789         return True
 790
 791
 792 # MAIN -------------------------------------------------------------------
 793
 794 def main():
 795         import parser as parsermodule
 796         parser = parsermodule.getParser()
 797
 798         parser.set_defaults(child=False, collect=False, nosetup=False, verbose=False,
 799                                                 force=None, quiet=False)
 800         parser.add_option("", "--child", dest="child", action="store_true",
 801                                                 help="This is the child mode of this process.")
 802         parser.add_option("", "--force", dest="force", metavar="boot_state",
 803                                                 help="Force a boot state passed to BootManager.py.")
 804         parser.add_option("", "--quiet", dest="quiet", action="store_true",
 805                                                 help="Extra quiet output messages.")
 806         parser.add_option("", "--verbose", dest="verbose", action="store_true",
 807                                                 help="Extra debug output messages.")
 808         parser.add_option("", "--nonet", dest="nonet", action="store_true",
 809                                                 help="Do not setup the network, use existing log files to re-run a test pass.")
 810         parser.add_option("", "--collect", dest="collect", action="store_true",
 811                                                 help="No action, just collect dmesg, and bm.log")
 812         parser.add_option("", "--nosetup", dest="nosetup", action="store_true",
 813                                                 help="Do not perform the orginary setup phase.")
 814
 815         parser = parsermodule.getParser(['nodesets', 'defaults'], parser)
 816         config = parsermodule.parse_args(parser)
 817
 818         if config.nodelist:
 819                 nodes = config.getListFromFile(config.nodelist)
 820         elif config.node:
 821                 nodes = [ config.node ]
 822         else:
 823                 parser.print_help()
 824                 sys.exit(1)
 825
 826         for node in nodes:
 827                 reboot(node, config)
 828
 829 if __name__ == "__main__":
 830         main()