grouprins.py

   1 #!/usr/bin/python
   2
   3 # This script is used to manipulate the operational state of nodes in
   4 # different node groups.  These are basically set operations on nodes via the
   5 # PLC api.
   6 #
   7 # Take the ng name as an argument....
   8 # optionally,
   9 #  * get a list of nodes in the given nodegroup.
  10 #  * set some or all in the set to rins.
  11 #  * restart them all.
  12 #  * do something else to them all.
  13 #
  14
  15 import plc
  16 api = plc.getAuthAPI()
  17
  18 import traceback
  19 import config
  20 import util.file
  21 from optparse import OptionParser
  22
  23 import const
  24 from nodecommon import *
  25 from nodequery import verify,query_to_dict,node_select
  26 import database
  27 from unified_model import *
  28 import os
  29
  30 import time
  31 import parser as parsermodule
  32
  33 from model import *
  34 import bootman          # debug nodes
  35 import reboot           # down nodes without pcu
  36 import mailmonitor      # down nodes with pcu
  37 from emailTxt import mailtxt
  38 #reboot.verbose = 0
  39 import sys
  40
  41 class Reboot(object):
  42         def __init__(self, fbnode):
  43                 self.fbnode = fbnode
  44
  45         def _send_pcunotice(self, host):
  46                 args = {}
  47                 args['hostname'] = host
  48                 try:
  49                         args['pcu_id'] = plc.getpcu(host)['pcu_id']
  50                 except:
  51                         args['pcu_id'] = host
  52
  53                 m = PersistMessage(host, mailtxt.pcudown_one[0] % args,
  54                                                                  mailtxt.pcudown_one[1] % args, True, db='pcu_persistmessages')
  55
  56                 loginbase = plc.siteId(host)
  57                 m.send([const.TECHEMAIL % loginbase])
  58
  59         def pcu(self, host):
  60                 # TODO: It should be possible to diagnose the various conditions of
  61                 #               the PCU here, and send different messages as appropriate.
  62                 print "'%s'" % self.fbnode['pcu']
  63                 if self.fbnode['pcu'] == "PCU" or "PCUOK" in self.fbnode['pcu']:
  64                         self.action = "reboot.reboot('%s')" % host
  65
  66                         pflags = PersistFlags(host, 2*60*60*24, db='pcu_persistflags')
  67                         #pflags.resetRecentFlag('pcutried')
  68                         if not pflags.getRecentFlag('pcutried'):
  69                                 try:
  70                                         print "CALLING REBOOT!!!"
  71                                         ret = reboot.reboot(host)
  72
  73                                         pflags.setRecentFlag('pcutried')
  74                                         pflags.save()
  75                                         return ret
  76
  77                                 except Exception,e:
  78                                         print traceback.print_exc(); print e
  79
  80                                         # NOTE: this failure could be an implementation issue on
  81                                         #               our end.  So, extra notices are confusing...
  82                                         # self._send_pcunotice(host)
  83
  84                                         pflags.setRecentFlag('pcufailed')
  85                                         pflags.save()
  86                                         return False
  87
  88                         elif not pflags.getRecentFlag('pcu_rins_tried'):
  89                                 try:
  90                                         # set node to 'rins' boot state.
  91                                         print "CALLING REBOOT +++ RINS"
  92                                         plc.nodeBootState(host, 'rins')
  93                                         ret = reboot.reboot(host)
  94
  95                                         pflags.setRecentFlag('pcu_rins_tried')
  96                                         pflags.save()
  97                                         return ret
  98
  99                                 except Exception,e:
 100                                         print traceback.print_exc(); print e
 101
 102                                         # NOTE: this failure could be an implementation issue on
 103                                         #               our end.  So, extra notices are confusing...
 104                                         # self._send_pcunotice(host)
 105
 106                                         pflags.setRecentFlag('pcufailed')
 107                                         pflags.save()
 108                                         return False
 109                         else:
 110                                 # we've tried the pcu recently, but it didn't work,
 111                                 # so did we send a message about it recently?
 112                                 if not pflags.getRecentFlag('pcumessagesent'):
 113
 114                                         self._send_pcunotice(host)
 115
 116                                         pflags.setRecentFlag('pcumessagesent')
 117                                         pflags.save()
 118
 119                                 # This will result in mail() being called next, to try to
 120                                 # engage the technical contact to take care of it also.
 121                                 print "RETURNING FALSE"
 122                                 return False
 123
 124                 else:
 125                         print "NO PCUOK"
 126                         self.action = "None"
 127                         return False
 128
 129         def mail(self, host):
 130
 131                 # Reset every 4 weeks or so
 132                 pflags = PersistFlags(host, 27*60*60*24, db='mail_persistflags')
 133                 if not pflags.getRecentFlag('endrecord'):
 134                         node_end_record(host)
 135                         pflags.setRecentFlag('endrecord')
 136                         pflags.save()
 137
 138                 # Then in either case, run mailmonitor.reboot()
 139                 self.action = "mailmonitor.reboot('%s')" % host
 140                 try:
 141                         return mailmonitor.reboot(host)
 142                 except Exception, e:
 143                         print traceback.print_exc(); print e
 144                         return False
 145
 146 class RebootDebug(Reboot):
 147
 148         def direct(self, host):
 149                 self.action = "bootman.reboot('%s', config, None)" % host
 150                 return bootman.reboot(host, config, None)
 151
 152 class RebootBoot(Reboot):
 153
 154         def direct(self, host):
 155                 self.action = "bootman.reboot('%s', config, 'reboot')" % host
 156                 return bootman.reboot(host, config, 'reboot')
 157
 158 class RebootDown(Reboot):
 159
 160         def direct(self, host):
 161                 self.action = "None"
 162                 return False    # this always fails, since the node will be down.
 163
 164 def set_node_to_rins(host, fb):
 165
 166         node = api.GetNodes(host, ['boot_state', 'last_contact', 'last_updated', 'date_created'])
 167         record = {'observation' : node[0],
 168                           'model' : 'USER_REQUEST',
 169                           'action' : 'api.UpdateNode(%s, {"boot_state" : "rins"})' % host,
 170                           'time' : time.time()}
 171         l = Log(host, record)
 172
 173         ret = api.UpdateNode(host, {'boot_state' : 'rins'})
 174         if ret:
 175                 # it's nice to see the current status rather than the previous status on the console
 176                 node = api.GetNodes(host)[0]
 177                 print l
 178                 print "%-2d" % (i-1), nodegroup_display(node, fb)
 179                 return l
 180         else:
 181                 print "FAILED TO UPDATE NODE BOOT STATE : %s" % host
 182                 return None
 183
 184
 185 try:
 186         rebootlog = database.dbLoad("rebootlog")
 187 except:
 188         rebootlog = LogRoll()
 189
 190 parser = parsermodule.getParser(['nodesets'])
 191 parser.set_defaults( timewait=0,
 192                                         skip=0,
 193                                         rins=False,
 194                                         reboot=False,
 195                                         findbad=False,
 196                                         force=False,
 197                                         nosetup=False,
 198                                         verbose=False,
 199                                         quiet=False,
 200                                         )
 201
 202 parser.add_option("", "--stopselect", dest="stopselect", metavar="",
 203                                         help="The select string that must evaluate to true for the node to be considered 'done'")
 204 parser.add_option("", "--findbad", dest="findbad", action="store_true",
 205                                         help="Re-run findbad on the nodes we're going to check before acting.")
 206 parser.add_option("", "--force", dest="force", action="store_true",
 207                                         help="Force action regardless of previous actions/logs.")
 208 parser.add_option("", "--rins", dest="rins", action="store_true",
 209                                         help="Set the boot_state to 'rins' for all nodes.")
 210 parser.add_option("", "--reboot", dest="reboot", action="store_true",
 211                                         help="Actively try to reboot the nodes, keeping a log of actions.")
 212
 213 parser.add_option("", "--verbose", dest="verbose", action="store_true",
 214                                         help="Extra debug output messages.")
 215 parser.add_option("", "--nosetup", dest="nosetup", action="store_true",
 216                                         help="Do not perform the orginary setup phase.")
 217 parser.add_option("", "--skip", dest="skip",
 218                                         help="Number of machines to skip on the input queue.")
 219 parser.add_option("", "--timewait", dest="timewait",
 220                                         help="Minutes to wait between iterations of 10 nodes.")
 221
 222 parser = parsermodule.getParser(['defaults'], parser)
 223 config = parsermodule.parse_args(parser)
 224
 225 # COLLECT nodegroups, nodes and node lists
 226 if config.nodegroup:
 227         ng = api.GetNodeGroups({'name' : config.nodegroup})
 228         nodelist = api.GetNodes(ng[0]['node_ids'])
 229         hostnames = [ n['hostname'] for n in nodelist ]
 230
 231 if config.node or config.nodelist:
 232         if config.node: hostnames = [ config.node ]
 233         else: hostnames = util.file.getListFromFile(config.nodelist)
 234
 235 fb = database.dbLoad("findbad")
 236
 237 if config.nodeselect:
 238         hostnames = node_select(config.nodeselect, fb['nodes'].keys(), fb)
 239
 240 if config.findbad:
 241         # rerun findbad with the nodes in the given nodes.
 242         file = "findbad.txt"
 243         util.file.setFileFromList(file, hostnames)
 244         os.system("./findbad.py --cachenodes --increment --nodelist %s" % file)
 245         # TODO: shouldn't we reload the node list now?
 246
 247 l_blacklist = database.if_cached_else(1, "l_blacklist", lambda : [])
 248 # commands:
 249 i = 1
 250 count = 1
 251 #print "hosts: %s" % hostnames
 252 for host in hostnames:
 253
 254         #if 'echo' in host or 'hptest-1' in host: continue
 255
 256         try:
 257                 try:
 258                         node = api.GetNodes(host)[0]
 259                 except:
 260                         print traceback.print_exc();
 261                         print "FAILED GETNODES for host: %s" % host
 262                         continue
 263
 264                 print "%-2d" % i, nodegroup_display(node, fb)
 265                 i += 1
 266                 if i-1 <= int(config.skip): continue
 267                 if host in l_blacklist:
 268                         print "%s is blacklisted.  Skipping." % host
 269                         continue
 270
 271                 if config.stopselect:
 272                         dict_query = query_to_dict(config.stopselect)
 273                         fbnode = fb['nodes'][host]['values']
 274                         observed_state = get_current_state(fbnode)
 275
 276                         if verify(dict_query, fbnode) and observed_state != "dbg ":
 277                                 # evaluates to true, therefore skip.
 278                                 print "%s evaluates true for %s ; skipping..." % ( config.stopselect, host )
 279                                 try:
 280                                         # todo: clean up act_all record here.
 281                                         # todo: send thank you, etc.
 282                                         mailmonitor.reboot(host)
 283                                 except Exception, e:
 284                                         print traceback.print_exc(); print e
 285
 286                                 continue
 287                         #else:
 288                                 #print "%s failed to match %s: -%s-" % ( host, dict_query, observed_state )
 289                                 #sys.exit(1)
 290
 291                 if not config.force and rebootlog.find(host, {'action' : ".*reboot"}, 60*60*2):
 292                         print "recently rebooted %s.  skipping... " % host
 293                         continue
 294
 295                 if config.reboot:
 296
 297                         fbnode = fb['nodes'][host]['values']
 298                         observed_state = get_current_state(fbnode)
 299
 300                         if       observed_state == "dbg ":
 301                                 o = RebootDebug(fbnode)
 302
 303                         elif observed_state == "boot" :
 304                                 if config.rins:
 305                                         l = set_node_to_rins(host, fb)
 306                                         if l: rebootlog.add(l)
 307
 308                                 o = RebootBoot(fbnode)
 309
 310                         elif observed_state == "down":
 311                                 if config.rins:
 312                                         l = set_node_to_rins(host, fb)
 313                                         if l: rebootlog.add(l)
 314
 315                                 o = RebootDown(fbnode)
 316
 317
 318                         if o.direct(host):
 319                                 record = {'observation' : "DIRECT_SUCCESS: %s" % observed_state,
 320                                                   'action' : o.action,
 321                                                   'model' : "none",
 322                                                   'time' : time.time()}
 323                         elif o.pcu(host):
 324                                 record = {'observation' : "PCU_SUCCESS: %s" % observed_state,
 325                                                   'action' : o.action,
 326                                                   'model' : "none",
 327                                                   'time' : time.time()}
 328                         elif o.mail(host):
 329                                 record = {'observation' : "MAIL_SUCCESS: %s" % observed_state,
 330                                                   'action' : o.action,
 331                                                   'model' : "none",
 332                                                   'time' : time.time()}
 333                         else:
 334                                 record = {'observation' : "REBOOT_FAILED: %s" %  observed_state,
 335                                                   'action' : "log failure",
 336                                                   'model' : "none",
 337                                                   'time' : time.time()}
 338
 339                                 print "ALL METHODS OF RESTARTING %s FAILED" % host
 340                                 args = {}
 341                                 args['hostname'] = host
 342                                 m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
 343                                                                                          "CANNOT CONTACT", False, db='suspect_persistmessages')
 344                                 m.reset()
 345                                 m.send(['monitor-list@lists.planet-lab.org'])
 346
 347                         l = Log(host, record)
 348                         print l
 349                         rebootlog.add(l)
 350         except KeyboardInterrupt:
 351                 print "Killed by interrupt"
 352                 sys.exit(0)
 353         except:
 354                 print traceback.print_exc();
 355                 print "Continuing..."
 356
 357         time.sleep(1)
 358         if count % 10 == 0:
 359                 print "Saving rebootlog"
 360                 database.dbDump("rebootlog", rebootlog)
 361                 wait_time = int(config.timewait)
 362                 print "Sleeping %d minutes" % wait_time
 363                 ti = 0
 364                 print "Minutes slept: ",
 365                 sys.stdout.flush()
 366                 while ti < wait_time:
 367                         print "%s" % ti,
 368                         sys.stdout.flush()
 369                         time.sleep(60)
 370                         ti = ti+1
 371
 372         count = count + 1
 373
 374 print "Saving rebootlog"
 375 database.dbDump("rebootlog", rebootlog)