grouprins.py

   1 #!/usr/bin/python
   2
   3 # This script is used to manipulate the operational state of nodes in
   4 # different node groups.  These are basically set operations on nodes via the
   5 # PLC api.
   6 #
   7 # Take the ng name as an argument....
   8 # optionally,
   9 #  * get a list of nodes in the given nodegroup.
  10 #  * set some or all in the set to rins.
  11 #  * restart them all.
  12 #  * do something else to them all.
  13 #
  14
  15 import plc
  16 api = plc.getAuthAPI()
  17
  18 import policy
  19 import traceback
  20 from config import config as cfg
  21 import config as configmodule
  22 from optparse import OptionParser
  23
  24 from nodecommon import *
  25 from nodequery import verify,query_to_dict,node_select
  26 import database
  27 from unified_model import *
  28 import os
  29
  30 import time
  31
  32 from model import *
  33 import bootman          # debug nodes
  34 import monitor          # down nodes with pcu
  35 import reboot           # down nodes without pcu
  36 from emailTxt import mailtxt
  37 #reboot.verbose = 0
  38 import sys
  39
  40 class Reboot(object):
  41         def __init__(self, fbnode):
  42                 self.fbnode = fbnode
  43
  44         def _send_pcunotice(self, host):
  45                 args = {}
  46                 args['hostname'] = host
  47                 try:
  48                         args['pcu_id'] = plc.getpcu(host)['pcu_id']
  49                 except:
  50                         args['pcu_id'] = host
  51
  52                 m = PersistMessage(host, mailtxt.pcudown_one[0] % args,
  53                                                                  mailtxt.pcudown_one[1] % args, True, db='pcu_persistmessages')
  54
  55                 loginbase = plc.siteId(host)
  56                 m.send([policy.TECHEMAIL % loginbase])
  57
  58         def pcu(self, host):
  59                 # TODO: It should be possible to diagnose the various conditions of
  60                 #               the PCU here, and send different messages as appropriate.
  61                 if self.fbnode['pcu'] == "PCU":
  62                         self.action = "reboot.reboot('%s')" % host
  63
  64                         pflags = PersistFlags(host, 2*60*60*24, db='pcu_persistflags')
  65                         if not pflags.getRecentFlag('pcutried'):
  66                                 pflags.setRecentFlag('pcutried')
  67                                 try:
  68                                         ret = reboot.reboot(host)
  69
  70                                         pflags.save()
  71                                         return ret
  72
  73                                 except Exception,e:
  74                                         print traceback.print_exc(); print e
  75
  76                                         # NOTE: this failure could be an implementation issue on
  77                                         #               our end.  So, extra notices are confusing...
  78                                         # self._send_pcunotice(host)
  79
  80                                         pflags.setRecentFlag('pcufailed')
  81                                         pflags.save()
  82                                         return False
  83                         else:
  84                                 # we've tried the pcu recently, but it didn't work,
  85                                 # so did we send a message about it recently?
  86                                 if not pflags.getRecentFlag('pcumessagesent'):
  87
  88                                         self._send_pcunotice(host)
  89
  90                                         pflags.setRecentFlag('pcumessagesent')
  91                                         pflags.save()
  92                                         # NOTE: this will result in just one message sent at a time.
  93                                         return True
  94
  95                                 else:
  96                                         return False
  97                 else:
  98                         self.action = "None"
  99                         return False
 100
 101         def mail(self, host):
 102
 103                 # Reset every 4 weeks or so
 104                 pflags = PersistFlags(host, 27*60*60*24, db='mail_persistflags')
 105                 if not pflags.getRecentFlag('endrecord'):
 106                         node_end_record(host)
 107                         pflags.setRecentFlag('endrecord')
 108                         pflags.save()
 109
 110                 # Then in either case, run monitor.reboot()
 111                 self.action = "monitor.reboot('%s')" % host
 112                 try:
 113                         return monitor.reboot(host)
 114                 except Exception, e:
 115                         print traceback.print_exc(); print e
 116                         return False
 117
 118 class RebootDebug(Reboot):
 119
 120         def direct(self, host):
 121                 self.action = "bootman.reboot('%s', config, None)" % host
 122                 return bootman.reboot(host, config, None)
 123
 124 class RebootBoot(Reboot):
 125
 126         def direct(self, host):
 127                 self.action = "bootman.reboot('%s', config, 'reboot')" % host
 128                 return bootman.reboot(host, config, 'reboot')
 129
 130 class RebootDown(Reboot):
 131
 132         def direct(self, host):
 133                 self.action = "None"
 134                 return False    # this always fails, since the node will be down.
 135
 136 def set_node_to_rins(host, fb):
 137
 138         node = api.GetNodes(host, ['boot_state', 'last_contact', 'last_updated', 'date_created'])
 139         record = {'observation' : node[0],
 140                           'model' : 'USER_REQUEST',
 141                           'action' : 'api.UpdateNode(%s, {"boot_state" : "rins"})' % host,
 142                           'time' : time.time()}
 143         l = Log(host, record)
 144
 145         ret = api.UpdateNode(host, {'boot_state' : 'rins'})
 146         if ret:
 147                 # it's nice to see the current status rather than the previous status on the console
 148                 node = api.GetNodes(host)[0]
 149                 print l
 150                 print "%-2d" % (i-1), nodegroup_display(node, fb)
 151                 return l
 152         else:
 153                 print "FAILED TO UPDATE NODE BOOT STATE : %s" % host
 154                 return None
 155
 156
 157 try:
 158         rebootlog = database.dbLoad("rebootlog")
 159 except:
 160         rebootlog = LogRoll()
 161
 162 parser = OptionParser()
 163 parser.set_defaults(nodegroup=None,
 164                                         node=None,
 165                                         nodelist=None,
 166                                         nodeselect=None,
 167                                         timewait=0,
 168                                         skip=0,
 169                                         rins=False,
 170                                         reboot=False,
 171                                         findbad=False,
 172                                         force=False,
 173                                         nosetup=False,
 174                                         verbose=False,
 175                                         stopkey=None,
 176                                         stopvalue=None,
 177                                         quiet=False,
 178                                         )
 179 parser.add_option("", "--node", dest="node", metavar="nodename.edu",
 180                                         help="A single node name to add to the nodegroup")
 181 parser.add_option("", "--nodelist", dest="nodelist", metavar="list.txt",
 182                                         help="Use all nodes in the given file for operation.")
 183 parser.add_option("", "--nodegroup", dest="nodegroup", metavar="NodegroupName",
 184                                         help="Specify a nodegroup to perform actions on")
 185 parser.add_option("", "--nodeselect", dest="nodeselect", metavar="querystring",
 186                                         help="Specify a query to perform on findbad db")
 187
 188 parser.add_option("", "--verbose", dest="verbose", action="store_true",
 189                                         help="Extra debug output messages.")
 190 parser.add_option("", "--nosetup", dest="nosetup", action="store_true",
 191                                         help="Do not perform the orginary setup phase.")
 192
 193 parser.add_option("", "--skip", dest="skip",
 194                                         help="Number of machines to skip on the input queue.")
 195 parser.add_option("", "--timewait", dest="timewait",
 196                                         help="Minutes to wait between iterations of 10 nodes.")
 197
 198 parser.add_option("", "--stopselect", dest="stopselect", metavar="",
 199                                         help="The select string that must evaluate to true for the node to be considered 'done'")
 200
 201 parser.add_option("", "--stopkey", dest="stopkey", metavar="",
 202                                         help="")
 203 parser.add_option("", "--stopvalue", dest="stopvalue", metavar="",
 204                                         help="")
 205
 206 parser.add_option("", "--findbad", dest="findbad", action="store_true",
 207                                         help="Re-run findbad on the nodes we're going to check before acting.")
 208 parser.add_option("", "--force", dest="force", action="store_true",
 209                                         help="Force action regardless of previous actions/logs.")
 210 parser.add_option("", "--rins", dest="rins", action="store_true",
 211                                         help="Set the boot_state to 'rins' for all nodes.")
 212 parser.add_option("", "--reboot", dest="reboot", action="store_true",
 213                                         help="Actively try to reboot the nodes, keeping a log of actions.")
 214 #config = config(parser)
 215 config = cfg(parser)
 216 config.parse_args()
 217
 218 # COLLECT nodegroups, nodes and node lists
 219 if config.nodegroup:
 220         ng = api.GetNodeGroups({'name' : config.nodegroup})
 221         nodelist = api.GetNodes(ng[0]['node_ids'])
 222         hostnames = [ n['hostname'] for n in nodelist ]
 223
 224 if config.node or config.nodelist:
 225         if config.node: hostnames = [ config.node ]
 226         else: hostnames = config.getListFromFile(config.nodelist)
 227
 228 if config.nodeselect:
 229         hostnames = node_select(config.nodeselect)
 230
 231 if config.findbad:
 232         # rerun findbad with the nodes in the given nodes.
 233         file = "findbad.txt"
 234         configmodule.setFileFromList(file, hostnames)
 235         os.system("./findbad.py --cachenodes --debug=0 --dbname=findbad --increment --nodelist %s" % file)
 236
 237 fb = database.dbLoad("findbad")
 238 # commands:
 239 i = 1
 240 count = 1
 241 for host in hostnames:
 242
 243         #if 'echo' in host or 'hptest-1' in host: continue
 244         try:
 245                 try:
 246                         node = api.GetNodes(host)[0]
 247                 except:
 248                         print traceback.print_exc();
 249                         print "FAILED GETNODES for host: %s" % host
 250                         continue
 251
 252                 print "%-2d" % i, nodegroup_display(node, fb)
 253                 i += 1
 254                 if i < int(config.skip): continue
 255
 256                 if config.stopselect:
 257                         dict_query = query_to_dict(config.stopselect)
 258                         fbnode = fb['nodes'][host]['values']
 259                         observed_state = get_current_state(fbnode)
 260
 261                         if verify(dict_query, fbnode) and observed_state != "dbg ":
 262                                 # evaluates to true, therefore skip.
 263                                 print "%s evaluates true for %s ; skipping..." % ( config.stopselect, host )
 264                                 continue
 265
 266                 if config.stopkey and config.stopvalue:
 267                         fbnode = fb['nodes'][host]['values']
 268                         observed_state = get_current_state(fbnode)
 269
 270                         if config.stopkey in fbnode:
 271                                 if config.stopvalue in fbnode[config.stopkey] and observed_state != "dbg ":
 272                                         print "%s has stopvalue; skipping..." % host
 273                                         continue
 274                         else:
 275                                 print "stopkey %s not in fbnode record for %s; skipping..." % (config.stopkey, host)
 276                                 print fbnode
 277                                 continue
 278
 279                 if not config.force and rebootlog.find(host, {'action' : ".*reboot"}, 60*60*2):
 280                         print "recently rebooted %s.  skipping... " % host
 281                         continue
 282
 283                 if config.reboot:
 284
 285                         fbnode = fb['nodes'][host]['values']
 286                         observed_state = get_current_state(fbnode)
 287
 288                         if       observed_state == "dbg ":
 289                                 o = RebootDebug(fbnode)
 290
 291                         elif observed_state == "boot" :
 292                                 if config.rins:
 293                                         l = set_node_to_rins(host, fb)
 294                                         if l: rebootlog.add(l)
 295
 296                                 o = RebootBoot(fbnode)
 297
 298                         elif observed_state == "down":
 299                                 if config.rins:
 300                                         l = set_node_to_rins(host, fb)
 301                                         if l: rebootlog.add(l)
 302
 303                                 o = RebootDown(fbnode)
 304
 305
 306                         if o.direct(host):
 307                                 record = {'observation' : "DIRECT_SUCCESS: %s" % observed_state,
 308                                                   'action' : o.action,
 309                                                   'model' : "none",
 310                                                   'time' : time.time()}
 311                         elif o.pcu(host):
 312                                 record = {'observation' : "PCU_SUCCESS: %s" % observed_state,
 313                                                   'action' : o.action,
 314                                                   'model' : "none",
 315                                                   'time' : time.time()}
 316                         elif o.mail(host):
 317                                 record = {'observation' : "MAIL_SUCCESS: %s" % observed_state,
 318                                                   'action' : o.action,
 319                                                   'model' : "none",
 320                                                   'time' : time.time()}
 321                         else:
 322                                 record = {'observation' : "REBOOT_FAILED: %s" %  observed_state,
 323                                                   'action' : "log failure",
 324                                                   'model' : "none",
 325                                                   'time' : time.time()}
 326
 327                                 print "ALL METHODS OF RESTARTING %s FAILED" % host
 328                                 args = {}
 329                                 args['hostname'] = host
 330                                 m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
 331                                                                                          "CANNOT CONTACT", False, db='suspect_persistmessages')
 332                                 m.reset()
 333                                 m.send(['monitor-list@lists.planet-lab.org'])
 334
 335                         l = Log(host, record)
 336                         print l
 337                         rebootlog.add(l)
 338         except KeyboardInterrupt:
 339                 print "Killed by interrupt"
 340                 sys.exit(0)
 341         except:
 342                 print traceback.print_exc();
 343                 print "Continuing..."
 344
 345         time.sleep(1)
 346         if count % 10 == 0:
 347                 print "Saving rebootlog"
 348                 database.dbDump("rebootlog", rebootlog)
 349                 wait_time = int(config.timewait)
 350                 print "Sleeping %d minutes" % wait_time
 351                 ti = 0
 352                 print "Minutes slept: ",
 353                 sys.stdout.flush()
 354                 while ti < wait_time:
 355                         print "%s" % ti,
 356                         sys.stdout.flush()
 357                         time.sleep(60)
 358                         ti = ti+1
 359
 360         count = count + 1
 361
 362 print "Saving rebootlog"
 363 database.dbDump("rebootlog", rebootlog)