grouprins.py

   1 #!/usr/bin/python
   2
   3 # This script is used to manipulate the operational state of nodes in
   4 # different node groups.  These are basically set operations on nodes via the
   5 # PLC api.
   6 #
   7 # Take the ng name as an argument....
   8 # optionally,
   9 #  * get a list of nodes in the given nodegroup.
  10 #  * set some or all in the set to rins.
  11 #  * restart them all.
  12 #  * do something else to them all.
  13 #
  14
  15 import plc
  16 import auth
  17 api = plc.PLC(auth.auth, auth.plc)
  18
  19 import policy
  20
  21 from config import config as cfg
  22 from optparse import OptionParser
  23
  24 from nodecommon import *
  25 from nodequery import verify,query_to_dict,node_select
  26 import soltesz
  27 from unified_model import *
  28
  29 import time
  30
  31 from model import *
  32 import bootman          # debug nodes
  33 import monitor          # down nodes with pcu
  34 import reboot           # down nodes without pcu
  35 reboot.verbose = 0
  36 import sys
  37
  38 class Reboot(object):
  39         def __init__(self, fbnode):
  40                 self.fbnode = fbnode
  41
  42         def _send_pcunotice(self, host):
  43                 args = {}
  44                 args['hostname'] = host
  45                 try:
  46                         args['pcu_id'] = plc.getpcu(host)['pcu_id']
  47                 except:
  48                         args['pcu_id'] = host
  49
  50                 m = PersistMessage(host, mailtxt.pcudown_one[0] % args,
  51                                                                  mailtxt.pcudown_one[1] % args, True, db='pcu_persistmessages')
  52
  53                 loginbase = plc.siteId(hostname)
  54                 m.send([policy.TECHEMAIL % loginbase])
  55
  56         def pcu(self, host):
  57                 # TODO: It should be possible to diagnose the various conditions of
  58                 #               the PCU here, and send different messages as appropriate.
  59                 if self.fbnode['pcu'] == "PCU":
  60                         self.action = "reboot.reboot('%s')" % host
  61
  62                         pflags = PersistFlags(host, 1*60*60*24, db='pcu_persistflags')
  63                         if not pflags.getRecentFlag('pcutried'): # or not pflags.getFlag('pcufailed'):
  64                                 pflags.setRecentFlag('pcutried')
  65                                 try:
  66                                         ret = reboot.reboot(host)
  67
  68                                         pflags.save()
  69                                         return ret
  70
  71                                 except Exception,e:
  72                                         import traceback; print traceback.print_exc(); print e
  73
  74                                         # NOTE: this failure could be an implementation issue on
  75                                         #               our end.  So, extra notices are confusing...
  76                                         # self._send_pcunotice(host)
  77
  78                                         pflags.setRecentFlag('pcufailed')
  79                                         pflags.save()
  80                                         return False
  81                         else:
  82                                 # we've tried the pcu recently, but it didn't work,
  83                                 # so did we send a message about it recently?
  84                                 if not pflags.getRecentFlag('pcumessagesent'):
  85
  86                                         self._send_pcunotice(host)
  87
  88                                         pflags.setRecentFlag('pcumessagesent')
  89                                         pflags.save()
  90
  91                                 else:
  92                                         pass # just skip it?
  93
  94                 else:
  95                         self.action = "None"
  96                         return False
  97
  98         def mail(self, host):
  99
 100                 # Reset every 4 weeks or so
 101                 pflags = PersistFlags(host, 27*60*60*24, db='mail_persistflags')
 102                 if not pflags.getRecentFlag('endrecord'):
 103                         node_end_record(host)
 104                         pflags.setRecentFlag('endrecord')
 105                         pflags.save()
 106
 107                 # Then in either case, run monitor.reboot()
 108                 self.action = "monitor.reboot('%s')" % host
 109                 try:
 110                         return monitor.reboot(host)
 111                 except Exception, e:
 112                         import traceback; print traceback.print_exc(); print e
 113                         return False
 114
 115 class RebootDebug(Reboot):
 116
 117         def direct(self, host):
 118                 self.action = "bootman.reboot('%s', config, None)" % host
 119                 return bootman.reboot(host, config, None)
 120
 121 class RebootBoot(Reboot):
 122
 123         def direct(self, host):
 124                 self.action = "bootman.reboot('%s', config, 'reboot')" % host
 125                 return bootman.reboot(host, config, 'reboot')
 126
 127 class RebootDown(Reboot):
 128
 129         def direct(self, host):
 130                 self.action = "None"
 131                 return False    # this always fails, since the node will be down.
 132
 133
 134 try:
 135         rebootlog = soltesz.dbLoad("rebootlog")
 136 except:
 137         rebootlog = LogRoll()
 138
 139 parser = OptionParser()
 140 parser.set_defaults(nodegroup=None,
 141                                         node=None,
 142                                         nodelist=None,
 143                                         nodeselect=None,
 144                                         timewait=30,
 145                                         skip=0,
 146                                         rins=False,
 147                                         reboot=False,
 148                                         findbad=False,
 149                                         force=False,
 150                                         nosetup=False,
 151                                         verbose=False,
 152                                         stopkey=None,
 153                                         stopvalue=None,
 154                                         quiet=False,
 155                                         )
 156 parser.add_option("", "--node", dest="node", metavar="nodename.edu",
 157                                         help="A single node name to add to the nodegroup")
 158 parser.add_option("", "--nodelist", dest="nodelist", metavar="list.txt",
 159                                         help="Use all nodes in the given file for operation.")
 160 parser.add_option("", "--nodegroup", dest="nodegroup", metavar="NodegroupName",
 161                                         help="Specify a nodegroup to perform actions on")
 162 parser.add_option("", "--nodeselect", dest="nodeselect", metavar="querystring",
 163                                         help="Specify a query to perform on findbad db")
 164
 165 parser.add_option("", "--verbose", dest="verbose", action="store_true",
 166                                         help="Extra debug output messages.")
 167 parser.add_option("", "--nosetup", dest="nosetup", action="store_true",
 168                                         help="Do not perform the orginary setup phase.")
 169
 170 parser.add_option("", "--skip", dest="skip",
 171                                         help="Number of machines to skip on the input queue.")
 172 parser.add_option("", "--timewait", dest="timewait",
 173                                         help="Minutes to wait between iterations of 10 nodes.")
 174
 175 parser.add_option("", "--stopselect", dest="stopselect", metavar="",
 176                                         help="The select string that must evaluate to true for the node to be considered 'done'")
 177
 178 parser.add_option("", "--stopkey", dest="stopkey", metavar="",
 179                                         help="")
 180 parser.add_option("", "--stopvalue", dest="stopvalue", metavar="",
 181                                         help="")
 182
 183 parser.add_option("", "--findbad", dest="findbad", action="store_true",
 184                                         help="Re-run findbad on the nodes we're going to check before acting.")
 185 parser.add_option("", "--force", dest="force", action="store_true",
 186                                         help="Force action regardless of previous actions/logs.")
 187 parser.add_option("", "--rins", dest="rins", action="store_true",
 188                                         help="Set the boot_state to 'rins' for all nodes.")
 189 parser.add_option("", "--reboot", dest="reboot", action="store_true",
 190                                         help="Actively try to reboot the nodes, keeping a log of actions.")
 191 #config = config(parser)
 192 config = cfg(parser)
 193 config.parse_args()
 194
 195 # COLLECT nodegroups, nodes and node lists
 196 if config.nodegroup:
 197         ng = api.GetNodeGroups({'name' : config.nodegroup})
 198         nodelist = api.GetNodes(ng[0]['node_ids'])
 199         hostnames = [ n['hostname'] for n in nodelist ]
 200
 201 if config.node or config.nodelist:
 202         if config.node: hostnames = [ config.node ]
 203         else: hostnames = config.getListFromFile(config.nodelist)
 204
 205 if config.nodeselect:
 206         hostnames = node_select(config.nodeselect)
 207
 208 if config.findbad:
 209         # rerun findbad with the nodes in the given nodes.
 210         import os
 211         file = "findbad.txt"
 212         config.setFileFromList(file, hostnames)
 213         os.system("./findbad.py --cachenodes --debug=0 --dbname=findbad --increment --nodelist %s" % file)
 214
 215 fb = soltesz.dbLoad("findbad")
 216 # commands:
 217 i = 1
 218 count = 1
 219 for host in hostnames:
 220
 221         #if 'echo' in host or 'hptest-1' in host: continue
 222
 223         try:
 224                 try:
 225                         node = api.GetNodes(host)[0]
 226                 except:
 227                         import traceback; print traceback.print_exc();
 228                         print "FAILED GETNODES for host: %s" % host
 229                         continue
 230
 231                 print "%-2d" % i, nodegroup_display(node, fb)
 232                 i += 1
 233                 if i < int(config.skip): continue
 234
 235                 if config.stopselect:
 236                         dict_query = query_to_dict(config.stopselect)
 237                         fbnode = fb['nodes'][host]['values']
 238                         observed_state = get_current_state(fbnode)
 239
 240                         if verify(dict_query, fbnode) and observed_state != "dbg ":
 241                                 # evaluates to true, therefore skip.
 242                                 print "%s evaluates true for %s ; skipping..." % ( config.stopselect, host )
 243                                 continue
 244
 245                 if config.stopkey and config.stopvalue:
 246                         fbnode = fb['nodes'][host]['values']
 247                         observed_state = get_current_state(fbnode)
 248
 249                         if config.stopkey in fbnode:
 250                                 if config.stopvalue in fbnode[config.stopkey] and observed_state != "dbg ":
 251                                         print "%s has stopvalue; skipping..." % host
 252                                         continue
 253                         else:
 254                                 print "stopkey %s not in fbnode record for %s; skipping..." % (config.stopkey, host)
 255                                 print fbnode
 256                                 continue
 257
 258                 if not config.force and rebootlog.find(host, {'action' : ".*reboot"}, 60*60*2):
 259                         print "recently rebooted %s.  skipping... " % host
 260                         continue
 261
 262                 if config.rins:
 263                         # reset the boot_state to 'rins'
 264                         node = api.GetNodes(host, ['boot_state', 'last_contact', 'last_updated', 'date_created'])
 265                         record = {'observation' : node[0],
 266                                           'model' : 'USER_REQUEST',
 267                                           'action' : 'api.UpdateNode(%s, {"boot_state" : "rins"})' % host,
 268                                           'time' : time.time()}
 269                         l = Log(host, record)
 270
 271                         ret = api.UpdateNode(host, {'boot_state' : 'rins'})
 272                         if ret:
 273                                 # it's nice to see the current status rather than the previous status on the console
 274                                 node = api.GetNodes(host)[0]
 275                                 print l
 276                                 print "%-2d" % (i-1), nodegroup_display(node, fb)
 277                                 rebootlog.add(l)
 278                         else:
 279                                 print "FAILED TO UPDATE NODE BOOT STATE : %s" % host
 280
 281
 282                 if config.reboot:
 283
 284                         fbnode = fb['nodes'][host]['values']
 285                         observed_state = get_current_state(fbnode)
 286
 287                         if       observed_state == "dbg ":
 288                                 o = RebootDebug(fbnode)
 289
 290                         elif observed_state == "boot" :
 291                                 o = RebootBoot(fbnode)
 292
 293                         elif observed_state == "down":
 294                                 o = RebootDown(fbnode)
 295
 296
 297                         if o.direct(host):
 298                                 record = {'observation' : "DIRECT_SUCCESS: %s" % observed_state,
 299                                                   'action' : o.action,
 300                                                   'model' : "none",
 301                                                   'time' : time.time()}
 302                         elif o.pcu(host):
 303                                 record = {'observation' : "PCU_SUCCESS: %s" % observed_state,
 304                                                   'action' : o.action,
 305                                                   'model' : "none",
 306                                                   'time' : time.time()}
 307                         elif o.mail(host):
 308                                 record = {'observation' : "MAIL_SUCCESS: %s" % observed_state,
 309                                                   'action' : o.action,
 310                                                   'model' : "none",
 311                                                   'time' : time.time()}
 312                         else:
 313                                 record = {'observation' : "REBOOT_FAILED: %s" %  observed_state,
 314                                                   'action' : "log failure",
 315                                                   'model' : "none",
 316                                                   'time' : time.time()}
 317
 318                                 print "ALL METHODS OF RESTARTING %s FAILED" % host
 319
 320                         l = Log(host, record)
 321                         print l
 322                         rebootlog.add(l)
 323         except KeyboardInterrupt:
 324                 print "Killed by interrupt"
 325                 sys.exit(0)
 326         except:
 327                 import traceback; print traceback.print_exc();
 328                 print "Continuing..."
 329
 330         time.sleep(1)
 331         if count % 10 == 0:
 332                 print "Saving rebootlog"
 333                 soltesz.dbDump("rebootlog", rebootlog)
 334                 wait_time = int(config.timewait)
 335                 print "Sleeping %d minutes" % wait_time
 336                 ti = 0
 337                 print "Minutes slept: ",
 338                 sys.stdout.flush()
 339                 while ti < wait_time:
 340                         print "%s" % ti,
 341                         sys.stdout.flush()
 342                         time.sleep(60)
 343                         ti = ti+1
 344
 345         count = count + 1
 346
 347 print "Saving rebootlog"
 348 soltesz.dbDump("rebootlog", rebootlog)