commands/pcubad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import sets
   8 from datetime import datetime,timedelta
   9
  10 from monitor import database
  11 from monitor import reboot
  12 from monitor import parser as parsermodule
  13 from monitor import config
  14 from monitor.database.info.model import HistoryPCURecord, FindbadPCURecord
  15 from monitor.database.dborm import mon_session as session
  16 from monitor.wrapper import plc,plccache
  17 from monitor.const import MINUP
  18
  19 from monitor.common import *
  20 from monitor.query import verify,query_to_dict,node_select
  21 from monitor.model import *
  22
  23 api = plc.getAuthAPI()
  24
  25 def main():
  26         main2(config)
  27
  28 def main2(config):
  29
  30         l_plcpcus = plccache.l_pcus
  31
  32         l_pcus = None
  33         if config.site is not None:
  34                 site = plccache.GetSitesByName([config.site])
  35                 l_nodes = plccache.GetNodesByIds(site[0]['node_ids'])
  36                 pcus = []
  37                 for node in l_nodes:
  38                         pcus += node['pcu_ids']
  39                 # clear out dups.
  40                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  41
  42         elif config.node:
  43                 node = plccache.GetNodeByName(config.node)
  44                 pcus = node['pcu_ids']
  45                 # clear out dups.
  46                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  47
  48         elif config.pcu:
  49                 for pcu in l_plcpcus:
  50                         if ( pcu['hostname'] is not None and config.pcu in pcu['hostname'] ) or \
  51                            ( pcu['ip'] is not None and config.pcu in pcu['ip'] ):
  52                                 l_pcus = [pcu['pcu_id']]
  53                 if not l_pcus:
  54                         print "ERROR: could not find pcu %s" % config.pcu
  55                         sys.exit(1)
  56         else:
  57                 l_pcus = [pcu['pcu_id'] for pcu in l_plcpcus]
  58
  59         checkAndRecordState(l_pcus, l_plcpcus)
  60
  61 hn2lb = plccache.plcdb_hn2lb
  62
  63 def check_pcu_state(rec, pcu):
  64
  65         pcu_state = rec.reboot_trial_status
  66         print pcu_state
  67         good_list = [0, "0", "Test: No error"]
  68
  69         # DOWN
  70         if pcu_state not in good_list and pcu.status not in ['offline', 'down']:
  71                         print "changed status from %s to offline" % pcu.status
  72                         pcu.status = 'offline'
  73                         pcu.last_changed = datetime.now()
  74
  75         # ONLINE
  76         if pcu_state in good_list and pcu.status not in [ 'online', 'good' ]:
  77                 print "changed status from %s to online" % pcu.status
  78                 pcu.status = 'online'
  79                 pcu.last_changed = datetime.now()
  80
  81
  82         # STATE TRANSITIONS
  83         if pcu.status == 'online' and changed_greaterthan(pcu.last_changed, 0.5):
  84                 #send thank you notice, or on-line notice.
  85                 print "changed status from %s to good" % pcu.status
  86                 pcu.status = 'good'
  87                 # NOTE: do not reset last_changed, or you lose how long it's been up.
  88
  89         if pcu.status == 'offline' and changed_greaterthan(pcu.last_changed, 2):
  90                 # send down pcu notice
  91                 print "changed status from %s to down" % pcu.status
  92                 pcu.status = 'down'
  93
  94 #       if pcu.status in [ 'offline', 'down' ] and changed_greaterthan(pcu.last_changed, 2*30):
  95 #               print "changed status from %s to down" % pcu.status
  96 #               pcu.status = 'down'
  97 #               pcu.last_changed = datetime.now()
  98
  99 def checkAndRecordState(l_pcus, l_plcpcus):
 100         count = 0
 101         for pcuname in l_pcus:
 102
 103                 d_pcu = None
 104                 for pcu in l_plcpcus:
 105                         if pcu['pcu_id'] == pcuname:
 106                                 d_pcu = pcu
 107                                 break
 108                 if not d_pcu:
 109                         continue
 110
 111                 pcuhist = HistoryPCURecord.findby_or_create(plc_pcuid=d_pcu['pcu_id'],
 112                                                                         if_new_set={'status' : 'offline',
 113                                                                                                 'last_changed' : datetime.now()})
 114                 pcuhist.last_checked = datetime.now()
 115
 116                 try:
 117                         # Find the most recent record
 118                         pcurec = FindbadPCURecord.query.filter(FindbadPCURecord.plc_pcuid==pcuname).first()
 119                 except:
 120                         print "COULD NOT FIND FB record for %s" % reboot.pcu_name(d_pcu)
 121                         import traceback
 122                         email_exception()
 123                         print traceback.print_exc()
 124                         # don't have the info to create a new entry right now, so continue.
 125                         continue
 126
 127                 if not pcurec:
 128                         print "none object for pcu %s"% reboot.pcu_name(d_pcu)
 129                         continue
 130
 131                 check_pcu_state(pcurec, pcuhist)
 132
 133                 count += 1
 134                 print "%d %35s %s since(%s)" % (count, reboot.pcu_name(d_pcu), pcuhist.status, diff_time(time.mktime(pcuhist.last_changed.timetuple())))
 135
 136         # NOTE: this commits all pending operations to the DB.  Do not remove, or
 137         # replace with another operations that also commits all pending ops, such
 138         # as session.commit() or flush() or something
 139         session.flush()
 140         print HistoryPCURecord.query.count()
 141
 142         return True
 143
 144 if __name__ == '__main__':
 145         parser = parsermodule.getParser()
 146         parser.set_defaults(filename=None, pcu=None, node=None, site=None, pcuselect=False, pcugroup=None, cachepcus=False)
 147         parser.add_option("", "--pcu", dest="pcu", metavar="hostname",
 148                                                 help="Provide a single pcu to operate on")
 149         parser.add_option("", "--site", dest="site", metavar="sitename",
 150                                                 help="Provide a single sitename to operate on")
 151         parser.add_option("", "--node", dest="node", metavar="nodename",
 152                                                 help="Provide a single node to operate on")
 153         parser.add_option("", "--pculist", dest="pculist", metavar="file.list",
 154                                                 help="Provide a list of files to operate on")
 155
 156         config = parsermodule.parse_args(parser)
 157
 158         try:
 159                 main2(config)
 160         except Exception, err:
 161                 import traceback
 162                 traceback.print_exc()
 163                 print "Exception: %s" % err
 164                 sys.exit(0)