commands/pcubad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import sets
   8 from datetime import datetime,timedelta
   9
  10 from monitor import database
  11 from monitor import reboot
  12 from monitor import parser as parsermodule
  13 from monitor import config
  14 from monitor.database.info.model import HistoryPCURecord, FindbadPCURecord
  15 from monitor.database.dborm import mon_session as session
  16 from monitor.wrapper import plc,plccache
  17 from monitor.const import MINUP
  18
  19 from monitor.common import *
  20 from monitor.query import verify,query_to_dict,node_select
  21 from monitor.model import *
  22
  23 api = plc.getAuthAPI()
  24
  25 def main():
  26         main2(config)
  27
  28 def main2(config):
  29
  30         l_plcpcus = plccache.l_pcus
  31
  32         l_pcus = None
  33         if config.site is not None:
  34                 site = plccache.GetSitesByName([config.site])
  35                 l_nodes = plccache.GetNodesByIds(site[0]['node_ids'])
  36                 pcus = []
  37                 for node in l_nodes:
  38                         pcus += node['pcu_ids']
  39                 # clear out dups.
  40                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  41
  42         elif config.node:
  43                 node = plccache.GetNodeByName(config.node)
  44                 pcus = node['pcu_ids']
  45                 # clear out dups.
  46                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  47
  48         elif config.pcu:
  49                 for pcu in l_plcpcus:
  50                         if ( pcu['hostname'] is not None and config.pcu in pcu['hostname'] ) or \
  51                            ( pcu['ip'] is not None and config.pcu in pcu['ip'] ):
  52                                 l_pcus = [pcu['pcu_id']]
  53                 if not l_pcus:
  54                         print "ERROR: could not find pcu %s" % config.pcu
  55                         sys.exit(1)
  56         else:
  57                 l_pcus = [pcu['pcu_id'] for pcu in l_plcpcus]
  58
  59         checkAndRecordState(l_pcus, l_plcpcus)
  60
  61 hn2lb = plccache.plcdb_hn2lb
  62
  63 def check_pcu_state(rec, pcu):
  64
  65         pcu_state = rec.reboot_trial_status
  66         good_list = [0, "0", "Test: No error"]
  67
  68         # DOWN
  69         if pcu_state not in good_list and pcu.status not in ['offline', 'down']:
  70                         print "changed status from %s to offline" % pcu.status
  71                         pcu.status = 'offline'
  72                         pcu.last_changed = datetime.now()
  73
  74         # ONLINE
  75         if pcu_state in good_list and pcu.status not in [ 'online', 'good' ]:
  76                 print "changed status from %s to online" % pcu.status
  77                 pcu.status = 'online'
  78                 pcu.last_changed = datetime.now()
  79
  80
  81         # STATE TRANSITIONS
  82         if pcu.status == 'online' and changed_greaterthan(pcu.last_changed, 0.5):
  83                 #send thank you notice, or on-line notice.
  84                 print "changed status from %s to good" % pcu.status
  85                 pcu.status = 'good'
  86                 # NOTE: do not reset last_changed, or you lose how long it's been up.
  87
  88         if pcu.status == 'offline' and changed_greaterthan(pcu.last_changed, 2):
  89                 # send down pcu notice
  90                 print "changed status from %s to down" % pcu.status
  91                 pcu.status = 'down'
  92
  93 #       if pcu.status in [ 'offline', 'down' ] and changed_greaterthan(pcu.last_changed, 2*30):
  94 #               print "changed status from %s to down" % pcu.status
  95 #               pcu.status = 'down'
  96 #               pcu.last_changed = datetime.now()
  97
  98 def checkAndRecordState(l_pcus, l_plcpcus):
  99         count = 0
 100         for pcuname in l_pcus:
 101
 102                 d_pcu = None
 103                 for pcu in l_plcpcus:
 104                         if pcu['pcu_id'] == pcuname:
 105                                 d_pcu = pcu
 106                                 break
 107                 if not d_pcu:
 108                         continue
 109
 110                 pcuhist = HistoryPCURecord.findby_or_create(plc_pcuid=d_pcu['pcu_id'],
 111                                                                         if_new_set={'status' : 'offline',
 112                                                                                                 'last_changed' : datetime.now()})
 113                 pcuhist.last_checked = datetime.now()
 114
 115                 try:
 116                         # Find the most recent record
 117                         pcurec = FindbadPCURecord.query.filter(FindbadPCURecord.plc_pcuid==pcuname).first()
 118                 except:
 119                         print "COULD NOT FIND FB record for %s" % reboot.pcu_name(d_pcu)
 120                         import traceback
 121                         email_exception()
 122                         print traceback.print_exc()
 123                         # don't have the info to create a new entry right now, so continue.
 124                         continue
 125
 126                 if not pcurec:
 127                         print "none object for pcu %s"% reboot.pcu_name(d_pcu)
 128                         continue
 129
 130                 check_pcu_state(pcurec, pcuhist)
 131
 132                 count += 1
 133                 print "%d %35s %s since(%s)" % (count, reboot.pcu_name(d_pcu), pcuhist.status, diff_time(time.mktime(pcuhist.last_changed.timetuple())))
 134
 135         # NOTE: this commits all pending operations to the DB.  Do not remove, or
 136         # replace with another operations that also commits all pending ops, such
 137         # as session.commit() or flush() or something
 138         session.flush()
 139         print HistoryPCURecord.query.count()
 140
 141         return True
 142
 143 if __name__ == '__main__':
 144         parser = parsermodule.getParser()
 145         parser.set_defaults(filename=None, pcu=None, node=None, site=None, pcuselect=False, pcugroup=None, cachepcus=False)
 146         parser.add_option("", "--pcu", dest="pcu", metavar="hostname",
 147                                                 help="Provide a single pcu to operate on")
 148         parser.add_option("", "--site", dest="site", metavar="sitename",
 149                                                 help="Provide a single sitename to operate on")
 150         parser.add_option("", "--node", dest="node", metavar="nodename",
 151                                                 help="Provide a single node to operate on")
 152         parser.add_option("", "--pculist", dest="pculist", metavar="file.list",
 153                                                 help="Provide a list of files to operate on")
 154
 155         config = parsermodule.parse_args(parser)
 156
 157         try:
 158                 main2(config)
 159         except Exception, err:
 160                 import traceback
 161                 traceback.print_exc()
 162                 print "Exception: %s" % err
 163                 sys.exit(0)