pcubad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import sets
   8 from datetime import datetime,timedelta
   9
  10 from monitor import database
  11 from monitor import reboot
  12 from monitor import parser as parsermodule
  13 from monitor import config
  14 from monitor.database.info.model import HistoryPCURecord, FindbadPCURecord
  15 from monitor.database.dborm import mon_session as session
  16 from monitor.wrapper import plc,plccache
  17 from monitor.const import MINUP
  18
  19 from monitor.common import *
  20 from nodequery import verify,query_to_dict,node_select
  21 from monitor.model import *
  22
  23 api = plc.getAuthAPI()
  24
  25 def main():
  26         main2(config)
  27
  28 def main2(config):
  29
  30         l_plcpcus = plccache.l_pcus
  31
  32         l_pcus = None
  33         if config.site is not None:
  34                 site = plccache.GetSitesByName([config.site])
  35                 l_nodes = plccache.GetNodesByIds(site[0]['node_ids'])
  36                 pcus = []
  37                 for node in l_nodes:
  38                         pcus += node['pcu_ids']
  39                 # clear out dups.
  40                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  41
  42         elif config.node:
  43                 node = plccache.GetNodeByName(config.node)
  44                 pcus = node['pcu_ids']
  45                 # clear out dups.
  46                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  47
  48         elif config.pcu:
  49                 for pcu in l_plcpcus:
  50                         if ( pcu['hostname'] is not None and config.pcu in pcu['hostname'] ) or \
  51                            ( pcu['ip'] is not None and config.pcu in pcu['ip'] ):
  52                                 l_pcus = [pcu['pcu_id']]
  53                 if not l_pcus:
  54                         print "ERROR: could not find pcu %s" % config.pcu
  55                         sys.exit(1)
  56         else:
  57                 l_pcus = [pcu['pcu_id'] for pcu in l_plcpcus]
  58
  59         checkAndRecordState(l_pcus, l_plcpcus)
  60
  61 hn2lb = plccache.plcdb_hn2lb
  62
  63 def check_pcu_state(rec, pcu):
  64
  65         pcu_state = rec.reboot_trial_status
  66
  67         if ( pcu_state == 'NetDown' or pcu_state == 'Not_Run' or not ( pcu_state == 0 or pcu_state == "0" ) ) and \
  68                         ( pcu.status == 'online' or pcu.status == 'good' ):
  69                 print "changed status from %s to offline" % pcu.status
  70                 pcu.status = 'offline'
  71                 pcu.last_changed = datetime.now()
  72
  73         if ( pcu_state == 0 or pcu_state == "0" ) and pcu.status not in [ 'online', 'good' ]:
  74                 print "changed status from %s to online" % pcu.status
  75                 pcu.status = 'online'
  76                 pcu.last_changed = datetime.now()
  77
  78         if pcu.status == 'online' and changed_greaterthan(pcu.last_changed, 0.5):
  79                 #send thank you notice, or on-line notice.
  80                 print "changed status from %s to good" % pcu.status
  81                 pcu.status = 'good'
  82                 # NOTE: do not reset last_changed, or you lose how long it's been up.
  83
  84         if pcu.status == 'offline' and changed_greaterthan(pcu.last_changed, 2):
  85                 # send down pcu notice
  86                 print "changed status from %s to down" % pcu.status
  87                 pcu.status = 'down'
  88                 pcu.last_changed = datetime.now()
  89
  90         if ( pcu.status == 'offline' or pcu.status == 'down' ) and changed_greaterthan(pcu.last_changed, 2*30):
  91                 print "changed status from %s to down" % pcu.status
  92                 pcu.status = 'down'
  93                 pcu.last_changed = datetime.now()
  94
  95 def checkAndRecordState(l_pcus, l_plcpcus):
  96         count = 0
  97         for pcuname in l_pcus:
  98
  99                 d_pcu = None
 100                 for pcu in l_plcpcus:
 101                         if pcu['pcu_id'] == pcuname:
 102                                 d_pcu = pcu
 103                                 break
 104                 if not d_pcu:
 105                         continue
 106
 107                 pcuhist = HistoryPCURecord.findby_or_create(plc_pcuid=d_pcu['pcu_id'],
 108                                                                         if_new_set={'status' : 'offline',
 109                                                                                                 'last_changed' : datetime.now()})
 110                 pcuhist.last_checked = datetime.now()
 111
 112                 try:
 113                         # Find the most recent record
 114                         pcurec = FindbadPCURecord.query.filter(FindbadPCURecord.plc_pcuid==pcuname).first()
 115                 except:
 116                         print "COULD NOT FIND FB record for %s" % reboot.pcu_name(d_pcu)
 117                         import traceback
 118                         email_exception()
 119                         print traceback.print_exc()
 120                         # don't have the info to create a new entry right now, so continue.
 121                         continue
 122
 123                 if not pcurec:
 124                         print "none object for pcu %s"% reboot.pcu_name(d_pcu)
 125                         continue
 126
 127                 check_pcu_state(pcurec, pcuhist)
 128
 129                 count += 1
 130                 print "%d %35s %s since(%s)" % (count, reboot.pcu_name(d_pcu), pcuhist.status, diff_time(time.mktime(pcuhist.last_changed.timetuple())))
 131
 132         # NOTE: this commits all pending operations to the DB.  Do not remove, or
 133         # replace with another operations that also commits all pending ops, such
 134         # as session.commit() or flush() or something
 135         session.flush()
 136         print HistoryPCURecord.query.count()
 137
 138         return True
 139
 140 if __name__ == '__main__':
 141         parser = parsermodule.getParser()
 142         parser.set_defaults(filename=None, pcu=None, node=None, site=None, pcuselect=False, pcugroup=None, cachepcus=False)
 143         parser.add_option("", "--pcu", dest="pcu", metavar="hostname",
 144                                                 help="Provide a single pcu to operate on")
 145         parser.add_option("", "--site", dest="site", metavar="sitename",
 146                                                 help="Provide a single sitename to operate on")
 147         parser.add_option("", "--node", dest="node", metavar="nodename",
 148                                                 help="Provide a single node to operate on")
 149         parser.add_option("", "--pculist", dest="pculist", metavar="file.list",
 150                                                 help="Provide a list of files to operate on")
 151
 152         config = parsermodule.parse_args(parser)
 153
 154         try:
 155                 main2(config)
 156         except Exception, err:
 157                 import traceback
 158                 traceback.print_exc()
 159                 print "Exception: %s" % err
 160                 sys.exit(0)