pcubad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import sets
   8 from datetime import datetime,timedelta
   9
  10 from monitor import database
  11 from monitor import reboot
  12 from monitor import parser as parsermodule
  13 from monitor import config
  14 from monitor.database.info.model import HistoryPCURecord, FindbadPCURecord
  15 from monitor.database.dborm import mon_session as session
  16 from monitor.wrapper import plc,plccache
  17 from monitor.const import MINUP
  18
  19 from monitor.common import *
  20 from nodequery import verify,query_to_dict,node_select
  21 from monitor.model import *
  22
  23 api = plc.getAuthAPI()
  24
  25 def main():
  26         main2(config)
  27
  28 def main2(config):
  29
  30         l_plcpcus = plccache.l_pcus
  31
  32         l_pcus = None
  33         if config.site is not None:
  34                 site = api.GetSites(config.site)
  35                 l_nodes = api.GetNodes(site[0]['node_ids'], ['pcu_ids'])
  36                 pcus = []
  37                 for node in l_nodes:
  38                         pcus += node['pcu_ids']
  39                 # clear out dups.
  40                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  41         elif config.pcu:
  42                 for pcu in l_plcpcus:
  43                         if ( pcu['hostname'] is not None and config.pcu in pcu['hostname'] ) or \
  44                            ( pcu['ip'] is not None and config.pcu in pcu['ip'] ):
  45                                 l_pcus = [pcu['pcu_id']]
  46                 if not l_pcus:
  47                         print "ERROR: could not find pcu %s" % config.pcu
  48                         sys.exit(1)
  49         else:
  50                 l_pcus = [pcu['pcu_id'] for pcu in l_plcpcus]
  51
  52         checkAndRecordState(l_pcus, l_plcpcus)
  53
  54 hn2lb = plccache.plcdb_hn2lb
  55
  56 def check_pcu_state(rec, pcu):
  57
  58         pcu_state = rec.reboot_trial_status
  59
  60         if ( pcu_state == 'NetDown' or pcu_state == 'Not_Run' or not ( pcu_state == 0 or pcu_state == "0" ) ) and \
  61                         ( pcu.status == 'online' or pcu.status == 'good' ):
  62                 print "changed status from %s to offline" % pcu.status
  63                 pcu.status = 'offline'
  64                 pcu.last_changed = datetime.now()
  65
  66         if ( pcu_state == 0 or pcu_state == "0" ) and changed_lessthan(pcu.last_changed, 0.5) and pcu.status != 'online':
  67                 print "changed status from %s to online" % pcu.status
  68                 pcu.status = 'online'
  69                 pcu.last_changed = datetime.now()
  70
  71         if pcu.status == 'online' and changed_greaterthan(pcu.last_changed, 0.5):
  72                 #send thank you notice, or on-line notice.
  73                 print "changed status from %s to good" % pcu.status
  74                 pcu.status = 'good'
  75                 # NOTE: do not reset last_changed, or you lose how long it's been up.
  76
  77         if pcu.status == 'offline' and changed_greaterthan(pcu.last_changed, 2):
  78                 # send down pcu notice
  79                 print "changed status from %s to down" % pcu.status
  80                 pcu.status = 'down'
  81                 pcu.last_changed = datetime.now()
  82
  83         if ( pcu.status == 'offline' or pcu.status == 'down' ) and changed_greaterthan(pcu.last_changed, 2*30):
  84                 print "changed status from %s to down" % pcu.status
  85                 pcu.status = 'down'
  86                 pcu.last_changed = datetime.now()
  87
  88 def checkAndRecordState(l_pcus, l_plcpcus):
  89         count = 0
  90         for pcuname in l_pcus:
  91
  92                 d_pcu = None
  93                 for pcu in l_plcpcus:
  94                         if pcu['pcu_id'] == pcuname:
  95                                 d_pcu = pcu
  96                                 break
  97                 if not d_pcu:
  98                         continue
  99
 100                 pcuhist = HistoryPCURecord.findby_or_create(plc_pcuid=d_pcu['pcu_id'],
 101                                                                         if_new_set={'status' : 'offline',
 102                                                                                                 'last_changed' : datetime.now()})
 103                 pcuhist.last_checked = datetime.now()
 104
 105                 try:
 106                         # Find the most recent record
 107                         pcurec = FindbadPCURecord.query.filter(FindbadPCURecord.plc_pcuid==pcuname).first()
 108                 except:
 109                         print "COULD NOT FIND FB record for %s" % reboot.pcu_name(d_pcu)
 110                         import traceback
 111                         print traceback.print_exc()
 112                         # don't have the info to create a new entry right now, so continue.
 113                         continue
 114
 115                 if not pcurec:
 116                         print "none object for pcu %s"% reboot.pcu_name(d_pcu)
 117                         continue
 118
 119                 check_pcu_state(pcurec, pcuhist)
 120
 121                 count += 1
 122                 print "%d %35s %s since(%s)" % (count, reboot.pcu_name(d_pcu), pcuhist.status, diff_time(time.mktime(pcuhist.last_changed.timetuple())))
 123
 124         # NOTE: this commits all pending operations to the DB.  Do not remove, or
 125         # replace with another operations that also commits all pending ops, such
 126         # as session.commit() or flush() or something
 127         session.flush()
 128         print HistoryPCURecord.query.count()
 129
 130         return True
 131
 132 if __name__ == '__main__':
 133         parser = parsermodule.getParser()
 134         parser.set_defaults(filename=None, pcu=None, site=None, pcuselect=False, pcugroup=None, cachepcus=False)
 135         parser.add_option("", "--pcu", dest="pcu", metavar="hostname",
 136                                                 help="Provide a single pcu to operate on")
 137         parser.add_option("", "--site", dest="site", metavar="sitename",
 138                                                 help="Provide a single sitename to operate on")
 139         parser.add_option("", "--pculist", dest="pculist", metavar="file.list",
 140                                                 help="Provide a list of files to operate on")
 141
 142         config = parsermodule.parse_args(parser)
 143
 144         try:
 145                 main2(config)
 146         except Exception, err:
 147                 import traceback
 148                 print traceback.print_exc()
 149                 print "Exception: %s" % err
 150                 sys.exit(0)