pcubad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import sets
   8 from datetime import datetime,timedelta
   9
  10 from monitor import database
  11 from monitor import reboot
  12 from monitor import parser as parsermodule
  13 from monitor import config
  14 from monitor.database.info.model import HistoryPCURecord, FindbadPCURecord
  15 from monitor.database.dborm import mon_session as session
  16 from monitor.wrapper import plc,plccache
  17 from monitor.const import MINUP
  18
  19 from monitor.common import *
  20 from nodequery import verify,query_to_dict,node_select
  21 from monitor.model import *
  22
  23 api = plc.getAuthAPI()
  24
  25 def main():
  26         main2(config)
  27
  28 def main2(config):
  29
  30         l_plcpcus = plccache.l_pcus
  31
  32         l_pcus = None
  33         if config.site is not None:
  34                 site = plccache.GetSitesByName([config.site])
  35                 l_nodes = plccache.GetNodesByIds(site[0]['node_ids'])
  36                 pcus = []
  37                 for node in l_nodes:
  38                         pcus += node['pcu_ids']
  39                 # clear out dups.
  40                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  41
  42         elif config.node:
  43                 l_nodes = plccache.GetNodeByName(config.node)
  44                 pcus = []
  45                 for node in l_nodes:
  46                         pcus += node['pcu_ids']
  47                 # clear out dups.
  48                 l_pcus = [pcu for pcu in sets.Set(pcus)]
  49
  50         elif config.pcu:
  51                 for pcu in l_plcpcus:
  52                         if ( pcu['hostname'] is not None and config.pcu in pcu['hostname'] ) or \
  53                            ( pcu['ip'] is not None and config.pcu in pcu['ip'] ):
  54                                 l_pcus = [pcu['pcu_id']]
  55                 if not l_pcus:
  56                         print "ERROR: could not find pcu %s" % config.pcu
  57                         sys.exit(1)
  58         else:
  59                 l_pcus = [pcu['pcu_id'] for pcu in l_plcpcus]
  60
  61         checkAndRecordState(l_pcus, l_plcpcus)
  62
  63 hn2lb = plccache.plcdb_hn2lb
  64
  65 def check_pcu_state(rec, pcu):
  66
  67         pcu_state = rec.reboot_trial_status
  68
  69         if ( pcu_state == 'NetDown' or pcu_state == 'Not_Run' or not ( pcu_state == 0 or pcu_state == "0" ) ) and \
  70                         ( pcu.status == 'online' or pcu.status == 'good' ):
  71                 print "changed status from %s to offline" % pcu.status
  72                 pcu.status = 'offline'
  73                 pcu.last_changed = datetime.now()
  74
  75         if ( pcu_state == 0 or pcu_state == "0" ) and pcu.status not in [ 'online', 'good' ]:
  76                 print "changed status from %s to online" % pcu.status
  77                 pcu.status = 'online'
  78                 pcu.last_changed = datetime.now()
  79
  80         if pcu.status == 'online' and changed_greaterthan(pcu.last_changed, 0.5):
  81                 #send thank you notice, or on-line notice.
  82                 print "changed status from %s to good" % pcu.status
  83                 pcu.status = 'good'
  84                 # NOTE: do not reset last_changed, or you lose how long it's been up.
  85
  86         if pcu.status == 'offline' and changed_greaterthan(pcu.last_changed, 2):
  87                 # send down pcu notice
  88                 print "changed status from %s to down" % pcu.status
  89                 pcu.status = 'down'
  90                 pcu.last_changed = datetime.now()
  91
  92         if ( pcu.status == 'offline' or pcu.status == 'down' ) and changed_greaterthan(pcu.last_changed, 2*30):
  93                 print "changed status from %s to down" % pcu.status
  94                 pcu.status = 'down'
  95                 pcu.last_changed = datetime.now()
  96
  97 def checkAndRecordState(l_pcus, l_plcpcus):
  98         count = 0
  99         for pcuname in l_pcus:
 100
 101                 d_pcu = None
 102                 for pcu in l_plcpcus:
 103                         if pcu['pcu_id'] == pcuname:
 104                                 d_pcu = pcu
 105                                 break
 106                 if not d_pcu:
 107                         continue
 108
 109                 pcuhist = HistoryPCURecord.findby_or_create(plc_pcuid=d_pcu['pcu_id'],
 110                                                                         if_new_set={'status' : 'offline',
 111                                                                                                 'last_changed' : datetime.now()})
 112                 pcuhist.last_checked = datetime.now()
 113
 114                 try:
 115                         # Find the most recent record
 116                         pcurec = FindbadPCURecord.query.filter(FindbadPCURecord.plc_pcuid==pcuname).first()
 117                 except:
 118                         print "COULD NOT FIND FB record for %s" % reboot.pcu_name(d_pcu)
 119                         import traceback
 120                         print traceback.print_exc()
 121                         # don't have the info to create a new entry right now, so continue.
 122                         continue
 123
 124                 if not pcurec:
 125                         print "none object for pcu %s"% reboot.pcu_name(d_pcu)
 126                         continue
 127
 128                 check_pcu_state(pcurec, pcuhist)
 129
 130                 count += 1
 131                 print "%d %35s %s since(%s)" % (count, reboot.pcu_name(d_pcu), pcuhist.status, diff_time(time.mktime(pcuhist.last_changed.timetuple())))
 132
 133         # NOTE: this commits all pending operations to the DB.  Do not remove, or
 134         # replace with another operations that also commits all pending ops, such
 135         # as session.commit() or flush() or something
 136         session.flush()
 137         print HistoryPCURecord.query.count()
 138
 139         return True
 140
 141 if __name__ == '__main__':
 142         parser = parsermodule.getParser()
 143         parser.set_defaults(filename=None, pcu=None, node=None, site=None, pcuselect=False, pcugroup=None, cachepcus=False)
 144         parser.add_option("", "--pcu", dest="pcu", metavar="hostname",
 145                                                 help="Provide a single pcu to operate on")
 146         parser.add_option("", "--site", dest="site", metavar="sitename",
 147                                                 help="Provide a single sitename to operate on")
 148         parser.add_option("", "--node", dest="node", metavar="nodename",
 149                                                 help="Provide a single node to operate on")
 150         parser.add_option("", "--pculist", dest="pculist", metavar="file.list",
 151                                                 help="Provide a list of files to operate on")
 152
 153         config = parsermodule.parse_args(parser)
 154
 155         try:
 156                 main2(config)
 157         except Exception, err:
 158                 import traceback
 159                 traceback.print_exc()
 160                 print "Exception: %s" % err
 161                 sys.exit(0)