sitebad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 from datetime import datetime,timedelta
   8
   9 from monitor import database
  10 from monitor import parser as parsermodule
  11 from monitor import config
  12 from monitor.database.info.model import HistorySiteRecord, HistoryNodeRecord, session
  13 from monitor.wrapper import plc, plccache
  14 from monitor.const import MINUP
  15
  16 from monitor.common import *
  17 from nodequery import verify,query_to_dict,node_select
  18 from monitor.model import *
  19
  20 api = plc.getAuthAPI()
  21 def main():
  22         main2(config)
  23
  24 def main2(config):
  25
  26         l_nodes = plccache.l_nodes
  27         l_plcsites = plccache.l_sites
  28
  29         if config.site:
  30                 l_sites = [config.site]
  31         elif config.sitelist:
  32                 site_list = config.sitelist.split(',')
  33                 l_sites = site_list
  34         else:
  35                 l_sites = [site['login_base'] for site in l_plcsites]
  36
  37         checkAndRecordState(l_sites, l_plcsites)
  38
  39 def getnodesup(nodelist):
  40         up = 0
  41         for node in nodelist:
  42                 try:
  43                         nodehist = HistoryNodeRecord.findby_or_create(hostname=node['hostname'])
  44                         if nodehist is not None and nodehist.status == "good":
  45                                 up = up + 1
  46                 except:
  47                         import traceback
  48                         print traceback.print_exc()
  49         return up
  50
  51 def check_site_state(rec, sitehist):
  52
  53         if sitehist.new and sitehist.status != 'new':
  54                 sitehist.status = 'new'
  55                 sitehist.last_changed = datetime.now()
  56
  57         if not sitehist.new:
  58
  59                 if sitehist.nodes_up >= MINUP:
  60
  61                         if sitehist.status != 'online' and sitehist.status != 'good':
  62                                 sitehist.last_changed = datetime.now()
  63
  64                         if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'online':
  65                                 print "changed status from %s to online" % sitehist.status
  66                                 sitehist.status = 'online'
  67
  68                         if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'good':
  69                                 print "changed status from %s to good" % sitehist.status
  70                                 sitehist.status = 'good'
  71
  72                 else: # sitehist.nodes_up < MINUP:
  73
  74                         if sitehist.status != 'offline' and sitehist.status != 'down':
  75                                 sitehist.last_changed = datetime.now()
  76
  77                         if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'offline':
  78                                 print "changed status from %s to offline" % sitehist.status
  79                                 sitehist.status = 'offline'
  80
  81                         if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'down':
  82                                 print "changed status from %s to down" % sitehist.status
  83                                 sitehist.status = 'down'
  84
  85 def checkAndRecordState(l_sites, l_plcsites):
  86         count = 0
  87         lb2hn = plccache.plcdb_lb2hn
  88         for sitename in l_sites:
  89                 d_site = None
  90                 for site in l_plcsites:
  91                         if site['login_base'] == sitename:
  92                                 d_site = site
  93                                 break
  94                 if not d_site:
  95                         continue
  96
  97                 if sitename in lb2hn:
  98                         sitehist = HistorySiteRecord.findby_or_create(loginbase=sitename,
  99                                                                                                 if_new_set={'status' : 'unknown',
 100                                                                                                                         'last_changed' : datetime.now(),
 101                                                                                                                         'message_id': 0,
 102                                                                                                                         'penalty_level' : 0})
 103                         sitehist.last_checked = datetime.now()
 104
 105                         sitehist.slices_total = d_site['max_slices']
 106                         sitehist.slices_used = len(d_site['slice_ids'])
 107                         sitehist.nodes_total = len(lb2hn[sitename])
 108                         if sitehist.message_id != 0:
 109                                 rtstatus = mailer.getTicketStatus(sitehist.message_id)
 110                                 sitehist.message_status = rtstatus['Status']
 111                                 sitehist.message_queue = rtstatus['Queue']
 112                                 sitehist.message_created = datetime.fromtimestamp(rtstatus['Created'])
 113
 114                         sitehist.nodes_up = getnodesup(lb2hn[sitename])
 115                         sitehist.new = changed_lessthan(datetime.fromtimestamp(d_site['date_created']), 30) # created < 30 days ago
 116                         sitehist.enabled = d_site['enabled']
 117
 118                         check_site_state(d_site, sitehist)
 119
 120                         count += 1
 121                         print "%d %15s slices(%2s) nodes(%2s) up(%2s) %s" % (count, sitename, sitehist.slices_used,
 122                                                                                         sitehist.nodes_total, sitehist.nodes_up, sitehist.status)
 123                         sitehist.flush()
 124
 125         print HistorySiteRecord.query.count()
 126         session.flush()
 127
 128         return True
 129
 130 if __name__ == '__main__':
 131         from monitor import parser as parsermodule
 132
 133         parser = parsermodule.getParser()
 134         parser.set_defaults(filename=None, node=None, site=None,
 135                                                 nodeselect=False, nodegroup=None, cachenodes=False)
 136
 137         parser.add_option("", "--site", dest="site", metavar="login_base",
 138                                                 help="Provide a single site to operate on")
 139         parser.add_option("", "--sitelist", dest="sitelist",
 140                                                 help="Provide a list of sites separated by ','")
 141
 142         config = parsermodule.parse_args(parser)
 143
 144         try:
 145                 main2(config)
 146         except Exception, err:
 147                 import traceback
 148                 print traceback.print_exc()
 149                 print "Exception: %s" % err
 150                 sys.exit(0)