sitebad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 from datetime import datetime,timedelta
   8
   9 from monitor import database
  10 from monitor import parser as parsermodule
  11 from monitor import config
  12 from monitor.database.info.model import HistorySiteRecord, HistoryNodeRecord, session, BlacklistRecord
  13 from monitor.wrapper import plc, plccache
  14 from monitor.const import MINUP
  15
  16 from monitor.common import *
  17 from nodequery import verify,query_to_dict,node_select
  18 from monitor.model import *
  19
  20 api = plc.getAuthAPI()
  21 def main():
  22         main2(config)
  23
  24 def main2(config):
  25
  26         l_nodes = plccache.l_nodes
  27         l_plcsites = plccache.l_sites
  28
  29         if config.site:
  30                 l_sites = [config.site]
  31         elif config.node:
  32                 l_sites = [plccache.plcdb_hn2lb[config.node]]
  33         elif config.sitelist:
  34                 site_list = config.sitelist.split(',')
  35                 l_sites = site_list
  36         else:
  37                 l_sites = [site['login_base'] for site in l_plcsites]
  38
  39         checkAndRecordState(l_sites, l_plcsites)
  40
  41 def getnodesup(nodelist):
  42         # NOTE : assume that a blacklisted node is fine, since we're told not to
  43         #               ignore it, no policy actions should be taken for it.
  44         up = 0
  45         for node in nodelist:
  46                 try:
  47                         nodehist = HistoryNodeRecord.findby_or_create(hostname=node['hostname'])
  48                         nodebl   = BlacklistRecord.get_by(hostname=node['hostname'])
  49                         if (nodehist is not None and nodehist.status != 'down') or \
  50                                 (nodebl is not None and not nodebl.expired()):
  51                                 up = up + 1
  52                 except:
  53                         import traceback
  54                         print traceback.print_exc()
  55         return up
  56
  57 def check_site_state(rec, sitehist):
  58
  59         if sitehist.new and sitehist.status != 'new':
  60                 sitehist.status = 'new'
  61                 sitehist.last_changed = datetime.now()
  62
  63         if not sitehist.new:
  64
  65                 if sitehist.nodes_up >= MINUP:
  66
  67                         if sitehist.status != 'online' and sitehist.status != 'good':
  68                                 sitehist.last_changed = datetime.now()
  69
  70                         if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'online':
  71                                 print "changed status from %s to online" % sitehist.status
  72                                 sitehist.status = 'online'
  73
  74                         if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'good':
  75                                 print "changed status from %s to good" % sitehist.status
  76                                 sitehist.status = 'good'
  77
  78                 else: # sitehist.nodes_up < MINUP:
  79
  80                         if sitehist.status != 'offline' and sitehist.status != 'down':
  81                                 sitehist.last_changed = datetime.now()
  82
  83                         if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'offline':
  84                                 print "changed status from %s to offline" % sitehist.status
  85                                 sitehist.status = 'offline'
  86
  87                         if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'down':
  88                                 print "changed status from %s to down" % sitehist.status
  89                                 sitehist.status = 'down'
  90
  91 def checkAndRecordState(l_sites, l_plcsites):
  92         count = 0
  93         lb2hn = plccache.plcdb_lb2hn
  94         for sitename in l_sites:
  95                 d_site = None
  96                 for site in l_plcsites:
  97                         if site['login_base'] == sitename:
  98                                 d_site = site
  99                                 break
 100                 if not d_site:
 101                         continue
 102
 103                 if sitename in lb2hn:
 104                         sitehist = HistorySiteRecord.findby_or_create(loginbase=sitename,
 105                                                                                                 if_new_set={'status' : 'unknown',
 106                                                                                                                         'last_changed' : datetime.now(),
 107                                                                                                                         'message_id': 0,
 108                                                                                                                         'penalty_level' : 0})
 109                         sitehist.last_checked = datetime.now()
 110
 111                         sitehist.slices_total = d_site['max_slices']
 112                         sitehist.slices_used = len(d_site['slice_ids'])
 113                         sitehist.nodes_total = len(lb2hn[sitename])
 114                         if sitehist.message_id != 0:
 115                                 rtstatus = mailer.getTicketStatus(sitehist.message_id)
 116                                 sitehist.message_status = rtstatus['Status']
 117                                 sitehist.message_queue = rtstatus['Queue']
 118                                 sitehist.message_created = datetime.fromtimestamp(rtstatus['Created'])
 119
 120                         sitehist.nodes_up = getnodesup(lb2hn[sitename])
 121                         sitehist.new = changed_lessthan(datetime.fromtimestamp(d_site['date_created']), 30) # created < 30 days ago
 122                         sitehist.enabled = d_site['enabled']
 123
 124                         check_site_state(d_site, sitehist)
 125
 126                         count += 1
 127                         print "%d %15s slices(%2s) nodes(%2s) notdown(%2s) %s" % (count, sitename, sitehist.slices_used,
 128                                                                                         sitehist.nodes_total, sitehist.nodes_up, sitehist.status)
 129                         sitehist.flush()
 130
 131         print HistorySiteRecord.query.count()
 132         session.flush()
 133
 134         return True
 135
 136 if __name__ == '__main__':
 137         from monitor import parser as parsermodule
 138
 139         parser = parsermodule.getParser()
 140         parser.set_defaults(filename=None, node=None, site=None,
 141                                                 nodeselect=False, nodegroup=None, cachenodes=False)
 142
 143         parser.add_option("", "--site", dest="site", metavar="login_base",
 144                                                 help="Provide a single site to operate on")
 145         parser.add_option("", "--sitelist", dest="sitelist",
 146                                                 help="Provide a list of sites separated by ','")
 147
 148         config = parsermodule.parse_args(parser)
 149
 150         try:
 151                 main2(config)
 152         except Exception, err:
 153                 import traceback
 154                 print traceback.print_exc()
 155                 print "Exception: %s" % err
 156                 sys.exit(0)