sitebad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 from datetime import datetime,timedelta
   8
   9 from monitor import database
  10 from monitor import parser as parsermodule
  11 from monitor import config
  12 from monitor.database.info.model import HistorySiteRecord, HistoryNodeRecord, session, BlacklistRecord
  13 from monitor.wrapper import plc, plccache
  14 from monitor.const import MINUP
  15
  16 from monitor.common import *
  17 from nodequery import verify,query_to_dict,node_select
  18 from monitor.model import *
  19
  20 api = plc.getAuthAPI()
  21 def main():
  22         main2(config)
  23
  24 def main2(config):
  25
  26         l_nodes = plccache.l_nodes
  27         l_plcsites = plccache.l_sites
  28
  29         if config.site:
  30                 l_sites = [config.site]
  31         elif config.node:
  32                 l_sites = [plccache.plcdb_hn2lb[config.node]]
  33         elif config.sitelist:
  34                 site_list = config.sitelist.split(',')
  35                 l_sites = site_list
  36         else:
  37                 l_sites = [site['login_base'] for site in l_plcsites]
  38
  39         checkAndRecordState(l_sites, l_plcsites)
  40
  41 def getnodesup(nodelist):
  42         # NOTE : assume that a blacklisted node is fine, since we're told not to
  43         #               ignore it, no policy actions should be taken for it.
  44         up = 0
  45         for node in nodelist:
  46                 try:
  47                         # NOTE: adding a condition for nodehist.haspcu would include pcus
  48                         #               in the calculation
  49                         nodehist = HistoryNodeRecord.findby_or_create(hostname=node['hostname'])
  50                         nodebl   = BlacklistRecord.get_by(hostname=node['hostname'])
  51                         if (nodehist is not None and nodehist.status != 'down') or \
  52                                 (nodebl is not None and not nodebl.expired()):
  53                                 up = up + 1
  54                 except:
  55                         import traceback
  56                         email_exception(node['hostname'])
  57                         print traceback.print_exc()
  58         return up
  59
  60 def check_site_state(rec, sitehist):
  61
  62         if sitehist.new and sitehist.status not in ['new', 'online', 'good']:
  63                 sitehist.status = 'new'
  64                 sitehist.penalty_applied = True         # because new sites are disabled by default, i.e. have a penalty.
  65                 sitehist.last_changed = datetime.now()
  66
  67         if sitehist.nodes_up >= MINUP:
  68
  69                 if sitehist.status != 'online' and sitehist.status != 'good':
  70                         sitehist.last_changed = datetime.now()
  71
  72                 if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'online':
  73                         print "changed status from %s to online" % sitehist.status
  74                         sitehist.status = 'online'
  75
  76                 if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'good':
  77                         print "changed status from %s to good" % sitehist.status
  78                         sitehist.status = 'good'
  79
  80         elif not sitehist.new:
  81
  82                 if sitehist.status != 'offline' and sitehist.status != 'down':
  83                         sitehist.last_changed = datetime.now()
  84
  85                 if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'offline':
  86                         print "changed status from %s to offline" % sitehist.status
  87                         sitehist.status = 'offline'
  88
  89                 if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'down':
  90                         print "changed status from %s to down" % sitehist.status
  91                         sitehist.status = 'down'
  92
  93 def checkAndRecordState(l_sites, l_plcsites):
  94         count = 0
  95         lb2hn = plccache.plcdb_lb2hn
  96         for sitename in l_sites:
  97                 d_site = None
  98                 for site in l_plcsites:
  99                         if site['login_base'] == sitename:
 100                                 d_site = site
 101                                 break
 102                 if not d_site:
 103                         continue
 104
 105                 if sitename in lb2hn:
 106                         sitehist = HistorySiteRecord.findby_or_create(loginbase=sitename,
 107                                                                                                 if_new_set={'status' : 'unknown',
 108                                                                                                                         'last_changed' : datetime.now(),
 109                                                                                                                         'message_id': 0,
 110                                                                                                                         'penalty_level' : 0})
 111                         sitehist.last_checked = datetime.now()
 112
 113                         sitehist.plc_siteid = d_site['site_id']
 114                         sitehist.slices_total = d_site['max_slices']
 115                         sitehist.slices_used = len(d_site['slice_ids'])
 116                         sitehist.nodes_total = len(lb2hn[sitename])
 117                         if sitehist.message_id != 0:
 118                                 rtstatus = mailer.getTicketStatus(sitehist.message_id)
 119                                 sitehist.message_status = rtstatus['Status']
 120                                 sitehist.message_queue = rtstatus['Queue']
 121                                 sitehist.message_created = datetime.fromtimestamp(rtstatus['Created'])
 122
 123                         sitehist.nodes_up = getnodesup(lb2hn[sitename])
 124                         sitehist.new = changed_lessthan(datetime.fromtimestamp(d_site['date_created']), 30) # created < 30 days ago
 125                         sitehist.enabled = d_site['enabled']
 126
 127                         check_site_state(d_site, sitehist)
 128
 129                         count += 1
 130                         print "%d %15s slices(%2s) nodes(%2s) notdown(%2s) %s" % (count, sitename, sitehist.slices_used,
 131                                                                                         sitehist.nodes_total, sitehist.nodes_up, sitehist.status)
 132                         sitehist.flush()
 133
 134         print HistorySiteRecord.query.count()
 135         session.flush()
 136
 137         return True
 138
 139 if __name__ == '__main__':
 140         from monitor import parser as parsermodule
 141
 142         parser = parsermodule.getParser()
 143         parser.set_defaults(filename=None, node=None, site=None,
 144                                                 nodeselect=False, nodegroup=None, cachenodes=False)
 145
 146         parser.add_option("", "--site", dest="site", metavar="login_base",
 147                                                 help="Provide a single site to operate on")
 148         parser.add_option("", "--sitelist", dest="sitelist",
 149                                                 help="Provide a list of sites separated by ','")
 150
 151         config = parsermodule.parse_args(parser)
 152
 153         try:
 154                 main2(config)
 155         except Exception, err:
 156                 import traceback
 157                 print traceback.print_exc()
 158                 print "Exception: %s" % err
 159                 sys.exit(0)