sitebad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 from datetime import datetime,timedelta
   8
   9 from monitor import database
  10 from monitor import parser as parsermodule
  11 from monitor import config
  12 from monitor.database.info.model import HistorySiteRecord, HistoryNodeRecord, session, BlacklistRecord
  13 from monitor.wrapper import plc, plccache
  14 from monitor.const import MINUP
  15
  16 from monitor.common import *
  17 from nodequery import verify,query_to_dict,node_select
  18 from monitor.model import *
  19
  20 api = plc.getAuthAPI()
  21 def main():
  22         main2(config)
  23
  24 def main2(config):
  25
  26         l_nodes = plccache.l_nodes
  27         l_plcsites = plccache.l_sites
  28
  29         if config.site:
  30                 l_sites = [config.site]
  31         elif config.node:
  32                 l_sites = [plccache.plcdb_hn2lb[config.node]]
  33         elif config.sitelist:
  34                 site_list = config.sitelist.split(',')
  35                 l_sites = site_list
  36         else:
  37                 l_sites = [site['login_base'] for site in l_plcsites]
  38
  39         checkAndRecordState(l_sites, l_plcsites)
  40
  41 def getnodesup(nodelist):
  42         # NOTE : assume that a blacklisted node is fine, since we're told not to
  43         #               ignore it, no policy actions should be taken for it.
  44         up = 0
  45         for node in nodelist:
  46                 try:
  47                         nodehist = HistoryNodeRecord.findby_or_create(hostname=node['hostname'])
  48                         nodebl   = BlacklistRecord.get_by(hostname=node['hostname'])
  49                         if (nodehist is not None and nodehist.status != 'down') or \
  50                                 (nodebl is not None and not nodebl.expired()):
  51                                 up = up + 1
  52                 except:
  53                         import traceback
  54                         print traceback.print_exc()
  55         return up
  56
  57 def check_site_state(rec, sitehist):
  58
  59         if sitehist.new and sitehist.status not in ['new', 'online', 'good']:
  60                 sitehist.status = 'new'
  61                 sitehist.penalty_applied = True         # because new sites are disabled by default, i.e. have a penalty.
  62                 sitehist.last_changed = datetime.now()
  63
  64         if sitehist.nodes_up >= MINUP:
  65
  66                 if sitehist.status != 'online' and sitehist.status != 'good':
  67                         sitehist.last_changed = datetime.now()
  68
  69                 if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'online':
  70                         print "changed status from %s to online" % sitehist.status
  71                         sitehist.status = 'online'
  72
  73                 if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'good':
  74                         print "changed status from %s to good" % sitehist.status
  75                         sitehist.status = 'good'
  76
  77         elif not sitehist.new:
  78
  79                 if sitehist.status != 'offline' and sitehist.status != 'down':
  80                         sitehist.last_changed = datetime.now()
  81
  82                 if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'offline':
  83                         print "changed status from %s to offline" % sitehist.status
  84                         sitehist.status = 'offline'
  85
  86                 if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'down':
  87                         print "changed status from %s to down" % sitehist.status
  88                         sitehist.status = 'down'
  89
  90 def checkAndRecordState(l_sites, l_plcsites):
  91         count = 0
  92         lb2hn = plccache.plcdb_lb2hn
  93         for sitename in l_sites:
  94                 d_site = None
  95                 for site in l_plcsites:
  96                         if site['login_base'] == sitename:
  97                                 d_site = site
  98                                 break
  99                 if not d_site:
 100                         continue
 101
 102                 if sitename in lb2hn:
 103                         sitehist = HistorySiteRecord.findby_or_create(loginbase=sitename,
 104                                                                                                 if_new_set={'status' : 'unknown',
 105                                                                                                                         'last_changed' : datetime.now(),
 106                                                                                                                         'message_id': 0,
 107                                                                                                                         'penalty_level' : 0})
 108                         sitehist.last_checked = datetime.now()
 109
 110                         sitehist.slices_total = d_site['max_slices']
 111                         sitehist.slices_used = len(d_site['slice_ids'])
 112                         sitehist.nodes_total = len(lb2hn[sitename])
 113                         if sitehist.message_id != 0:
 114                                 rtstatus = mailer.getTicketStatus(sitehist.message_id)
 115                                 sitehist.message_status = rtstatus['Status']
 116                                 sitehist.message_queue = rtstatus['Queue']
 117                                 sitehist.message_created = datetime.fromtimestamp(rtstatus['Created'])
 118
 119                         sitehist.nodes_up = getnodesup(lb2hn[sitename])
 120                         sitehist.new = changed_lessthan(datetime.fromtimestamp(d_site['date_created']), 30) # created < 30 days ago
 121                         sitehist.enabled = d_site['enabled']
 122
 123                         check_site_state(d_site, sitehist)
 124
 125                         count += 1
 126                         print "%d %15s slices(%2s) nodes(%2s) notdown(%2s) %s" % (count, sitename, sitehist.slices_used,
 127                                                                                         sitehist.nodes_total, sitehist.nodes_up, sitehist.status)
 128                         sitehist.flush()
 129
 130         print HistorySiteRecord.query.count()
 131         session.flush()
 132
 133         return True
 134
 135 if __name__ == '__main__':
 136         from monitor import parser as parsermodule
 137
 138         parser = parsermodule.getParser()
 139         parser.set_defaults(filename=None, node=None, site=None,
 140                                                 nodeselect=False, nodegroup=None, cachenodes=False)
 141
 142         parser.add_option("", "--site", dest="site", metavar="login_base",
 143                                                 help="Provide a single site to operate on")
 144         parser.add_option("", "--sitelist", dest="sitelist",
 145                                                 help="Provide a list of sites separated by ','")
 146
 147         config = parsermodule.parse_args(parser)
 148
 149         try:
 150                 main2(config)
 151         except Exception, err:
 152                 import traceback
 153                 print traceback.print_exc()
 154                 print "Exception: %s" % err
 155                 sys.exit(0)