sitebad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 from datetime import datetime,timedelta
   8
   9 from monitor import database
  10 from monitor import parser as parsermodule
  11 from monitor import config
  12 from monitor.database.info.model import HistorySiteRecord, HistoryNodeRecord, session
  13 from monitor.wrapper import plc, plccache
  14 from monitor.const import MINUP
  15
  16 from monitor.common import *
  17 from nodequery import verify,query_to_dict,node_select
  18 from monitor.model import *
  19
  20 api = plc.getAuthAPI()
  21 def main():
  22         main2(config)
  23
  24 def main2(config):
  25
  26         l_nodes = plccache.l_nodes
  27         l_plcsites = plccache.l_sites
  28
  29         if config.site:
  30                 l_sites = [config.site]
  31         elif config.sitelist:
  32                 site_list = config.sitelist.split(',')
  33                 l_sites = site_list
  34         else:
  35                 l_sites = [site['login_base'] for site in l_plcsites]
  36
  37         checkAndRecordState(l_sites, l_plcsites)
  38
  39 def getnodesup(nodelist):
  40         # NOTE : assume that a blacklisted node is fine, since we're told not to
  41         #               ignore it, no policy actions should be taken for it.
  42         up = 0
  43         for node in nodelist:
  44                 try:
  45                         nodehist = HistoryNodeRecord.findby_or_create(hostname=node['hostname'])
  46                         nodebl   = BlacklistRecord.get_by(hostname=node['hostname'])
  47                         if (nodehist is not None and nodehist.status != 'down') or \
  48                                 (nodebl is not None and not nodebl.expired():
  49                                 up = up + 1
  50                 except:
  51                         import traceback
  52                         print traceback.print_exc()
  53         return up
  54
  55 def check_site_state(rec, sitehist):
  56
  57         if sitehist.new and sitehist.status != 'new':
  58                 sitehist.status = 'new'
  59                 sitehist.last_changed = datetime.now()
  60
  61         if not sitehist.new:
  62
  63                 if sitehist.nodes_up >= MINUP:
  64
  65                         if sitehist.status != 'online' and sitehist.status != 'good':
  66                                 sitehist.last_changed = datetime.now()
  67
  68                         if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'online':
  69                                 print "changed status from %s to online" % sitehist.status
  70                                 sitehist.status = 'online'
  71
  72                         if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'good':
  73                                 print "changed status from %s to good" % sitehist.status
  74                                 sitehist.status = 'good'
  75
  76                 else: # sitehist.nodes_up < MINUP:
  77
  78                         if sitehist.status != 'offline' and sitehist.status != 'down':
  79                                 sitehist.last_changed = datetime.now()
  80
  81                         if changed_lessthan(sitehist.last_changed, 0.5) and sitehist.status != 'offline':
  82                                 print "changed status from %s to offline" % sitehist.status
  83                                 sitehist.status = 'offline'
  84
  85                         if changed_greaterthan(sitehist.last_changed, 0.5) and sitehist.status != 'down':
  86                                 print "changed status from %s to down" % sitehist.status
  87                                 sitehist.status = 'down'
  88
  89 def checkAndRecordState(l_sites, l_plcsites):
  90         count = 0
  91         lb2hn = plccache.plcdb_lb2hn
  92         for sitename in l_sites:
  93                 d_site = None
  94                 for site in l_plcsites:
  95                         if site['login_base'] == sitename:
  96                                 d_site = site
  97                                 break
  98                 if not d_site:
  99                         continue
 100
 101                 if sitename in lb2hn:
 102                         sitehist = HistorySiteRecord.findby_or_create(loginbase=sitename,
 103                                                                                                 if_new_set={'status' : 'unknown',
 104                                                                                                                         'last_changed' : datetime.now(),
 105                                                                                                                         'message_id': 0,
 106                                                                                                                         'penalty_level' : 0})
 107                         sitehist.last_checked = datetime.now()
 108
 109                         sitehist.slices_total = d_site['max_slices']
 110                         sitehist.slices_used = len(d_site['slice_ids'])
 111                         sitehist.nodes_total = len(lb2hn[sitename])
 112                         if sitehist.message_id != 0:
 113                                 rtstatus = mailer.getTicketStatus(sitehist.message_id)
 114                                 sitehist.message_status = rtstatus['Status']
 115                                 sitehist.message_queue = rtstatus['Queue']
 116                                 sitehist.message_created = datetime.fromtimestamp(rtstatus['Created'])
 117
 118                         sitehist.nodes_up = getnodesup(lb2hn[sitename])
 119                         sitehist.new = changed_lessthan(datetime.fromtimestamp(d_site['date_created']), 30) # created < 30 days ago
 120                         sitehist.enabled = d_site['enabled']
 121
 122                         check_site_state(d_site, sitehist)
 123
 124                         count += 1
 125                         print "%d %15s slices(%2s) nodes(%2s) up(%2s) %s" % (count, sitename, sitehist.slices_used,
 126                                                                                         sitehist.nodes_total, sitehist.nodes_up, sitehist.status)
 127                         sitehist.flush()
 128
 129         print HistorySiteRecord.query.count()
 130         session.flush()
 131
 132         return True
 133
 134 if __name__ == '__main__':
 135         from monitor import parser as parsermodule
 136
 137         parser = parsermodule.getParser()
 138         parser.set_defaults(filename=None, node=None, site=None,
 139                                                 nodeselect=False, nodegroup=None, cachenodes=False)
 140
 141         parser.add_option("", "--site", dest="site", metavar="login_base",
 142                                                 help="Provide a single site to operate on")
 143         parser.add_option("", "--sitelist", dest="sitelist",
 144                                                 help="Provide a list of sites separated by ','")
 145
 146         config = parsermodule.parse_args(parser)
 147
 148         try:
 149                 main2(config)
 150         except Exception, err:
 151                 import traceback
 152                 print traceback.print_exc()
 153                 print "Exception: %s" % err
 154                 sys.exit(0)