comon.py

   1 #
   2 # Copyright (c) 2004  The Trustees of Princeton University (Trustees).
   3 #
   4 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
   5 #
   6 # $Id: comon.py,v 1.7 2007/07/03 19:59:02 soltesz Exp $
   7 #
   8 # Get CoMon data, unsorted, in CSV, and create a huge hash.
   9 #
  10
  11
  12 import urllib2
  13 import httplib
  14 import time
  15 import Queue
  16 import logging
  17 import pickle
  18 from threading import *
  19 #httplib.HTTPConnection.debuglevel = 1
  20
  21 logger = logging.getLogger("monitor")
  22
  23 # Time between comon refresh
  24 COSLEEP=1200
  25
  26 # CoMon
  27 COMONURL = "http://summer.cs.princeton.edu/status/tabulator.cgi?table=table_nodeview"
  28
  29 # node type:
  30 # null == <not in DB?>
  31 #        0 ==
  32 #        1 == Prod
  33 #        2 == alpha
  34 #        3 == beta
  35
  36 # boot state:
  37 #       0 == new
  38 #       1 == boot
  39 #       2 == dbg
  40 #       3 == rins
  41 #       4 == ins
  42
  43
  44 class Comon(Thread):
  45         """
  46         cdb is the comon database (dictionary)
  47         all buckets is a queue of all problem nodes. This gets sent to rt to find
  48         tickets open for host.
  49         """
  50         def __init__(self, cdb=None, d_allplc_nodes=None, q_allbuckets=None):
  51
  52                 self.accept_all_nodes = False
  53
  54                 if cdb == None:
  55                         cdb = {}
  56                 if d_allplc_nodes == None:
  57                         self.accept_all_nodes = True # TODO :get from plc.
  58
  59                 self.codata = cdb
  60                 self.d_allplc_nodes = d_allplc_nodes
  61                 self.updated = time.time()
  62                 self.q_allbuckets = q_allbuckets
  63                 #self.comon_buckets = {"down" : "resptime%20==%200%20&&%20keyok==null",
  64                 #       "ssh": "sshstatus%20%3E%202h",
  65                 #       "clock_drift": "drift%20%3E%201m",
  66                 #       "dns": "dns1udp%20%3E%2080%20&&%20dns2udp%20%3E%2080",
  67                 #       "filerw": "filerw%3E0",
  68                 #       "dbg" : "keyok==0"}
  69                 self.comon_buckets = {
  70                         #"down" : "resptime==0&&keyok==null",
  71                         #"ssh": "sshstatus > 2h",
  72                         #"clock_drift": "drift > 1m",
  73                         #"dns": "dns1udp>80 && dns2udp>80",
  74                         #"filerw": "filerw > 0",
  75                         #"all" : ""
  76                         "dbg" : "keyok==0",
  77                         }
  78                 Thread.__init__(self)
  79
  80         def __tohash(self,rawdata):
  81                 # First line Comon returns is list of keys with respect to index
  82                 keys = rawdata.readline().rstrip().split(", ")
  83                 l_host = []
  84                 hash = {}
  85                 try:
  86                         i_ignored = 0
  87                         for line in rawdata.readlines():
  88                                 l_host = line.rstrip().split(", ")              # split the line on ', '
  89                                 hostname = l_host[0]
  90                                 add = False
  91                                 if self.accept_all_nodes:
  92                                         add=True
  93                                 else:
  94                                         if hostname in self.d_allplc_nodes:             # then we'll track it
  95                                                 add = True
  96
  97                                 if add:
  98                                         hash[hostname] = {}
  99                                         for i in range(1,len(keys)):
 100                                                 hash[hostname][keys[i]]=l_host[i]
 101                                 else:
 102                                         i_ignored += 1
 103
 104                         print "Retrieved %s hosts" % len(hash.keys())
 105                         print "Ignoring %d hosts" % i_ignored
 106
 107                         logger.debug("Retrieved %s hosts" % len(hash.keys()))
 108                         logger.debug("Ignoring %d hosts" % i_ignored)
 109                 except Exception, err:
 110                         logger.debug("No hosts retrieved")
 111                         return {}
 112                 return hash
 113
 114         # Update individual buckekts.  Hostnames only.
 115         def updatebuckets(self):
 116                 for (bucket,url) in self.comon_buckets.items():
 117                         logger.debug("COMON:  Updating bucket %s" % bucket)
 118                         tmp = self.coget(COMONURL + "&format=formatcsv&select='" + url + "'").keys()
 119                         setattr(self, bucket, tmp)
 120
 121         # Update ALL node information
 122         def updatedb(self):
 123                 # Get time of update
 124                 self.updated = time.time()
 125                 # Make a Hash, put in self.
 126                 self.codata.update(self.coget(COMONURL + "&format=formatcsv"))
 127
 128         def coget(self,url):
 129                 rawdata = None
 130                 print "Getting: %s" % url
 131                 try:
 132                         coserv = urllib2.Request(url)
 133                         coserv.add_header('User-Agent',
 134                                 'PL_Monitor +http://monitor.planet-lab.org/')
 135                         opener = urllib2.build_opener()
 136                         # Initial web get from summer.cs in CSV
 137                         rawdata = opener.open(coserv)
 138                 except urllib2.URLError, (err):
 139                         print "Attempting %s" %COMONURL
 140                         print "URL error (%s)" % (err)
 141                         rawdata = None
 142                 return self.__tohash(rawdata)
 143
 144         # Push nodes that are bad (in *a* bucket) into q(q_allbuckets)
 145         def push(self):
 146                 #buckets_per_node = []
 147                 #for bucket in self.comon.comon_buckets.keys():
 148                 #       if (hostname in getattr(self.comon, bucket)):
 149                 #               buckets_per_node.append(bucket)
 150
 151                 #loginbase = self.plcdb_hn2lb[hostname] # plc.siteId(node)
 152
 153                 #if not loginbase in self.sickdb:
 154                 #       self.sickdb[loginbase] = [{hostname: buckets_per_node}]
 155                 #else:
 156                 #       self.sickdb[loginbase].append({hostname: buckets_per_node})
 157
 158
 159                 print "calling Comon.push()"
 160                 for bucket in self.comon_buckets.keys():
 161                         #print "bucket: %s" % bucket
 162                         for host in getattr(self,bucket):
 163                                 diag_node = {}
 164                                 diag_node['nodename'] = host
 165                                 diag_node['message'] = None
 166                                 diag_node['bucket'] = [bucket]
 167                                 diag_node['stage'] = ""
 168                                 #diag_node['ticket_id'] = ""
 169                                 diag_node['args'] = None
 170                                 diag_node['info'] = None
 171                                 diag_node['time'] = time.time()
 172                                 #print "host: %s" % host
 173                                 self.q_allbuckets.put(diag_node)
 174
 175         def run(self):
 176                 self.updatedb()
 177                 self.updatebuckets()
 178                 self.push()
 179                 # insert signal that this is the final host
 180                 self.q_allbuckets.put("None")
 181
 182         def __repr__(self):
 183             return self
 184
 185 def main():
 186         logger.setLevel(logging.DEBUG)
 187         ch = logging.StreamHandler()
 188         ch.setLevel(logging.DEBUG)
 189         formatter = logging.Formatter('%(message)s')
 190         ch.setFormatter(formatter)
 191         logger.addHandler(ch)
 192
 193
 194         t = Queue.Queue()
 195         cdb = {}
 196         a = Comon(cdb,t)
 197         #for i in a.comon_buckets: print "%s : %s" % ( i, a.comon_buckets[i])
 198         a.start()
 199
 200         time.sleep(5)
 201         #for i in a.down: print i
 202
 203         time.sleep(5)
 204         #print cdb
 205         for host in cdb.keys():
 206                 #if cdb[host]['keyok'] == "0":
 207                 # null implies that it may not be in PL DB.
 208                 if  cdb[host]['bootstate'] != "null" and \
 209                         cdb[host]['bootstate'] == "2" and \
 210                         cdb[host]['keyok'] == "0":
 211                         print("%-40s \t Bootstate %s nodetype %s kernver %s keyok %s" % (
 212                                 host, cdb[host]['bootstate'], cdb[host]['nodetype'],
 213                                 cdb[host]['kernver'], cdb[host]['keyok']))
 214                         #ssh = soltesz.SSH('root', host)
 215                         #try:
 216                         #       val = ssh.run("uname -r")
 217                         #       print "%s == %s" % (host, val),
 218                         #except:
 219                         #       pass
 220         #       else:
 221         #               print("key mismatch at: %s" % host)
 222         #print a.codata['michelangelo.ani.univie.ac.at']
 223         #time.sleep(3)
 224         #a.push()
 225         #print a.filerw
 226         #print a.coget(COMONURL + "&format=formatcsv&select='" + a.comon_buckets['filerw'])
 227
 228         #os._exit(0)
 229 if __name__ == '__main__':
 230         import os
 231         try:
 232                 main()
 233         except KeyboardInterrupt:
 234                 print "Killed.  Exitting."
 235                 logger.info('Monitor Killed')
 236                 os._exit(0)