findbad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7
   8 from config import config
   9 from optparse import OptionParser
  10 parser = OptionParser()
  11 parser.set_defaults(filename="", increment=False, dbname="findbadnodes", cachenodes=False)
  12 parser.add_option("-f", "--nodes", dest="filename", metavar="FILE",
  13                                         help="Provide the input file for the node list")
  14 parser.add_option("", "--cachenodes", action="store_true",
  15                                         help="Cache node lookup from PLC")
  16 parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
  17                                         help="Specify the name of the database to which the information is saved")
  18 parser.add_option("-i", "--increment", action="store_true", dest="increment",
  19                                         help="Increment round number to force refresh or retry")
  20 config = config(parser)
  21 config.parse_args()
  22
  23 # QUERY all nodes.
  24 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
  25                                         "table=table_nodeview&" + \
  26                                     "dumpcols='name,resptime,sshstatus,uptime,lastcotop'&" + \
  27                                     "formatcsv"
  28                                     #"formatcsv&" + \
  29                                         #"select='lastcotop!=0'"
  30
  31 import threading
  32 plc_lock = threading.Lock()
  33 round = 1
  34 externalState = {'round': round, 'nodes': {}}
  35 count = 0
  36
  37
  38 import soltesz
  39 import plc
  40 import comon
  41 import threadpool
  42 import syncplcdb
  43
  44 def collectPingAndSSH(nodename, cohash):
  45         ### RUN PING ######################
  46         ping = soltesz.CMD()
  47         (oval,eval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
  48
  49         values = {}
  50
  51         if oval == "":
  52                 # An error occurred
  53                 values['ping'] = "NOPING"
  54         else:
  55                 values['ping'] = "PING"
  56
  57         ### RUN SSH ######################
  58         b_getbootcd_id = True
  59         ssh = soltesz.SSH('root', nodename)
  60         oval = ""
  61         eval = ""
  62         (oval, eval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
  63         val = oval
  64         if "2.6.17" in oval or "2.6.20" in oval:
  65                 values['ssh'] = 'SSH'
  66                 if "bm.log" in oval:
  67                         values['category'] = 'ALPHA'
  68                         values['state'] = 'DEBUG'
  69                 else:
  70                         values['category'] = 'ALPHA'
  71                         values['state'] = 'BOOT'
  72         elif "2.6.12" in oval or "2.6.10" in oval:
  73                 values['ssh'] = 'SSH'
  74                 values['category'] = 'PROD'
  75                 if "bm.log" in oval:
  76                         values['state'] = 'DEBUG'
  77                 else:
  78                         values['state'] = 'BOOT'
  79         elif "2.4" in oval:
  80                 b_getbootcd_id = False
  81                 values['ssh'] = 'SSH'
  82                 values['category'] = 'OLDBOOTCD'
  83                 values['state'] = 'DEBUG'
  84         elif oval != "":
  85                 values['ssh'] = 'SSH'
  86                 values['category'] = 'UNKNOWN'
  87                 if "bm.log" in oval:
  88                         values['state'] = 'DEBUG'
  89                 else:
  90                         values['state'] = 'BOOT'
  91         else:
  92                 # An error occurred.
  93                 b_getbootcd_id = False
  94                 values['ssh'] = 'NOSSH'
  95                 values['category'] = 'ERROR'
  96                 values['state'] = 'DOWN'
  97                 val = eval.strip()
  98
  99         values['kernel'] = val
 100
 101         if b_getbootcd_id:
 102                 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
 103                 (oval, eval) = ssh.run_noexcept('cat /mnt/cdrom/bootme/ID')
 104                 val = oval
 105                 if "BootCD" in val:
 106                         values['bootcd'] = val
 107                         if "v2" in val and nodename is not "planetlab1.cs.unc.edu":
 108                                 values['category'] = 'OLDBOOTCD'
 109                 else:
 110                         values['bootcd'] = ""
 111         else:
 112                 values['bootcd'] = ""
 113
 114         # TODO: get bm.log for debug nodes.
 115         # 'zcat /tmp/bm.log'
 116
 117         if nodename in cohash:
 118                 values['comonstats'] = cohash[nodename]
 119         else:
 120                 values['comonstats'] = {'resptime':  '-1',
 121                                                                 'uptime':    '-1',
 122                                                                 'sshstatus': '-1',
 123                                                                 'lastcotop': '-1'}
 124         # include output value
 125         ### GET PLC NODE ######################
 126         b_except = False
 127         plc_lock.acquire()
 128
 129         try:
 130                 d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'last_contact', 'boot_state'])
 131         except:
 132                 b_except = True
 133                 import traceback
 134                 traceback.print_exc()
 135
 136         plc_lock.release()
 137         if b_except: return (None, None)
 138
 139         site_id = -1
 140         if d_node and len(d_node) > 0:
 141                 pcu = d_node[0]['pcu_ids']
 142                 if len(pcu) > 0:
 143                         values['pcu'] = "PCU"
 144                 else:
 145                         values['pcu'] = "NOPCU"
 146                 site_id = d_node[0]['site_id']
 147                 last_contact = d_node[0]['last_contact']
 148                 values['plcnode'] = {'status' : 'SUCCESS',
 149                                                         'pcu_ids': pcu,
 150                                                         'boot_state' : d_node[0]['boot_state'],
 151                                                         'site_id': site_id,
 152                                                         'last_contact': last_contact}
 153         else:
 154                 values['pcu']     = "UNKNOWN"
 155                 values['plcnode'] = {'status' : "GN_FAILED"}
 156
 157
 158         ### GET PLC SITE ######################
 159         b_except = False
 160         plc_lock.acquire()
 161
 162         try:
 163                 d_site = plc.getSites({'site_id': site_id},
 164                                                         ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
 165         except:
 166                 b_except = True
 167                 import traceback
 168                 traceback.print_exc()
 169
 170         plc_lock.release()
 171         if b_except: return (None, None)
 172
 173         if d_site and len(d_site) > 0:
 174                 max_slices = d_site[0]['max_slices']
 175                 num_slices = len(d_site[0]['slice_ids'])
 176                 num_nodes = len(d_site[0]['node_ids'])
 177                 loginbase = d_site[0]['login_base']
 178                 values['plcsite'] = {'num_nodes' : num_nodes,
 179                                                         'max_slices' : max_slices,
 180                                                         'num_slices' : num_slices,
 181                                                         'login_base' : loginbase,
 182                                                         'status'     : 'SUCCESS'}
 183         else:
 184                 values['plcsite'] = {'status' : "GS_FAILED"}
 185
 186         return (nodename, values)
 187
 188 def recordPingAndSSH(request, result):
 189         global externalState
 190         global count
 191         (nodename, values) = result
 192
 193         if values is not None:
 194                 global_round = externalState['round']
 195                 externalState['nodes'][nodename]['values'] = values
 196                 externalState['nodes'][nodename]['round'] = global_round
 197
 198                 count += 1
 199                 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 200                 soltesz.dbDump(config.dbname, externalState)
 201
 202 # this will be called when an exception occurs within a thread
 203 def handle_exception(request, result):
 204         print "Exception occured in request %s" % request.requestID
 205         for i in result:
 206                 print "Result: %s" % i
 207
 208
 209 def checkAndRecordState(l_nodes, cohash):
 210         global externalState
 211         global count
 212         global_round = externalState['round']
 213
 214         tp = threadpool.ThreadPool(20)
 215
 216         # CREATE all the work requests
 217         for nodename in l_nodes:
 218                 if nodename not in externalState['nodes']:
 219                         externalState['nodes'][nodename] = {'round': 0, 'values': []}
 220
 221                 node_round   = externalState['nodes'][nodename]['round']
 222                 if node_round < global_round:
 223                         # recreate node stats when refreshed
 224                         #print "%s" % nodename
 225                         req = threadpool.WorkRequest(collectPingAndSSH, [nodename, cohash], {},
 226                                                                                  None, recordPingAndSSH, handle_exception)
 227                         tp.putRequest(req)
 228                 else:
 229                         # We just skip it, since it's "up to date"
 230                         count += 1
 231                         print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 232                         pass
 233
 234         # WAIT while all the work requests are processed.
 235         while 1:
 236                 try:
 237                         time.sleep(1)
 238                         tp.poll()
 239                 except KeyboardInterrupt:
 240                         print "Interrupted!"
 241                         break
 242                 except threadpool.NoResultsPending:
 243                         print "All results collected."
 244                         break
 245
 246
 247
 248 def main():
 249         global externalState
 250
 251         externalState = soltesz.if_cached_else(1, config.dbname, lambda : externalState)
 252
 253         if config.increment:
 254                 # update global round number to force refreshes across all nodes
 255                 externalState['round'] += 1
 256
 257         cotop = comon.Comon()
 258         # lastcotop measures whether cotop is actually running.  this is a better
 259         # metric than sshstatus, or other values from CoMon
 260         cotop_url = COMON_COTOPURL
 261
 262         # history information for all nodes
 263         cohash = cotop.coget(cotop_url)
 264
 265         if config.filename == "":
 266                 l_nodes = syncplcdb.create_plcdb()
 267                 l_nodes = [node['hostname'] for node in l_nodes]
 268                 #l_nodes = cohash.keys()
 269         else:
 270                 l_nodes = config.getListFromFile(config.filename)
 271
 272         checkAndRecordState(l_nodes, cohash)
 273
 274         return 0
 275
 276
 277 if __name__ == '__main__':
 278         try:
 279                 main()
 280         except Exception, err:
 281                 print "Exception: %s" % err
 282                 print "Saving data... exitting."
 283                 soltesz.dbDump(config.dbname, externalState)
 284                 sys.exit(0)