findbad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import config
   8
   9
  10 # QUERY all nodes.
  11 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
  12                                 "table=table_nodeview&" + \
  13                                 "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
  14                                 "formatcsv"
  15                                     #"formatcsv&" + \
  16                                         #"select='lastcotop!=0'"
  17
  18 import threading
  19 plc_lock = threading.Lock()
  20 round = 1
  21 externalState = {'round': round, 'nodes': {}}
  22 count = 0
  23
  24
  25 import database
  26 import moncommands
  27 import comon
  28 import threadpool
  29 import syncplcdb
  30 from nodequery import verify,query_to_dict,node_select
  31 import traceback
  32 import plc
  33 api = plc.getAuthAPI()
  34
  35 def collectPingAndSSH(nodename, cohash):
  36         ### RUN PING ######################
  37         ping = moncommands.CMD()
  38         (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
  39
  40         values = {}
  41
  42         if oval == "":
  43                 # An error occurred
  44                 values['ping'] = "NOPING"
  45         else:
  46                 values['ping'] = "PING"
  47
  48         try:
  49                 for port in [22, 806]:
  50                         ssh = moncommands.SSH('root', nodename, port)
  51
  52                         (oval, errval) = ssh.run_noexcept2(""" <<\EOF
  53                                 echo "{"
  54                                 echo '  "kernel":"'`uname -a`'",'
  55                                 echo '  "bmlog":"'`ls /tmp/bm.log`'",'
  56                                 echo '  "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",'
  57                                 echo '  "nm":"'`ps ax | grep nm.py | grep -v grep`'",'
  58                                 echo '  "princeton_comon":"'`ls -d /vservers/princeton_comon`'",'
  59
  60                                 ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'`
  61
  62                                 echo '  "princeton_comon_running":"'`ls -d /proc/virtual/$ID`'",'
  63                                 echo '  "princeton_comon_procs":"'`vps ax | grep $ID | grep -v grep | wc -l`'",'
  64                                 echo "}"
  65 EOF                     """)
  66
  67                         if len(oval) > 0:
  68                                 values.update(eval(oval))
  69                                 values['sshport'] = port
  70                                 break
  71                         else:
  72                                 values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', 'nm' :
  73                                 '', 'princeton_comon' : '', 'princeton_comon_running' : '',
  74                                 'princeton_comon_procs' : '', 'sshport' : None})
  75         except:
  76                 print traceback.print_exc()
  77                 sys.exit(1)
  78
  79         ### RUN SSH ######################
  80         b_getbootcd_id = True
  81         #ssh = moncommands.SSH('root', nodename)
  82         #oval = ""
  83         #errval = ""
  84         #(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
  85
  86         oval = values['kernel']
  87         if "2.6.17" in oval or "2.6.2" in oval:
  88                 values['ssh'] = 'SSH'
  89                 values['category'] = 'ALPHA'
  90                 if "bm.log" in values['bmlog']:
  91                         values['state'] = 'DEBUG'
  92                 else:
  93                         values['state'] = 'BOOT'
  94         elif "2.6.12" in oval or "2.6.10" in oval:
  95                 values['ssh'] = 'SSH'
  96                 values['category'] = 'PROD'
  97                 if "bm.log" in values['bmlog']:
  98                         values['state'] = 'DEBUG'
  99                 else:
 100                         values['state'] = 'BOOT'
 101
 102         # NOTE: on 2.6.8 kernels, with 4.2 bootstrapfs, the chroot command fails.  I have no idea why.
 103         elif "2.4" in oval or "2.6.8" in oval:
 104                 b_getbootcd_id = False
 105                 values['ssh'] = 'SSH'
 106                 values['category'] = 'OLDBOOTCD'
 107                 values['state'] = 'DEBUG'
 108         elif oval != "":
 109                 values['ssh'] = 'SSH'
 110                 values['category'] = 'UNKNOWN'
 111                 if "bm.log" in values['bmlog']:
 112                         values['state'] = 'DEBUG'
 113                 else:
 114                         values['state'] = 'BOOT'
 115         else:
 116                 # An error occurred.
 117                 b_getbootcd_id = False
 118                 values['ssh'] = 'NOSSH'
 119                 values['category'] = 'ERROR'
 120                 values['state'] = 'DOWN'
 121                 val = errval.strip()
 122                 values['kernel'] = val
 123
 124         #values['kernel'] = val
 125
 126         if b_getbootcd_id:
 127                 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
 128                 #(oval, errval) = ssh.run_noexcept('cat /mnt/cdrom/bootme/ID')
 129                 oval = values['bootcd']
 130                 if "BootCD" in oval:
 131                         values['bootcd'] = oval
 132                         if "v2" in oval and \
 133                                 ( nodename is not "planetlab1.cs.unc.edu" and \
 134                                   nodename is not "planetlab2.cs.unc.edu" ):
 135                                 values['category'] = 'OLDBOOTCD'
 136                 else:
 137                         values['bootcd'] = ""
 138         else:
 139                 values['bootcd'] = ""
 140
 141         # TODO: get bm.log for debug nodes.
 142         # 'zcat /tmp/bm.log'
 143
 144         #(oval, errval) = ssh.run_noexcept('ps ax | grep nm.py | grep -v grep')
 145         oval = values['nm']
 146         if "nm.py" in oval:
 147                 values['nm'] = "Y"
 148         else:
 149                 values['nm'] = "N"
 150
 151         continue_slice_check = True
 152         #(oval, errval) = ssh.run_noexcept('ls -d /vservers/princeton_comon')
 153         oval = values['princeton_comon']
 154         if "princeton_comon" in oval:
 155                 values['princeton_comon'] = "Y"
 156         else:
 157                 values['princeton_comon'] = "N"
 158                 continue_slice_check = False
 159
 160         if continue_slice_check:
 161                 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; ls -d /proc/virtual/$ID')
 162                 oval = values['princeton_comon_running']
 163                 if len(oval) > len('/proc/virtual/'):
 164                         values['princeton_comon_running'] = "Y"
 165                 else:
 166                         values['princeton_comon_running'] = "N"
 167                         continue_slice_check = False
 168         else:
 169                 values['princeton_comon_running'] = "-"
 170
 171         if continue_slice_check:
 172                 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; vps ax | grep $ID | grep -v grep | wc -l')
 173                 oval = values['princeton_comon_procs']
 174                 values['princeton_comon_procs'] = oval
 175         else:
 176                 values['princeton_comon_procs'] = "-"
 177
 178
 179         if nodename in cohash:
 180                 values['comonstats'] = cohash[nodename]
 181         else:
 182                 values['comonstats'] = {'resptime':  '-1',
 183                                                                 'uptime':    '-1',
 184                                                                 'sshstatus': '-1',
 185                                                                 'lastcotop': '-1',
 186                                                                 'cpuspeed' : "null",
 187                                                                 'disksize' : 'null',
 188                                                                 'memsize'  : 'null'}
 189         # include output value
 190         ### GET PLC NODE ######################
 191         b_except = False
 192         plc_lock.acquire()
 193
 194         try:
 195                 d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'date_created', 'last_updated', 'last_contact', 'boot_state', 'nodegroup_ids'])
 196         except:
 197                 b_except = True
 198                 traceback.print_exc()
 199
 200         plc_lock.release()
 201         if b_except: return (None, None)
 202
 203         site_id = -1
 204         if d_node and len(d_node) > 0:
 205                 pcu = d_node[0]['pcu_ids']
 206                 if len(pcu) > 0:
 207                         values['pcu'] = "PCU"
 208                 else:
 209                         values['pcu'] = "NOPCU"
 210                 site_id = d_node[0]['site_id']
 211                 last_contact = d_node[0]['last_contact']
 212                 nodegroups = [ i['name'] for i in api.GetNodeGroups(d_node[0]['nodegroup_ids']) ]
 213                 values['plcnode'] = {'status' : 'SUCCESS',
 214                                                         'pcu_ids': pcu,
 215                                                         'boot_state' : d_node[0]['boot_state'],
 216                                                         'site_id': site_id,
 217                                                         'nodegroups' : nodegroups,
 218                                                         'last_contact': last_contact,
 219                                                         'date_created': d_node[0]['date_created'],
 220                                                         'last_updated': d_node[0]['last_updated']}
 221         else:
 222                 values['pcu']     = "UNKNOWN"
 223                 values['plcnode'] = {'status' : "GN_FAILED"}
 224
 225
 226         ### GET PLC SITE ######################
 227         b_except = False
 228         plc_lock.acquire()
 229
 230         try:
 231                 d_site = plc.getSites({'site_id': site_id},
 232                                                         ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
 233         except:
 234                 b_except = True
 235                 traceback.print_exc()
 236
 237         plc_lock.release()
 238         if b_except: return (None, None)
 239
 240         if d_site and len(d_site) > 0:
 241                 max_slices = d_site[0]['max_slices']
 242                 num_slices = len(d_site[0]['slice_ids'])
 243                 num_nodes = len(d_site[0]['node_ids'])
 244                 loginbase = d_site[0]['login_base']
 245                 values['plcsite'] = {'num_nodes' : num_nodes,
 246                                                         'max_slices' : max_slices,
 247                                                         'num_slices' : num_slices,
 248                                                         'login_base' : loginbase,
 249                                                         'status'     : 'SUCCESS'}
 250         else:
 251                 values['plcsite'] = {'status' : "GS_FAILED"}
 252
 253         values['checked'] = time.time()
 254
 255         return (nodename, values)
 256
 257 def recordPingAndSSH(request, result):
 258         global externalState
 259         global count
 260         (nodename, values) = result
 261
 262         if values is not None:
 263                 global_round = externalState['round']
 264                 externalState['nodes'][nodename]['values'] = values
 265                 externalState['nodes'][nodename]['round'] = global_round
 266
 267                 count += 1
 268                 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 269                 if count % 20 == 0:
 270                         database.dbDump(config.dbname, externalState)
 271
 272 # this will be called when an exception occurs within a thread
 273 def handle_exception(request, result):
 274         print "Exception occured in request %s" % request.requestID
 275         for i in result:
 276                 print "Result: %s" % i
 277
 278
 279 def checkAndRecordState(l_nodes, cohash):
 280         global externalState
 281         global count
 282         global_round = externalState['round']
 283
 284         tp = threadpool.ThreadPool(20)
 285
 286         # CREATE all the work requests
 287         for nodename in l_nodes:
 288                 if nodename not in externalState['nodes']:
 289                         externalState['nodes'][nodename] = {'round': 0, 'values': []}
 290
 291                 node_round   = externalState['nodes'][nodename]['round']
 292                 if node_round < global_round:
 293                         # recreate node stats when refreshed
 294                         #print "%s" % nodename
 295                         req = threadpool.WorkRequest(collectPingAndSSH, [nodename, cohash], {},
 296                                                                                  None, recordPingAndSSH, handle_exception)
 297                         tp.putRequest(req)
 298                 else:
 299                         # We just skip it, since it's "up to date"
 300                         count += 1
 301                         print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 302                         pass
 303
 304         # WAIT while all the work requests are processed.
 305         begin = time.time()
 306         while 1:
 307                 try:
 308                         time.sleep(1)
 309                         tp.poll()
 310                         # if more than two hours
 311                         if time.time() - begin > (60*60*1.5):
 312                                 print "findbad.py has run out of time!!!!!!"
 313                                 database.dbDump(config.dbname, externalState)
 314                                 os._exit(1)
 315                 except KeyboardInterrupt:
 316                         print "Interrupted!"
 317                         break
 318                 except threadpool.NoResultsPending:
 319                         print "All results collected."
 320                         break
 321
 322         database.dbDump(config.dbname, externalState)
 323
 324
 325
 326 def main():
 327         global externalState
 328
 329         externalState = database.if_cached_else(1, config.dbname, lambda : externalState)
 330
 331         if config.increment:
 332                 # update global round number to force refreshes across all nodes
 333                 externalState['round'] += 1
 334
 335         cotop = comon.Comon()
 336         # lastcotop measures whether cotop is actually running.  this is a better
 337         # metric than sshstatus, or other values from CoMon
 338         cotop_url = COMON_COTOPURL
 339
 340         # history information for all nodes
 341         #cohash = {}
 342         cohash = cotop.coget(cotop_url)
 343         l_nodes = syncplcdb.create_plcdb()
 344         if config.nodelist:
 345                 f_nodes = config.getListFromFile(config.nodelist)
 346                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 347         elif config.node:
 348                 f_nodes = [config.node]
 349                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 350         elif config.nodegroup:
 351                 ng = api.GetNodeGroups({'name' : config.nodegroup})
 352                 l_nodes = api.GetNodes(ng[0]['node_ids'])
 353         elif config.site:
 354                 site = api.GetSites(config.site)
 355                 l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
 356
 357         l_nodes = [node['hostname'] for node in l_nodes]
 358
 359         # perform this query after the above options, so that the filter above
 360         # does not break.
 361         if config.nodeselect:
 362                 fb = database.dbLoad("findbad")
 363                 l_nodes = node_select(config.nodeselect, fb['nodes'].keys(), fb)
 364
 365         print "fetching %s hosts" % len(l_nodes)
 366
 367         checkAndRecordState(l_nodes, cohash)
 368
 369         return 0
 370
 371
 372 if __name__ == '__main__':
 373         import parser as parsermodule
 374
 375         parser = parsermodule.getParser(['nodesets'])
 376
 377         parser.set_defaults( increment=False, dbname="findbad", cachenodes=False)
 378         parser.add_option("", "--cachenodes", action="store_true",
 379                                                 help="Cache node lookup from PLC")
 380         parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
 381                                                 help="Specify the name of the database to which the information is saved")
 382         parser.add_option("-i", "--increment", action="store_true", dest="increment",
 383                                                 help="Increment round number to force refresh or retry")
 384
 385         parser = parsermodule.getParser(['defaults'], parser)
 386
 387         cfg = parsermodule.parse_args(parser)
 388
 389         try:
 390                 main()
 391         except Exception, err:
 392                 print traceback.print_exc()
 393                 print "Exception: %s" % err
 394                 print "Saving data... exitting."
 395                 database.dbDump(config.dbname, externalState)
 396                 sys.exit(0)