findbad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import config
   8 import util.file
   9
  10
  11 # QUERY all nodes.
  12 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
  13                                 "table=table_nodeview&" + \
  14                                 "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
  15                                 "formatcsv"
  16                                     #"formatcsv&" + \
  17                                         #"select='lastcotop!=0'"
  18
  19 import threading
  20 plc_lock = threading.Lock()
  21 round = 1
  22 externalState = {'round': round, 'nodes': {}}
  23 count = 0
  24
  25
  26 import database
  27 import moncommands
  28 import comon
  29 import threadpool
  30 import syncplcdb
  31 from nodequery import verify,query_to_dict,node_select
  32 import traceback
  33 import plc
  34 api = plc.getAuthAPI()
  35
  36 def collectPingAndSSH(nodename, cohash):
  37         ### RUN PING ######################
  38         ping = moncommands.CMD()
  39         (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
  40
  41         values = {}
  42
  43         if oval == "":
  44                 # An error occurred
  45                 values['ping'] = "NOPING"
  46         else:
  47                 values['ping'] = "PING"
  48
  49         try:
  50                 for port in [22, 806]:
  51                         ssh = moncommands.SSH('root', nodename, port)
  52
  53                         (oval, errval) = ssh.run_noexcept2(""" <<\EOF
  54                                 echo "{"
  55                                 echo '  "kernel":"'`uname -a`'",'
  56                                 echo '  "bmlog":"'`ls /tmp/bm.log`'",'
  57                                 echo '  "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",'
  58                                 echo '  "nm":"'`ps ax | grep nm.py | grep -v grep`'",'
  59                                 echo '  "readonlyfs":"'`touch /var/log/monitor 2>&1 ; touch /vservers/monitor.log 2>&1`'",'
  60                                 echo '  "dns":"'`host boot.planet-lab.org 2>&1`'",'
  61                                 echo '  "princeton_comon":"'`ls -d /vservers/princeton_comon`'",'
  62
  63                                 ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'`
  64
  65                                 echo '  "princeton_comon_running":"'`ls -d /proc/virtual/$ID`'",'
  66                                 echo '  "princeton_comon_procs":"'`vps ax | grep $ID | grep -v grep | wc -l`'",'
  67                                 echo "}"
  68 EOF                     """)
  69
  70                         if len(oval) > 0:
  71                                 values.update(eval(oval))
  72                                 values['sshport'] = port
  73                                 break
  74                         else:
  75                                 values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '',
  76                                                                 'nm' : '',
  77                                                                 'readonlyfs' : '',
  78                                                                 'dns' : '',
  79                                                                 'princeton_comon' : '',
  80                                                                 'princeton_comon_running' : '',
  81                                                                 'princeton_comon_procs' : '', 'sshport' : None})
  82         except:
  83                 print traceback.print_exc()
  84                 from nodecommon import email_exception
  85                 email_exception()
  86                 sys.exit(1)
  87
  88         ### RUN SSH ######################
  89         b_getbootcd_id = True
  90         #ssh = moncommands.SSH('root', nodename)
  91         #oval = ""
  92         #errval = ""
  93         #(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
  94
  95         oval = values['kernel']
  96         if "2.6.17" in oval or "2.6.2" in oval:
  97                 values['ssh'] = 'SSH'
  98                 values['category'] = 'PROD'
  99                 if "bm.log" in values['bmlog']:
 100                         values['state'] = 'DEBUG'
 101                 else:
 102                         values['state'] = 'BOOT'
 103         elif "2.6.12" in oval or "2.6.10" in oval:
 104                 values['ssh'] = 'SSH'
 105                 values['category'] = 'OLDPROD'
 106                 if "bm.log" in values['bmlog']:
 107                         values['state'] = 'DEBUG'
 108                 else:
 109                         values['state'] = 'BOOT'
 110
 111         # NOTE: on 2.6.8 kernels, with 4.2 bootstrapfs, the chroot command fails.  I have no idea why.
 112         elif "2.4" in oval or "2.6.8" in oval:
 113                 b_getbootcd_id = False
 114                 values['ssh'] = 'SSH'
 115                 values['category'] = 'OLDBOOTCD'
 116                 values['state'] = 'DEBUG'
 117         elif oval != "":
 118                 values['ssh'] = 'SSH'
 119                 values['category'] = 'UNKNOWN'
 120                 if "bm.log" in values['bmlog']:
 121                         values['state'] = 'DEBUG'
 122                 else:
 123                         values['state'] = 'BOOT'
 124         else:
 125                 # An error occurred.
 126                 b_getbootcd_id = False
 127                 values['ssh'] = 'NOSSH'
 128                 values['category'] = 'ERROR'
 129                 values['state'] = 'DOWN'
 130                 val = errval.strip()
 131                 values['kernel'] = val
 132
 133         #values['kernel'] = val
 134
 135         if b_getbootcd_id:
 136                 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
 137                 #(oval, errval) = ssh.run_noexcept('cat /mnt/cdrom/bootme/ID')
 138                 oval = values['bootcd']
 139                 if "BootCD" in oval:
 140                         values['bootcd'] = oval
 141                         if "v2" in oval and \
 142                                 ( nodename is not "planetlab1.cs.unc.edu" and \
 143                                   nodename is not "planetlab2.cs.unc.edu" ):
 144                                 values['category'] = 'OLDBOOTCD'
 145                 else:
 146                         values['bootcd'] = ""
 147         else:
 148                 values['bootcd'] = ""
 149
 150         # TODO: get bm.log for debug nodes.
 151         # 'zcat /tmp/bm.log'
 152
 153         #(oval, errval) = ssh.run_noexcept('ps ax | grep nm.py | grep -v grep')
 154         oval = values['nm']
 155         if "nm.py" in oval:
 156                 values['nm'] = "Y"
 157         else:
 158                 values['nm'] = "N"
 159
 160         continue_slice_check = True
 161         #(oval, errval) = ssh.run_noexcept('ls -d /vservers/princeton_comon')
 162         oval = values['princeton_comon']
 163         if "princeton_comon" in oval:
 164                 values['princeton_comon'] = "Y"
 165         else:
 166                 values['princeton_comon'] = "N"
 167                 continue_slice_check = False
 168
 169         if continue_slice_check:
 170                 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; ls -d /proc/virtual/$ID')
 171                 oval = values['princeton_comon_running']
 172                 if len(oval) > len('/proc/virtual/'):
 173                         values['princeton_comon_running'] = "Y"
 174                 else:
 175                         values['princeton_comon_running'] = "N"
 176                         continue_slice_check = False
 177         else:
 178                 values['princeton_comon_running'] = "-"
 179
 180         if continue_slice_check:
 181                 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; vps ax | grep $ID | grep -v grep | wc -l')
 182                 oval = values['princeton_comon_procs']
 183                 values['princeton_comon_procs'] = oval
 184         else:
 185                 values['princeton_comon_procs'] = "-"
 186
 187
 188         if nodename in cohash:
 189                 values['comonstats'] = cohash[nodename]
 190         else:
 191                 values['comonstats'] = {'resptime':  '-1',
 192                                                                 'uptime':    '-1',
 193                                                                 'sshstatus': '-1',
 194                                                                 'lastcotop': '-1',
 195                                                                 'cpuspeed' : "null",
 196                                                                 'disksize' : 'null',
 197                                                                 'memsize'  : 'null'}
 198         # include output value
 199         ### GET PLC NODE ######################
 200         b_except = False
 201         plc_lock.acquire()
 202
 203         try:
 204                 d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'date_created', 'last_updated', 'last_contact', 'boot_state', 'nodegroup_ids'])
 205         except:
 206                 b_except = True
 207                 traceback.print_exc()
 208                 from nodecommon import email_exception
 209                 email_exception()
 210
 211         plc_lock.release()
 212         if b_except: return (None, None)
 213
 214         site_id = -1
 215         if d_node and len(d_node) > 0:
 216                 pcu = d_node[0]['pcu_ids']
 217                 if len(pcu) > 0:
 218                         values['pcu'] = "PCU"
 219                 else:
 220                         values['pcu'] = "NOPCU"
 221                 site_id = d_node[0]['site_id']
 222                 last_contact = d_node[0]['last_contact']
 223                 nodegroups = [ i['groupname'] for i in api.GetNodeGroups(d_node[0]['nodegroup_ids']) ]
 224                 values['plcnode'] = {'status' : 'SUCCESS',
 225                                                         'pcu_ids': pcu,
 226                                                         'boot_state' : d_node[0]['boot_state'],
 227                                                         'site_id': site_id,
 228                                                         'nodegroups' : nodegroups,
 229                                                         'last_contact': last_contact,
 230                                                         'date_created': d_node[0]['date_created'],
 231                                                         'last_updated': d_node[0]['last_updated']}
 232         else:
 233                 values['pcu']     = "UNKNOWN"
 234                 values['plcnode'] = {'status' : "GN_FAILED"}
 235
 236
 237         ### GET PLC SITE ######################
 238         b_except = False
 239         plc_lock.acquire()
 240
 241         try:
 242                 d_site = plc.getSites({'site_id': site_id},
 243                                                         ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
 244         except:
 245                 b_except = True
 246                 traceback.print_exc()
 247                 from nodecommon import email_exception
 248                 email_exception()
 249
 250         plc_lock.release()
 251         if b_except: return (None, None)
 252
 253         if d_site and len(d_site) > 0:
 254                 max_slices = d_site[0]['max_slices']
 255                 num_slices = len(d_site[0]['slice_ids'])
 256                 num_nodes = len(d_site[0]['node_ids'])
 257                 loginbase = d_site[0]['login_base']
 258                 values['plcsite'] = {'num_nodes' : num_nodes,
 259                                                         'max_slices' : max_slices,
 260                                                         'num_slices' : num_slices,
 261                                                         'login_base' : loginbase,
 262                                                         'status'     : 'SUCCESS'}
 263         else:
 264                 values['plcsite'] = {'status' : "GS_FAILED"}
 265
 266         values['checked'] = time.time()
 267
 268         return (nodename, values)
 269
 270 def recordPingAndSSH(request, result):
 271         global externalState
 272         global count
 273         (nodename, values) = result
 274
 275         if values is not None:
 276                 global_round = externalState['round']
 277                 externalState['nodes'][nodename]['values'] = values
 278                 externalState['nodes'][nodename]['round'] = global_round
 279
 280                 count += 1
 281                 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 282                 if count % 20 == 0:
 283                         database.dbDump(config.dbname, externalState)
 284
 285 # this will be called when an exception occurs within a thread
 286 def handle_exception(request, result):
 287         print "Exception occured in request %s" % request.requestID
 288         for i in result:
 289                 print "Result: %s" % i
 290
 291
 292 def checkAndRecordState(l_nodes, cohash):
 293         global externalState
 294         global count
 295         global_round = externalState['round']
 296
 297         tp = threadpool.ThreadPool(20)
 298
 299         # CREATE all the work requests
 300         for nodename in l_nodes:
 301                 if nodename not in externalState['nodes']:
 302                         externalState['nodes'][nodename] = {'round': 0, 'values': []}
 303
 304                 node_round   = externalState['nodes'][nodename]['round']
 305                 if node_round < global_round:
 306                         # recreate node stats when refreshed
 307                         #print "%s" % nodename
 308                         req = threadpool.WorkRequest(collectPingAndSSH, [nodename, cohash], {},
 309                                                                                  None, recordPingAndSSH, handle_exception)
 310                         tp.putRequest(req)
 311                 else:
 312                         # We just skip it, since it's "up to date"
 313                         count += 1
 314                         print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 315                         pass
 316
 317         # WAIT while all the work requests are processed.
 318         begin = time.time()
 319         while 1:
 320                 try:
 321                         time.sleep(1)
 322                         tp.poll()
 323                         # if more than two hours
 324                         if time.time() - begin > (60*60*1.5):
 325                                 print "findbad.py has run out of time!!!!!!"
 326                                 database.dbDump(config.dbname, externalState)
 327                                 os._exit(1)
 328                 except KeyboardInterrupt:
 329                         print "Interrupted!"
 330                         break
 331                 except threadpool.NoResultsPending:
 332                         print "All results collected."
 333                         break
 334
 335         database.dbDump(config.dbname, externalState)
 336
 337
 338
 339 def main():
 340         global externalState
 341
 342         externalState = database.if_cached_else(1, config.dbname, lambda : externalState)
 343
 344         if config.increment:
 345                 # update global round number to force refreshes across all nodes
 346                 externalState['round'] += 1
 347
 348         cotop = comon.Comon()
 349         # lastcotop measures whether cotop is actually running.  this is a better
 350         # metric than sshstatus, or other values from CoMon
 351         cotop_url = COMON_COTOPURL
 352
 353         # history information for all nodes
 354         #cohash = {}
 355         cohash = cotop.coget(cotop_url)
 356         l_nodes = syncplcdb.create_plcdb()
 357         if config.nodelist:
 358                 f_nodes = util.file.getListFromFile(config.nodelist)
 359                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 360         elif config.node:
 361                 f_nodes = [config.node]
 362                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 363         elif config.nodegroup:
 364                 ng = api.GetNodeGroups({'groupname' : config.nodegroup})
 365                 l_nodes = api.GetNodes(ng[0]['node_ids'])
 366         elif config.site:
 367                 site = api.GetSites(config.site)
 368                 l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
 369
 370         l_nodes = [node['hostname'] for node in l_nodes]
 371
 372         # perform this query after the above options, so that the filter above
 373         # does not break.
 374         if config.nodeselect:
 375                 fb = database.dbLoad("findbad")
 376                 l_nodes = node_select(config.nodeselect, fb['nodes'].keys(), fb)
 377
 378         print "fetching %s hosts" % len(l_nodes)
 379
 380         checkAndRecordState(l_nodes, cohash)
 381
 382         return 0
 383
 384
 385 if __name__ == '__main__':
 386         import parser as parsermodule
 387
 388         parser = parsermodule.getParser(['nodesets'])
 389
 390         parser.set_defaults( increment=False, dbname="findbad", cachenodes=False)
 391         parser.add_option("", "--cachenodes", action="store_true",
 392                                                 help="Cache node lookup from PLC")
 393         parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
 394                                                 help="Specify the name of the database to which the information is saved")
 395         parser.add_option("-i", "--increment", action="store_true", dest="increment",
 396                                                 help="Increment round number to force refresh or retry")
 397
 398         parser = parsermodule.getParser(['defaults'], parser)
 399
 400         cfg = parsermodule.parse_args(parser)
 401
 402         try:
 403                 main()
 404         except Exception, err:
 405                 print traceback.print_exc()
 406                 from nodecommon import email_exception
 407                 email_exception()
 408                 print "Exception: %s" % err
 409                 print "Saving data... exitting."
 410                 database.dbDump(config.dbname, externalState)
 411                 sys.exit(0)