findbad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7
   8
   9 # QUERY all nodes.
  10 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
  11                                         "table=table_nodeview&" + \
  12                                     "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
  13                                     "formatcsv"
  14                                     #"formatcsv&" + \
  15                                         #"select='lastcotop!=0'"
  16
  17 import threading
  18 plc_lock = threading.Lock()
  19 round = 1
  20 externalState = {'round': round, 'nodes': {}}
  21 count = 0
  22
  23
  24 import soltesz
  25 import comon
  26 import threadpool
  27 import syncplcdb
  28 from nodequery import verify,query_to_dict,node_select
  29
  30 import plc
  31 import auth
  32 api = plc.PLC(auth.auth, auth.plc)
  33
  34 def collectPingAndSSH(nodename, cohash):
  35         ### RUN PING ######################
  36         ping = soltesz.CMD()
  37         (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
  38
  39         values = {}
  40
  41         if oval == "":
  42                 # An error occurred
  43                 values['ping'] = "NOPING"
  44         else:
  45                 values['ping'] = "PING"
  46
  47         try:
  48                 for port in [22, 806]:
  49                         ssh = soltesz.SSH('root', nodename, port)
  50
  51                         (oval, errval) = ssh.run_noexcept2(""" <<\EOF
  52                                 echo "{"
  53                                 echo '  "kernel":"'`uname -a`'",'
  54                                 echo '  "bmlog":"'`ls /tmp/bm.log`'",'
  55                                 echo '  "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",'
  56                                 echo '  "nm":"'`ps ax | grep nm.py | grep -v grep`'",'
  57                                 echo '  "princeton_comon":"'`ls -d /vservers/princeton_comon`'",'
  58
  59                                 ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'`
  60
  61                                 echo '  "princeton_comon_running":"'`ls -d /proc/virtual/$ID`'",'
  62                                 echo '  "princeton_comon_procs":"'`vps ax | grep $ID | grep -v grep | wc -l`'",'
  63                                 echo "}"
  64 EOF                     """)
  65
  66                         if len(oval) > 0:
  67                                 values.update(eval(oval))
  68                                 values['sshport'] = port
  69                                 break
  70                         else:
  71                                 values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', 'nm' :
  72                                 '', 'princeton_comon' : '', 'princeton_comon_running' : '',
  73                                 'princeton_comon_procs' : '', 'sshport' : None})
  74         except:
  75                 import traceback; print traceback.print_exc()
  76                 sys.exit(1)
  77
  78         ### RUN SSH ######################
  79         b_getbootcd_id = True
  80         #ssh = soltesz.SSH('root', nodename)
  81         #oval = ""
  82         #errval = ""
  83         #(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
  84
  85         oval = values['kernel']
  86         if "2.6.17" in oval or "2.6.2" in oval:
  87                 values['ssh'] = 'SSH'
  88                 values['category'] = 'ALPHA'
  89                 if "bm.log" in values['bmlog']:
  90                         values['state'] = 'DEBUG'
  91                 else:
  92                         values['state'] = 'BOOT'
  93         elif "2.6.12" in oval or "2.6.10" in oval:
  94                 values['ssh'] = 'SSH'
  95                 values['category'] = 'PROD'
  96                 if "bm.log" in values['bmlog']:
  97                         values['state'] = 'DEBUG'
  98                 else:
  99                         values['state'] = 'BOOT'
 100
 101         # NOTE: on 2.6.8 kernels, with 4.2 bootstrapfs, the chroot command fails.  I have no idea why.
 102         elif "2.4" in oval or "2.6.8" in oval:
 103                 b_getbootcd_id = False
 104                 values['ssh'] = 'SSH'
 105                 values['category'] = 'OLDBOOTCD'
 106                 values['state'] = 'DEBUG'
 107         elif oval != "":
 108                 values['ssh'] = 'SSH'
 109                 values['category'] = 'UNKNOWN'
 110                 if "bm.log" in values['bmlog']:
 111                         values['state'] = 'DEBUG'
 112                 else:
 113                         values['state'] = 'BOOT'
 114         else:
 115                 # An error occurred.
 116                 b_getbootcd_id = False
 117                 values['ssh'] = 'NOSSH'
 118                 values['category'] = 'ERROR'
 119                 values['state'] = 'DOWN'
 120                 val = errval.strip()
 121                 values['kernel'] = val
 122
 123         #values['kernel'] = val
 124
 125         if b_getbootcd_id:
 126                 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
 127                 #(oval, errval) = ssh.run_noexcept('cat /mnt/cdrom/bootme/ID')
 128                 oval = values['bootcd']
 129                 if "BootCD" in oval:
 130                         values['bootcd'] = oval
 131                         if "v2" in oval and \
 132                                 ( nodename is not "planetlab1.cs.unc.edu" and \
 133                                   nodename is not "planetlab2.cs.unc.edu" ):
 134                                 values['category'] = 'OLDBOOTCD'
 135                 else:
 136                         values['bootcd'] = ""
 137         else:
 138                 values['bootcd'] = ""
 139
 140         # TODO: get bm.log for debug nodes.
 141         # 'zcat /tmp/bm.log'
 142
 143         #(oval, errval) = ssh.run_noexcept('ps ax | grep nm.py | grep -v grep')
 144         oval = values['nm']
 145         if "nm.py" in oval:
 146                 values['nm'] = "Y"
 147         else:
 148                 values['nm'] = "N"
 149
 150         continue_slice_check = True
 151         #(oval, errval) = ssh.run_noexcept('ls -d /vservers/princeton_comon')
 152         oval = values['princeton_comon']
 153         if "princeton_comon" in oval:
 154                 values['princeton_comon'] = "Y"
 155         else:
 156                 values['princeton_comon'] = "N"
 157                 continue_slice_check = False
 158
 159         if continue_slice_check:
 160                 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; ls -d /proc/virtual/$ID')
 161                 oval = values['princeton_comon_running']
 162                 if len(oval) > len('/proc/virtual/'):
 163                         values['princeton_comon_running'] = "Y"
 164                 else:
 165                         values['princeton_comon_running'] = "N"
 166                         continue_slice_check = False
 167         else:
 168                 values['princeton_comon_running'] = "-"
 169
 170         if continue_slice_check:
 171                 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; vps ax | grep $ID | grep -v grep | wc -l')
 172                 oval = values['princeton_comon_procs']
 173                 values['princeton_comon_procs'] = oval
 174         else:
 175                 values['princeton_comon_procs'] = "-"
 176
 177
 178         if nodename in cohash:
 179                 values['comonstats'] = cohash[nodename]
 180         else:
 181                 values['comonstats'] = {'resptime':  '-1',
 182                                                                 'uptime':    '-1',
 183                                                                 'sshstatus': '-1',
 184                                                                 'lastcotop': '-1'}
 185         # include output value
 186         ### GET PLC NODE ######################
 187         b_except = False
 188         plc_lock.acquire()
 189
 190         try:
 191                 d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'last_contact', 'boot_state', 'nodegroup_ids'])
 192         except:
 193                 b_except = True
 194                 import traceback
 195                 b_except = True
 196                 import traceback
 197                 traceback.print_exc()
 198
 199         plc_lock.release()
 200         if b_except: return (None, None)
 201
 202         site_id = -1
 203         if d_node and len(d_node) > 0:
 204                 pcu = d_node[0]['pcu_ids']
 205                 if len(pcu) > 0:
 206                         values['pcu'] = "PCU"
 207                 else:
 208                         values['pcu'] = "NOPCU"
 209                 site_id = d_node[0]['site_id']
 210                 last_contact = d_node[0]['last_contact']
 211                 nodegroups = d_node[0]['nodegroup_ids']
 212                 values['plcnode'] = {'status' : 'SUCCESS',
 213                                                         'pcu_ids': pcu,
 214                                                         'boot_state' : d_node[0]['boot_state'],
 215                                                         'site_id': site_id,
 216                                                         'nodegroups' : nodegroups,
 217                                                         'last_contact': last_contact}
 218         else:
 219                 values['pcu']     = "UNKNOWN"
 220                 values['plcnode'] = {'status' : "GN_FAILED"}
 221
 222
 223         ### GET PLC SITE ######################
 224         b_except = False
 225         plc_lock.acquire()
 226
 227         try:
 228                 d_site = plc.getSites({'site_id': site_id},
 229                                                         ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
 230         except:
 231                 b_except = True
 232                 import traceback
 233                 traceback.print_exc()
 234
 235         plc_lock.release()
 236         if b_except: return (None, None)
 237
 238         if d_site and len(d_site) > 0:
 239                 max_slices = d_site[0]['max_slices']
 240                 num_slices = len(d_site[0]['slice_ids'])
 241                 num_nodes = len(d_site[0]['node_ids'])
 242                 loginbase = d_site[0]['login_base']
 243                 values['plcsite'] = {'num_nodes' : num_nodes,
 244                                                         'max_slices' : max_slices,
 245                                                         'num_slices' : num_slices,
 246                                                         'login_base' : loginbase,
 247                                                         'status'     : 'SUCCESS'}
 248         else:
 249                 values['plcsite'] = {'status' : "GS_FAILED"}
 250
 251         values['checked'] = time.time()
 252
 253         return (nodename, values)
 254
 255 def recordPingAndSSH(request, result):
 256         global externalState
 257         global count
 258         (nodename, values) = result
 259
 260         if values is not None:
 261                 global_round = externalState['round']
 262                 externalState['nodes'][nodename]['values'] = values
 263                 externalState['nodes'][nodename]['round'] = global_round
 264
 265                 count += 1
 266                 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 267                 if count % 20 == 0:
 268                         soltesz.dbDump(config.dbname, externalState)
 269
 270 # this will be called when an exception occurs within a thread
 271 def handle_exception(request, result):
 272         print "Exception occured in request %s" % request.requestID
 273         for i in result:
 274                 print "Result: %s" % i
 275
 276
 277 def checkAndRecordState(l_nodes, cohash):
 278         global externalState
 279         global count
 280         global_round = externalState['round']
 281
 282         tp = threadpool.ThreadPool(20)
 283
 284         # CREATE all the work requests
 285         for nodename in l_nodes:
 286                 if nodename not in externalState['nodes']:
 287                         externalState['nodes'][nodename] = {'round': 0, 'values': []}
 288
 289                 node_round   = externalState['nodes'][nodename]['round']
 290                 if node_round < global_round:
 291                         # recreate node stats when refreshed
 292                         #print "%s" % nodename
 293                         req = threadpool.WorkRequest(collectPingAndSSH, [nodename, cohash], {},
 294                                                                                  None, recordPingAndSSH, handle_exception)
 295                         tp.putRequest(req)
 296                 else:
 297                         # We just skip it, since it's "up to date"
 298                         count += 1
 299                         print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 300                         pass
 301
 302         # WAIT while all the work requests are processed.
 303         while 1:
 304                 try:
 305                         time.sleep(1)
 306                         tp.poll()
 307                 except KeyboardInterrupt:
 308                         print "Interrupted!"
 309                         break
 310                 except threadpool.NoResultsPending:
 311                         print "All results collected."
 312                         break
 313
 314         soltesz.dbDump(config.dbname, externalState)
 315
 316
 317
 318 def main():
 319         global externalState
 320
 321         externalState = soltesz.if_cached_else(1, config.dbname, lambda : externalState)
 322
 323         if config.increment:
 324                 # update global round number to force refreshes across all nodes
 325                 externalState['round'] += 1
 326
 327         cotop = comon.Comon()
 328         # lastcotop measures whether cotop is actually running.  this is a better
 329         # metric than sshstatus, or other values from CoMon
 330         cotop_url = COMON_COTOPURL
 331
 332         # history information for all nodes
 333         #cohash = {}
 334         cohash = cotop.coget(cotop_url)
 335         l_nodes = syncplcdb.create_plcdb()
 336         if config.filename:
 337                 f_nodes = config.getListFromFile(config.filename)
 338                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 339         elif config.node:
 340                 f_nodes = [config.node]
 341                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 342         elif config.nodegroup:
 343                 ng = api.GetNodeGroups({'name' : config.nodegroup})
 344                 l_nodes = api.GetNodes(ng[0]['node_ids'])
 345         elif config.site:
 346                 site = api.GetSites(config.site)
 347                 l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
 348
 349         l_nodes = [node['hostname'] for node in l_nodes]
 350
 351         # perform this query after the above options, so that the filter above
 352         # does not break.
 353         if config.nodeselect:
 354                 l_nodes = node_select(config.nodeselect)
 355
 356         print "fetching %s hosts" % len(l_nodes)
 357
 358         checkAndRecordState(l_nodes, cohash)
 359
 360         return 0
 361
 362
 363 if __name__ == '__main__':
 364         from config import config
 365         from optparse import OptionParser
 366         parser = OptionParser()
 367         parser.set_defaults(filename=None, node=None, site=None, nodeselect=False, nodegroup=None,
 368                                                 increment=False, dbname="findbadnodes", cachenodes=False)
 369         parser.add_option("", "--node", dest="node", metavar="hostname",
 370                                                 help="Provide a single node to operate on")
 371         parser.add_option("-f", "--nodelist", dest="filename", metavar="FILE",
 372                                                 help="Provide the input file for the node list")
 373         parser.add_option("", "--nodeselect", dest="nodeselect", metavar="query string",
 374                                                 help="Provide a selection string to return a node list.")
 375         parser.add_option("", "--nodegroup", dest="nodegroup", metavar="FILE",
 376                                                 help="Provide the nodegroup for the list of nodes.")
 377         parser.add_option("", "--site", dest="site", metavar="site name",
 378                                                 help="Specify a site to view node status")
 379
 380         parser.add_option("", "--cachenodes", action="store_true",
 381                                                 help="Cache node lookup from PLC")
 382         parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
 383                                                 help="Specify the name of the database to which the information is saved")
 384         parser.add_option("-i", "--increment", action="store_true", dest="increment",
 385                                                 help="Increment round number to force refresh or retry")
 386         config = config(parser)
 387         config.parse_args()
 388
 389         try:
 390                 main()
 391         except Exception, err:
 392                 import traceback
 393                 print traceback.print_exc()
 394                 print "Exception: %s" % err
 395                 print "Saving data... exitting."
 396                 soltesz.dbDump(config.dbname, externalState)
 397                 sys.exit(0)