findbad.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 from datetime import datetime,timedelta
   8 import threadpool
   9 import threading
  10
  11 from monitor import util
  12 from monitor.util import command
  13 from monitor import config
  14 from monitor.database import FindbadNodeRecordSync, FindbadNodeRecord
  15 from monitor.sources import comon
  16 from monitor.wrapper import plc
  17
  18 import syncplcdb
  19 from nodequery import verify,query_to_dict,node_select
  20 import traceback
  21
  22 print "starting sqlfindbad.py"
  23 # QUERY all nodes.
  24 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
  25                                 "table=table_nodeview&" + \
  26                                 "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
  27                                 "formatcsv"
  28                                     #"formatcsv&" + \
  29                                         #"select='lastcotop!=0'"
  30
  31 api = plc.getAuthAPI()
  32 plc_lock = threading.Lock()
  33 round = 1
  34 global_round = round
  35 count = 0
  36
  37 def collectPingAndSSH(nodename, cohash):
  38         ### RUN PING ######################
  39         ping = command.CMD()
  40         (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
  41
  42         try:
  43                 values = {}
  44
  45                 if oval == "":
  46                         # An error occurred
  47                         values['ping'] = "NOPING"
  48                 else:
  49                         values['ping'] = "PING"
  50
  51                 try:
  52                         for port in [22, 806]:
  53                                 ssh = command.SSH('root', nodename, port)
  54
  55                                 (oval, errval) = ssh.run_noexcept2(""" <<\EOF
  56                                         echo "{"
  57                                         echo '  "kernel":"'`uname -a`'",'
  58                                         echo '  "bmlog":"'`ls /tmp/bm.log`'",'
  59                                         echo '  "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",'
  60                                         echo '  "nm":"'`ps ax | grep nm.py | grep -v grep`'",'
  61                                         echo '  "readonlyfs":"'`touch /var/log/monitor 2>&1`'",'
  62                                         echo '  "dns":"'`host boot.planet-lab.org 2>&1`'",'
  63                                         echo '  "princeton_comon":"'`ls -d /vservers/princeton_comon`'",'
  64
  65                                         ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'`
  66
  67                                         echo '  "princeton_comon_running":"'`ls -d /proc/virtual/$ID`'",'
  68                                         echo '  "princeton_comon_procs":"'`vps ax | grep $ID | grep -v grep | wc -l`'",'
  69                                         echo "}"
  70 EOF                             """)
  71
  72                                 values['ssherror'] = errval
  73                                 if len(oval) > 0:
  74                                         #print "OVAL: %s" % oval
  75                                         values.update(eval(oval))
  76                                         values['sshport'] = port
  77                                         break
  78                                 else:
  79                                         values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '',
  80                                                                         'nm' : '',
  81                                                                         'readonlyfs' : '',
  82                                                                         'dns' : '',
  83                                                                         'princeton_comon' : "",
  84                                                                         'princeton_comon_running' : "",
  85                                                                         'princeton_comon_procs' : "", 'sshport' : None})
  86                 except:
  87                         print traceback.print_exc()
  88                         sys.exit(1)
  89
  90                 ### RUN SSH ######################
  91                 b_getbootcd_id = True
  92                 #ssh = command.SSH('root', nodename)
  93                 #oval = ""
  94                 #errval = ""
  95                 #(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
  96
  97                 oval = values['kernel']
  98                 if "2.6.17" in oval or "2.6.2" in oval:
  99                         values['ssh'] = 'SSH'
 100                         values['category'] = 'ALPHA'
 101                         if "bm.log" in values['bmlog']:
 102                                 values['state'] = 'DEBUG'
 103                         else:
 104                                 values['state'] = 'BOOT'
 105                 elif "2.6.12" in oval or "2.6.10" in oval:
 106                         values['ssh'] = 'SSH'
 107                         values['category'] = 'PROD'
 108                         if "bm.log" in values['bmlog']:
 109                                 values['state'] = 'DEBUG'
 110                         else:
 111                                 values['state'] = 'BOOT'
 112
 113                 # NOTE: on 2.6.8 kernels, with 4.2 bootstrapfs, the chroot command fails.  I have no idea why.
 114                 elif "2.4" in oval or "2.6.8" in oval:
 115                         b_getbootcd_id = False
 116                         values['ssh'] = 'SSH'
 117                         values['category'] = 'OLDBOOTCD'
 118                         values['state'] = 'DEBUG'
 119                 elif oval != "":
 120                         values['ssh'] = 'SSH'
 121                         values['category'] = 'UNKNOWN'
 122                         if "bm.log" in values['bmlog']:
 123                                 values['state'] = 'DEBUG'
 124                         else:
 125                                 values['state'] = 'BOOT'
 126                 else:
 127                         # An error occurred.
 128                         b_getbootcd_id = False
 129                         values['ssh'] = 'NOSSH'
 130                         values['category'] = 'ERROR'
 131                         values['state'] = 'DOWN'
 132                         val = errval.strip()
 133                         values['ssherror'] = val
 134                         values['kernel'] = ""
 135
 136                 #values['kernel'] = val
 137
 138                 if b_getbootcd_id:
 139                         # try to get BootCD for all nodes that are not 2.4 nor inaccessible
 140                         #(oval, errval) = ssh.run_noexcept('cat /mnt/cdrom/bootme/ID')
 141                         oval = values['bootcd']
 142                         if "BootCD" in oval:
 143                                 values['bootcd'] = oval
 144                                 if "v2" in oval and \
 145                                         ( nodename is not "planetlab1.cs.unc.edu" and \
 146                                           nodename is not "planetlab2.cs.unc.edu" ):
 147                                         values['category'] = 'OLDBOOTCD'
 148                         else:
 149                                 values['bootcd'] = ""
 150                 else:
 151                         values['bootcd'] = ""
 152
 153                 # TODO: get bm.log for debug nodes.
 154                 # 'zcat /tmp/bm.log'
 155
 156                 #(oval, errval) = ssh.run_noexcept('ps ax | grep nm.py | grep -v grep')
 157                 oval = values['nm']
 158                 if "nm.py" in oval:
 159                         values['nm'] = "Y"
 160                 else:
 161                         values['nm'] = "N"
 162
 163                 continue_slice_check = True
 164                 #(oval, errval) = ssh.run_noexcept('ls -d /vservers/princeton_comon')
 165                 oval = values['princeton_comon']
 166                 if "princeton_comon" in oval:
 167                         values['princeton_comon'] = True
 168                 else:
 169                         values['princeton_comon'] = False
 170                         continue_slice_check = False
 171
 172                 if continue_slice_check:
 173                         #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; ls -d /proc/virtual/$ID')
 174                         oval = values['princeton_comon_running']
 175                         if len(oval) > len('/proc/virtual/'):
 176                                 values['princeton_comon_running'] = True
 177                         else:
 178                                 values['princeton_comon_running'] = False
 179                                 continue_slice_check = False
 180                 else:
 181                         values['princeton_comon_running'] = False
 182
 183                 if continue_slice_check:
 184                         #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; vps ax | grep $ID | grep -v grep | wc -l')
 185                         oval = values['princeton_comon_procs']
 186                         values['princeton_comon_procs'] = int(oval)
 187                 else:
 188                         values['princeton_comon_procs'] = None
 189
 190
 191                 if nodename in cohash:
 192                         values['comonstats'] = cohash[nodename]
 193                 else:
 194                         values['comonstats'] = {'resptime':  '-1',
 195                                                                         'uptime':    '-1',
 196                                                                         'sshstatus': '-1',
 197                                                                         'lastcotop': '-1',
 198                                                                         'cpuspeed' : "null",
 199                                                                         'disksize' : 'null',
 200                                                                         'memsize'  : 'null'}
 201                 # include output value
 202                 ### GET PLC NODE ######################
 203                 plc_lock.acquire()
 204                 d_node = None
 205                 try:
 206                         d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'date_created',
 207                                                                         'last_updated', 'last_contact', 'boot_state', 'nodegroup_ids'])[0]
 208                 except:
 209                         traceback.print_exc()
 210                 plc_lock.release()
 211                 values['plcnode'] = d_node
 212
 213                 ### GET PLC PCU ######################
 214                 site_id = -1
 215                 d_pcu = None
 216                 if d_node:
 217                         pcu = d_node['pcu_ids']
 218                         if len(pcu) > 0:
 219                                 d_pcu = pcu[0]
 220
 221                         site_id = d_node['site_id']
 222
 223                 values['pcu'] = d_pcu
 224
 225                 ### GET PLC SITE ######################
 226                 plc_lock.acquire()
 227                 d_site = None
 228                 values['loginbase'] = ""
 229                 try:
 230                         d_site = plc.getSites({'site_id': site_id},
 231                                                                 ['max_slices', 'slice_ids', 'node_ids', 'login_base'])[0]
 232                         values['loginbase'] = d_site['login_base']
 233                 except:
 234                         traceback.print_exc()
 235                 plc_lock.release()
 236
 237                 values['plcsite'] = d_site
 238                 values['date_checked'] = time.time()
 239         except:
 240                 print traceback.print_exc()
 241
 242         return (nodename, values)
 243
 244 def recordPingAndSSH(request, result):
 245         global global_round
 246         global count
 247         (nodename, values) = result
 248
 249         try:
 250                 if values is not None:
 251                         fbsync = FindbadNodeRecordSync.findby_or_create(hostname="global",
 252                                                                                                                         if_new_set={'round' : global_round})
 253                         global_round = fbsync.round
 254                         fbnodesync = FindbadNodeRecordSync.findby_or_create(hostname=nodename,
 255                                                                                                                         if_new_set={'round' : global_round})
 256
 257                         fbrec = FindbadNodeRecord(
 258                                                 date_checked=datetime.fromtimestamp(values['date_checked']),
 259                                                 hostname=nodename,
 260                                                 loginbase=values['loginbase'],
 261                                                 kernel_version=values['kernel'],
 262                                                 bootcd_version=values['bootcd'],
 263                                                 nm_status=values['nm'],
 264                                                 fs_status=values['readonlyfs'],
 265                                                 dns_status=values['dns'],
 266                                                 princeton_comon_dir=values['princeton_comon'],
 267                                                 princeton_comon_running=values['princeton_comon_running'],
 268                                                 princeton_comon_procs=values['princeton_comon_procs'],
 269                                                 plc_node_stats = values['plcnode'],
 270                                                 plc_site_stats = values['plcsite'],
 271                                                 plc_pcuid = values['pcu'],
 272                                                 comon_stats = values['comonstats'],
 273                                                 ping_status = (values['ping'] == "PING"),
 274                                                 ssh_portused = values['sshport'],
 275                                                 ssh_status = (values['ssh'] == "SSH"),
 276                                                 ssh_error = values['ssherror'],
 277                                                 observed_status = values['state'],
 278                                         )
 279                         fbnodesync.round = global_round
 280
 281                         count += 1
 282                         print "%d %s %s" % (count, nodename, values)
 283         except:
 284                 print "ERROR:"
 285                 print traceback.print_exc()
 286
 287 # this will be called when an exception occurs within a thread
 288 def handle_exception(request, result):
 289         print "Exception occured in request %s" % request.requestID
 290         for i in result:
 291                 print "Result: %s" % i
 292
 293
 294 def checkAndRecordState(l_nodes, cohash):
 295         global global_round
 296         global count
 297
 298         tp = threadpool.ThreadPool(20)
 299
 300         # CREATE all the work requests
 301         for nodename in l_nodes:
 302                 fbnodesync = FindbadNodeRecordSync.findby_or_create(hostname=nodename, if_new_set={'round':0})
 303
 304                 node_round   = fbnodesync.round
 305                 if node_round < global_round:
 306                         # recreate node stats when refreshed
 307                         #print "%s" % nodename
 308                         req = threadpool.WorkRequest(collectPingAndSSH, [nodename, cohash], {},
 309                                                                                  None, recordPingAndSSH, handle_exception)
 310                         tp.putRequest(req)
 311                 else:
 312                         # We just skip it, since it's "up to date"
 313                         count += 1
 314                         #print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 315                         print "%d %s %s" % (count, nodename, node_round)
 316
 317         # WAIT while all the work requests are processed.
 318         begin = time.time()
 319         while 1:
 320                 try:
 321                         time.sleep(1)
 322                         tp.poll()
 323                         # if more than two hours
 324                         if time.time() - begin > (60*60*1.5):
 325                                 print "findbad.py has run out of time!!!!!!"
 326                                 os._exit(1)
 327                 except KeyboardInterrupt:
 328                         print "Interrupted!"
 329                         break
 330                 except threadpool.NoResultsPending:
 331                         print "All results collected."
 332                         break
 333
 334         print FindbadNodeRecordSync.query.count()
 335         print FindbadNodeRecord.query.count()
 336
 337 def main():
 338         global global_round
 339
 340         fbsync = FindbadNodeRecordSync.findby_or_create(hostname="global",
 341                                                                                                         if_new_set={'round' : global_round})
 342         global_round = fbsync.round
 343
 344         if config.increment:
 345                 # update global round number to force refreshes across all nodes
 346                 global_round += 1
 347                 fbsync.round = global_round
 348
 349         cotop = comon.Comon()
 350         # lastcotop measures whether cotop is actually running.  this is a better
 351         # metric than sshstatus, or other values from CoMon
 352         cotop_url = COMON_COTOPURL
 353
 354         # history information for all nodes
 355         #cohash = {}
 356         cohash = cotop.coget(cotop_url)
 357         l_nodes = syncplcdb.create_plcdb()
 358         if config.nodelist:
 359                 f_nodes = util.file.getListFromFile(config.nodelist)
 360                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 361         elif config.node:
 362                 f_nodes = [config.node]
 363                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 364         elif config.nodegroup:
 365                 ng = api.GetNodeGroups({'name' : config.nodegroup})
 366                 l_nodes = api.GetNodes(ng[0]['node_ids'])
 367         elif config.site:
 368                 site = api.GetSites(config.site)
 369                 l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
 370
 371         l_nodes = [node['hostname'] for node in l_nodes]
 372
 373         # perform this query after the above options, so that the filter above
 374         # does not break.
 375         if config.nodeselect:
 376                 plcnodes = api.GetNodes({'peer_id' : None}, ['hostname'])
 377                 plcnodes = [ node['hostname'] for node in plcnodes ]
 378                 l_nodes = node_select(config.nodeselect, plcnodes, None)
 379
 380         print "fetching %s hosts" % len(l_nodes)
 381
 382         checkAndRecordState(l_nodes, cohash)
 383
 384         return 0
 385
 386
 387 if __name__ == '__main__':
 388         from monitor import parser as parsermodule
 389
 390         parser = parsermodule.getParser(['nodesets'])
 391
 392         parser.set_defaults( increment=False, dbname="findbad", cachenodes=False)
 393         parser.add_option("", "--cachenodes", action="store_true",
 394                                                 help="Cache node lookup from PLC")
 395         parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
 396                                                 help="Specify the name of the database to which the information is saved")
 397         parser.add_option("-i", "--increment", action="store_true", dest="increment",
 398                                                 help="Increment round number to force refresh or retry")
 399
 400         parser = parsermodule.getParser(['defaults'], parser)
 401
 402         cfg = parsermodule.parse_args(parser)
 403
 404         try:
 405                 main()
 406         except Exception, err:
 407                 print traceback.print_exc()
 408                 print "Exception: %s" % err
 409                 print "Saving data... exitting."
 410                 sys.exit(0)
 411         print "sleeping"
 412         #print "final commit"
 413         #time.sleep(10)