monitor/scanapi.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 from datetime import datetime,timedelta
   8 import threadpool
   9 import threading
  10
  11 import socket
  12 from pcucontrol import reboot
  13
  14 from pcucontrol.util import command
  15 from monitor import config
  16
  17 from monitor.database.info.model import *
  18
  19 from monitor.sources import comon
  20 from monitor.wrapper import plc, plccache
  21
  22 import traceback
  23 from monitor.common import nmap_port_status
  24
  25 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
  26                         "table=table_nodeview&" + \
  27                         "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
  28                         "formatcsv"
  29
  30 api = plc.getAuthAPI()
  31 plc_lock = threading.Lock()
  32 round = 1
  33 global_round = round
  34 count = 0
  35
  36
  37 def get_pcu(pcuname):
  38         plc_lock.acquire()
  39         try:
  40                 #print "GetPCU from PLC %s" % pcuname
  41                 l_pcu  = plc.GetPCUs({'pcu_id' : pcuname})
  42                 #print l_pcu
  43                 if len(l_pcu) > 0:
  44                         l_pcu = l_pcu[0]
  45         except:
  46                 try:
  47                         #print "GetPCU from file %s" % pcuname
  48                         l_pcus = plccache.l_pcus
  49                         for i in l_pcus:
  50                                 if i['pcu_id'] == pcuname:
  51                                         l_pcu = i
  52                 except:
  53                         traceback.print_exc()
  54                         l_pcu = None
  55
  56         plc_lock.release()
  57         return l_pcu
  58
  59 def get_nodes(node_ids):
  60         plc_lock.acquire()
  61         l_node = []
  62         try:
  63                 l_node = plc.getNodes(node_ids, ['hostname', 'last_contact', 'node_id', 'ports'])
  64         except:
  65                 try:
  66                         plc_nodes = plccache.l_plcnodes
  67                         for n in plc_nodes:
  68                                 if n['node_id'] in node_ids:
  69                                         l_node.append(n)
  70                 except:
  71                         traceback.print_exc()
  72                         l_node = None
  73
  74         plc_lock.release()
  75         if l_node == []:
  76                 l_node = None
  77         return l_node
  78
  79
  80 def get_plc_pcu_values(pcuname):
  81         """
  82                 Try to contact PLC to get the PCU info.
  83                 If that fails, try a backup copy from the last run.
  84                 If that fails, return None
  85         """
  86         values = {}
  87
  88         l_pcu = get_pcu(pcuname)
  89
  90         if l_pcu is not None:
  91                 site_id = l_pcu['site_id']
  92                 node_ids = l_pcu['node_ids']
  93                 l_node = get_nodes(node_ids)
  94
  95                 if l_node is not None:
  96                         for node in l_node:
  97                                 values[node['hostname']] = node['ports'][0]
  98
  99                         values['nodenames'] = [node['hostname'] for node in l_node]
 100
 101                         # NOTE: this is for a dry run later. It doesn't matter which node.
 102                         values['node_id'] = l_node[0]['node_id']
 103
 104                 values.update(l_pcu)
 105         else:
 106                 values = None
 107
 108         return values
 109
 110 class ScanInterface(object):
 111         recordclass = None
 112         syncclass = None
 113         primarykey = 'hostname'
 114
 115         def __init__(self, round):
 116                 self.round = round
 117                 self.count = 1
 118
 119         def __getattr__(self, name):
 120                 if 'collect' in name or 'record' in name:
 121                         method = getattr(self, name, None)
 122                         if method is None:
 123                                 raise Exception("No such method %s" % name)
 124                         return method
 125                 else:
 126                         raise Exception("No such method %s" % name)
 127
 128         def collect(self, nodename, data):
 129                 pass
 130
 131         def record(self, request, (nodename, values) ):
 132
 133                 try:
 134                         if values is None:
 135                                 return
 136
 137                         fbnodesync = self.syncclass.findby_or_create(
 138                                                                                                 if_new_set={'round' : self.round},
 139                                                                                                 **{ self.primarykey : nodename})
 140                         # NOTE: This code will either add a new record for the new self.round,
 141                         #       OR it will find the previous value, and update it with new information.
 142                         #       The data that is 'lost' is not that important, b/c older
 143                         #       history still exists.
 144                         fbrec = self.recordclass.findby_or_create(
 145                                                 **{'round':self.round, self.primarykey:nodename})
 146
 147                         fbrec.set( **values )
 148
 149                         fbrec.flush()
 150                         fbnodesync.round = self.round
 151                         fbnodesync.flush()
 152
 153                         print "%d %s %s" % (self.count, nodename, values)
 154                         self.count += 1
 155
 156                 except:
 157                         print "ERROR:"
 158                         print traceback.print_exc()
 159                         pass
 160
 161 class ScanNodeInternal(ScanInterface):
 162         recordclass = FindbadNodeRecord
 163         syncclass = FindbadNodeRecordSync
 164         primarykey = 'hostname'
 165
 166         def collectNMAP(self, nodename, cohash):
 167                 #### RUN NMAP ###############################
 168                 values = {}
 169                 nmap = command.CMD()
 170                 print "nmap -oG - -P0 -p22,80,806 %s | grep Host:" % nodename
 171                 (oval,eval) = nmap.run_noexcept("nmap -oG - -P0 -p22,80,806 %s | grep Host:" % nodename)
 172                 # NOTE: an empty / error value for oval, will still work.
 173                 (values['port_status'], continue_probe) = nmap_port_status(oval)
 174
 175                 values['date_checked'] = datetime.now()
 176
 177                 return (nodename, values)
 178
 179         def collectInternal(self, nodename, cohash):
 180                 ### RUN PING ######################
 181                 ping = command.CMD()
 182                 (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
 183
 184                 try:
 185                         values = {}
 186
 187                         if oval == "":
 188                                 # An error occurred
 189                                 values['ping_status'] = False
 190                         else:
 191                                 values['ping_status'] = True
 192
 193                         try:
 194                                 for port in [22, 806]:
 195                                         ssh = command.SSH('root', nodename, port)
 196
 197                                         (oval, errval) = ssh.run_noexcept2(""" <<\EOF
 198                                                 echo "{"
 199                                                 echo '  "kernel_version":"'`uname -a`'",'
 200                                                 echo '  "bmlog":"'`ls /tmp/bm.log`'",'
 201                                                 echo '  "bootcd_version":"'`cat /mnt/cdrom/bootme/ID`'",'
 202                                                 echo '  "nm_status":"'`ps ax | grep nm.py | grep -v grep`'",'
 203                                                 echo '  "fs_status":"'`touch /var/log/monitor 2>&1`'",'
 204                                                 echo '  "dns_status":"'`host boot.planet-lab.org 2>&1`'",'
 205                                                 echo '  "princeton_comon_dir":"'`ls -d /vservers/princeton_comon`'",'
 206
 207                                                 ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'`
 208                                                 echo '  "princeton_comon_running":"'`ls -d /proc/virtual/$ID`'",'
 209                                                 echo '  "princeton_comon_procs":"'`vps ax | grep $ID | grep -v grep | wc -l`'",'
 210                                                 echo "}"
 211         EOF                             """)
 212
 213                                         values['ssh_error'] = errval
 214                                         if len(oval) > 0:
 215                                                 #print "OVAL: %s" % oval
 216                                                 values.update(eval(oval))
 217                                                 values['ssh_portused'] = port
 218                                                 break
 219                                         else:
 220                                                 values.update({'kernel_version': "", 'bmlog' : "", 'bootcd_version' : '',
 221                                                                                 'nm_status' : '',
 222                                                                                 'fs_status' : '',
 223                                                                                 'dns_status' : '',
 224                                                                                 'princeton_comon_dir' : "",
 225                                                                                 'princeton_comon_running' : "",
 226                                                                                 'princeton_comon_procs' : "", 'ssh_portused' : None})
 227                         except:
 228                                 print traceback.print_exc()
 229                                 sys.exit(1)
 230
 231                         ### RUN SSH ######################
 232                         b_getbootcd_id = True
 233
 234                         oval = values['kernel_version']
 235                         if "2.6.17" in oval or "2.6.2" in oval:
 236                                 values['ssh_status'] = True
 237                                 values['observed_category'] = 'PROD'
 238                                 if "bm.log" in values['bmlog']:
 239                                         values['observed_status'] = 'DEBUG'
 240                                 else:
 241                                         values['observed_status'] = 'BOOT'
 242                         elif "2.6.12" in oval or "2.6.10" in oval:
 243                                 values['ssh_status'] = True
 244                                 values['observed_category'] = 'OLDPROD'
 245                                 if "bm.log" in values['bmlog']:
 246                                         values['observed_status'] = 'DEBUG'
 247                                 else:
 248                                         values['observed_status'] = 'BOOT'
 249
 250                         # NOTE: on 2.6.8 kernels, with 4.2 bootstrapfs, the chroot
 251                         #       command fails.  I have no idea why.
 252                         elif "2.4" in oval or "2.6.8" in oval:
 253                                 b_getbootcd_id = False
 254                                 values['ssh_status'] = True
 255                                 values['observed_category'] = 'OLDBOOTCD'
 256                                 values['observed_status'] = 'DEBUG'
 257                         elif oval != "":
 258                                 values['ssh_status'] = True
 259                                 values['observed_category'] = 'UNKNOWN'
 260                                 if "bm.log" in values['bmlog']:
 261                                         values['observed_status'] = 'DEBUG'
 262                                 else:
 263                                         values['observed_status'] = 'BOOT'
 264                         else:
 265                                 # An error occurred.
 266                                 b_getbootcd_id = False
 267                                 values['ssh_status'] = False
 268                                 values['observed_category'] = 'ERROR'
 269                                 values['observed_status'] = 'DOWN'
 270                                 val = errval.strip()
 271                                 values['ssh_error'] = val
 272                                 values['kernel_version'] = ""
 273
 274                         if b_getbootcd_id:
 275                                 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
 276                                 oval = values['bootcd_version']
 277                                 if "BootCD" in oval:
 278                                         values['bootcd_version'] = oval
 279                                         if "v2" in oval and \
 280                                                 ( nodename is not "planetlab1.cs.unc.edu" and \
 281                                                   nodename is not "planetlab2.cs.unc.edu" ):
 282                                                 values['observed_category'] = 'OLDBOOTCD'
 283                                 else:
 284                                         values['bootcd_version'] = ""
 285                         else:
 286                                 values['bootcd_version'] = ""
 287
 288                         oval = values['nm_status']
 289                         if "nm.py" in oval:
 290                                 values['nm_status'] = "Y"
 291                         else:
 292                                 values['nm_status'] = "N"
 293
 294                         continue_slice_check = True
 295                         oval = values['princeton_comon_dir']
 296                         if "princeton_comon_dir" in oval:
 297                                 values['princeton_comon_dir'] = True
 298                         else:
 299                                 values['princeton_comon_dir'] = False
 300                                 continue_slice_check = False
 301
 302                         if continue_slice_check:
 303                                 oval = values['princeton_comon_running']
 304                                 if len(oval) > len('/proc/virtual/'):
 305                                         values['princeton_comon_running'] = True
 306                                 else:
 307                                         values['princeton_comon_running'] = False
 308                                         continue_slice_check = False
 309                         else:
 310                                 values['princeton_comon_running'] = False
 311
 312                         if continue_slice_check:
 313                                 oval = values['princeton_comon_procs']
 314                                 values['princeton_comon_procs'] = int(oval)
 315                         else:
 316                                 values['princeton_comon_procs'] = None
 317
 318
 319                         if nodename in cohash:
 320                                 values['comon_stats'] = cohash[nodename]
 321                         else:
 322                                 values['comon_stats'] = {'resptime':  '-1',
 323                                                                                 'uptime':    '-1',
 324                                                                                 'sshstatus': '-1',
 325                                                                                 'lastcotop': '-1',
 326                                                                                 'cpuspeed' : "null",
 327                                                                                 'disksize' : 'null',
 328                                                                                 'memsize'  : 'null'}
 329                         # include output value
 330                         ### GET PLC NODE ######################
 331                         plc_lock.acquire()
 332                         d_node = None
 333                         try:
 334                                 d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id',
 335                                                                                 'date_created', 'last_updated',
 336                                                                                 'last_contact', 'boot_state', 'nodegroup_ids'])[0]
 337                         except:
 338                                 traceback.print_exc()
 339                         plc_lock.release()
 340                         values['plc_node_stats'] = d_node
 341
 342                         ##### NMAP  ###################
 343                         (n, v) = self.collectNMAP(nodename, None)
 344                         values.update(v)
 345
 346                         ### GET PLC PCU ######################
 347                         site_id = -1
 348                         d_pcu = None
 349                         if d_node:
 350                                 pcu = d_node['pcu_ids']
 351                                 if len(pcu) > 0:
 352                                         d_pcu = pcu[0]
 353
 354                                 site_id = d_node['site_id']
 355
 356                         values['plc_pcuid'] = d_pcu
 357
 358                         ### GET PLC SITE ######################
 359                         plc_lock.acquire()
 360                         d_site = None
 361                         values['loginbase'] = ""
 362                         try:
 363                                 d_site = plc.getSites({'site_id': site_id},
 364                                                                         ['max_slices', 'slice_ids', 'node_ids', 'login_base'])[0]
 365                                 values['loginbase'] = d_site['login_base']
 366                         except:
 367                                 traceback.print_exc()
 368                         plc_lock.release()
 369
 370                         values['plc_site_stats'] = d_site
 371                         values['date_checked'] = datetime.now()
 372                 except:
 373                         print traceback.print_exc()
 374
 375                 return (nodename, values)
 376
 377 def internalprobe(hostname):
 378         fbsync = FindbadNodeRecordSync.findby_or_create(hostname="global",
 379                                                                                                         if_new_set={'round' : 1})
 380         scannode = ScanNodeInternal(fbsync.round)
 381         try:
 382                 (nodename, values) = scannode.collectInternal(hostname, {})
 383                 scannode.record(None, (nodename, values))
 384                 session.flush()
 385                 return True
 386         except:
 387                 print traceback.print_exc()
 388                 return False
 389
 390 def externalprobe(hostname):
 391         fbsync = FindbadNodeRecordSync.findby_or_create(hostname="global",
 392                                                                                                         if_new_set={'round' : 1})
 393         scannode = ScanNodeInternal(fbsync.round)
 394         try:
 395                 (nodename, values) = scannode.collectNMAP(hostname, {})
 396                 scannode.record(None, (nodename, values))
 397                 session.flush()
 398                 return True
 399         except:
 400                 print traceback.print_exc()
 401                 return False
 402
 403 class ScanPCU(ScanInterface):
 404         recordclass = FindbadPCURecord
 405         syncclass = FindbadPCURecordSync
 406         primarykey = 'plc_pcuid'
 407
 408         def collectInternal(self, pcuname, cohash):
 409
 410                 continue_probe = True
 411                 errors = None
 412                 values = {'reboot_trial_status' : 'novalue'}
 413                 ### GET PCU ######################
 414                 try:
 415                         b_except = False
 416                         try:
 417                                 v = get_plc_pcu_values(pcuname)
 418                                 if v['hostname'] is not None: v['hostname'] = v['hostname'].strip()
 419                                 if v['ip'] is not None: v['ip'] = v['ip'].strip()
 420
 421                                 if v is not None:
 422                                         values['plc_pcu_stats'] = v
 423                                 else:
 424                                         continue_probe = False
 425                         except:
 426                                 b_except = True
 427                                 traceback.print_exc()
 428                                 continue_probe = False
 429
 430                         if b_except or not continue_probe: return (None, None, None)
 431
 432                         #### RUN NMAP ###############################
 433                         if continue_probe:
 434                                 nmap = command.CMD()
 435                                 print "nmap -oG - -P0 -p22,23,80,443,5869,9100,16992 %s | grep Host:" % reboot.pcu_name(values['plc_pcu_stats'])
 436                                 (oval,eval) = nmap.run_noexcept("nmap -oG - -P0 -p22,23,80,443,5869,9100,16992 %s | grep Host:" % reboot.pcu_name(values['plc_pcu_stats']))
 437                                 # NOTE: an empty / error value for oval, will still work.
 438                                 (values['port_status'], continue_probe) = nmap_port_status(oval)
 439                         else:
 440                                 values['port_status'] = None
 441
 442                         #### COMPLETE ENTRY   #######################
 443
 444                         values['entry_complete'] = []
 445                         #if values['protocol'] is None or values['protocol'] is "":
 446                         #       values['entry_complete'] += ["protocol"]
 447                         if values['plc_pcu_stats']['model'] is None or values['plc_pcu_stats']['model'] is "":
 448                                 values['entry_complete'] += ["model"]
 449                                 # Cannot continue due to this condition
 450                                 continue_probe = False
 451
 452                         if values['plc_pcu_stats']['password'] is None or values['plc_pcu_stats']['password'] is "":
 453                                 values['entry_complete'] += ["password"]
 454                                 # Cannot continue due to this condition
 455                                 continue_probe = False
 456
 457                         if len(values['entry_complete']) > 0:
 458                                 continue_probe = False
 459
 460                         if values['plc_pcu_stats']['hostname'] is None or values['plc_pcu_stats']['hostname'] is "":
 461                                 values['entry_complete'] += ["hostname"]
 462                         if values['plc_pcu_stats']['ip'] is None or values['plc_pcu_stats']['ip'] is "":
 463                                 values['entry_complete'] += ["ip"]
 464
 465                         # If there are no nodes associated with this PCU, then we cannot continue.
 466                         if len(values['plc_pcu_stats']['node_ids']) == 0:
 467                                 continue_probe = False
 468                                 values['entry_complete'] += ['nodeids']
 469
 470
 471                         #### DNS and IP MATCH #######################
 472                         if values['plc_pcu_stats']['hostname'] is not None and values['plc_pcu_stats']['hostname'] is not "" and \
 473                            values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
 474                                 try:
 475                                         ipaddr = socket.gethostbyname(values['plc_pcu_stats']['hostname'])
 476                                         if ipaddr == values['plc_pcu_stats']['ip']:
 477                                                 values['dns_status'] = "DNS-OK"
 478                                         else:
 479                                                 values['dns_status'] = "DNS-MISMATCH"
 480                                                 continue_probe = False
 481
 482                                 except Exception, err:
 483                                         values['dns_status'] = "DNS-NOENTRY"
 484                                         values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
 485                         else:
 486                                 if values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
 487                                         values['dns_status'] = "NOHOSTNAME"
 488                                         values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
 489                                 else:
 490                                         values['dns_status'] = "NO-DNS-OR-IP"
 491                                         values['plc_pcu_stats']['hostname'] = "No_entry_in_DB"
 492                                         continue_probe = False
 493
 494
 495                         ######  DRY RUN  ############################
 496                         if continue_probe and 'node_ids' in values['plc_pcu_stats'] and \
 497                                 len(values['plc_pcu_stats']['node_ids']) > 0:
 498                                 rb_ret = reboot.reboot_test_new(values['plc_pcu_stats']['nodenames'][0],
 499                                                                                                 values, 1, True)
 500                         else:
 501                                 rb_ret = "Not_Run" # No nodes to test"
 502
 503                         values['reboot_trial_status'] = rb_ret
 504
 505                 except:
 506                         print "____________________________________"
 507                         print values
 508                         errors = values
 509                         print "____________________________________"
 510                         errors['traceback'] = traceback.format_exc()
 511                         print errors['traceback']
 512                         values['reboot_trial_status'] = str(errors['traceback'])
 513                         print values
 514
 515                 values['entry_complete']=" ".join(values['entry_complete'])
 516
 517                 values['date_checked'] = datetime.now()
 518                 return (pcuname, values)
 519