findbadpcu.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import socket
   8 import sets
   9 import signal
  10 import traceback
  11 from datetime import datetime,timedelta
  12 import threadpool
  13 import threading
  14
  15 import monitor
  16 from monitor.pcu import reboot
  17 from monitor import config
  18 from monitor.database import FindbadPCURecordSync, FindbadPCURecord
  19 from monitor import util
  20 from monitor.wrapper import plc, plccache
  21 from nodequery import pcu_select
  22
  23 plc_lock = threading.Lock()
  24 global_round = 1
  25 errorState = {}
  26 count = 0
  27
  28 def nmap_portstatus(status):
  29         ps = {}
  30         l_nmap = status.split()
  31         ports = l_nmap[4:]
  32
  33         continue_probe = False
  34         for port in ports:
  35                 results = port.split('/')
  36                 ps[results[0]] = results[1]
  37                 if results[1] == "open":
  38                         continue_probe = True
  39         return (ps, continue_probe)
  40
  41 def get_pcu(pcuname):
  42         plc_lock.acquire()
  43         try:
  44                 #print "GetPCU from PLC %s" % pcuname
  45                 l_pcu  = plc.GetPCUs({'pcu_id' : pcuname})
  46                 #print l_pcu
  47                 if len(l_pcu) > 0:
  48                         l_pcu = l_pcu[0]
  49         except:
  50                 try:
  51                         #print "GetPCU from file %s" % pcuname
  52                         l_pcus = plccache.l_pcus
  53                         for i in l_pcus:
  54                                 if i['pcu_id'] == pcuname:
  55                                         l_pcu = i
  56                 except:
  57                         traceback.print_exc()
  58                         l_pcu = None
  59
  60         plc_lock.release()
  61         return l_pcu
  62
  63 def get_nodes(node_ids):
  64         plc_lock.acquire()
  65         l_node = []
  66         try:
  67                 l_node = plc.getNodes(node_ids, ['hostname', 'last_contact', 'node_id', 'ports'])
  68         except:
  69                 try:
  70                         plc_nodes = plccache.l_plcnodes
  71                         for n in plc_nodes:
  72                                 if n['node_id'] in node_ids:
  73                                         l_node.append(n)
  74                 except:
  75                         traceback.print_exc()
  76                         l_node = None
  77
  78         plc_lock.release()
  79         if l_node == []:
  80                 l_node = None
  81         return l_node
  82
  83
  84 def get_plc_pcu_values(pcuname):
  85         """
  86                 Try to contact PLC to get the PCU info.
  87                 If that fails, try a backup copy from the last run.
  88                 If that fails, return None
  89         """
  90         values = {}
  91
  92         l_pcu = get_pcu(pcuname)
  93
  94         if l_pcu is not None:
  95                 site_id = l_pcu['site_id']
  96                 node_ids = l_pcu['node_ids']
  97                 l_node = get_nodes(node_ids)
  98
  99                 if l_node is not None:
 100                         for node in l_node:
 101                                 values[node['hostname']] = node['ports'][0]
 102
 103                         values['nodenames'] = [node['hostname'] for node in l_node]
 104
 105                         # NOTE: this is for a dry run later. It doesn't matter which node.
 106                         values['node_id'] = l_node[0]['node_id']
 107
 108                 values.update(l_pcu)
 109         else:
 110                 values = None
 111
 112         return values
 113
 114 def get_plc_site_values(site_id):
 115         ### GET PLC SITE ######################
 116         plc_lock.acquire()
 117         values = {}
 118         d_site = None
 119
 120         try:
 121                 d_site = plc.getSites({'site_id': site_id}, ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
 122                 if len(d_site) > 0:
 123                         d_site = d_site[0]
 124         except:
 125                 try:
 126                         plc_sites = plccache.l_plcsites
 127                         for site in plc_sites:
 128                                 if site['site_id'] == site_id:
 129                                         d_site = site
 130                                         break
 131                 except:
 132                         traceback.print_exc()
 133                         values = None
 134
 135         plc_lock.release()
 136
 137         if d_site is not None:
 138                 max_slices = d_site['max_slices']
 139                 num_slices = len(d_site['slice_ids'])
 140                 num_nodes = len(d_site['node_ids'])
 141                 loginbase = d_site['login_base']
 142                 values['plcsite'] = {'num_nodes' : num_nodes,
 143                                                         'max_slices' : max_slices,
 144                                                         'num_slices' : num_slices,
 145                                                         'login_base' : loginbase,
 146                                                         'status'     : 'SUCCESS'}
 147         else:
 148                 values = None
 149
 150
 151         return values
 152
 153
 154 def collectPingAndSSH(pcuname, cohash):
 155
 156         continue_probe = True
 157         errors = None
 158         values = {'reboot' : 'novalue'}
 159         ### GET PCU ######################
 160         try:
 161                 b_except = False
 162                 try:
 163                         v = get_plc_pcu_values(pcuname)
 164                         if v['hostname'] is not None: v['hostname'] = v['hostname'].strip()
 165                         if v['ip'] is not None: v['ip'] = v['ip'].strip()
 166
 167                         if v is not None:
 168                                 values['plc_pcu_stats'] = v
 169                         else:
 170                                 continue_probe = False
 171                 except:
 172                         b_except = True
 173                         traceback.print_exc()
 174                         continue_probe = False
 175
 176                 if b_except or not continue_probe: return (None, None, None)
 177
 178
 179                 #### COMPLETE ENTRY   #######################
 180
 181                 values['complete_entry'] = []
 182                 #if values['protocol'] is None or values['protocol'] is "":
 183                 #       values['complete_entry'] += ["protocol"]
 184                 if values['plc_pcu_stats']['model'] is None or values['plc_pcu_stats']['model'] is "":
 185                         values['complete_entry'] += ["model"]
 186                         # Cannot continue due to this condition
 187                         continue_probe = False
 188
 189                 if values['plc_pcu_stats']['password'] is None or values['plc_pcu_stats']['password'] is "":
 190                         values['complete_entry'] += ["password"]
 191                         # Cannot continue due to this condition
 192                         continue_probe = False
 193
 194                 if len(values['complete_entry']) > 0:
 195                         continue_probe = False
 196
 197                 if values['plc_pcu_stats']['hostname'] is None or values['plc_pcu_stats']['hostname'] is "":
 198                         values['complete_entry'] += ["hostname"]
 199                 if values['plc_pcu_stats']['ip'] is None or values['plc_pcu_stats']['ip'] is "":
 200                         values['complete_entry'] += ["ip"]
 201
 202                 # If there are no nodes associated with this PCU, then we cannot continue.
 203                 if len(values['plc_pcu_stats']['node_ids']) == 0:
 204                         continue_probe = False
 205                         values['complete_entry'] += ['NoNodeIds']
 206
 207                 #### DNS and IP MATCH #######################
 208                 if values['plc_pcu_stats']['hostname'] is not None and values['plc_pcu_stats']['hostname'] is not "" and \
 209                    values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
 210                         #print "Calling socket.gethostbyname(%s)" % values['hostname']
 211                         try:
 212                                 ipaddr = socket.gethostbyname(values['plc_pcu_stats']['hostname'])
 213                                 if ipaddr == values['plc_pcu_stats']['ip']:
 214                                         values['dnsmatch'] = "DNS-OK"
 215                                 else:
 216                                         values['dnsmatch'] = "DNS-MISMATCH"
 217                                         continue_probe = False
 218
 219                         except Exception, err:
 220                                 values['dnsmatch'] = "DNS-NOENTRY"
 221                                 values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
 222                                 #print err
 223                 else:
 224                         if values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
 225                                 values['dnsmatch'] = "NOHOSTNAME"
 226                                 values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
 227                         else:
 228                                 values['dnsmatch'] = "NO-DNS-OR-IP"
 229                                 values['plc_pcu_stats']['hostname'] = "No_entry_in_DB"
 230                                 continue_probe = False
 231
 232                 #### RUN NMAP ###############################
 233                 if continue_probe:
 234                         nmap = util.command.CMD()
 235                         (oval,eval) = nmap.run_noexcept("nmap -oG - -P0 -p22,23,80,443,5869,9100,16992 %s | grep Host:" % reboot.pcu_name(values['plc_pcu_stats']))
 236                         # NOTE: an empty / error value for oval, will still work.
 237                         (values['portstatus'], continue_probe) = nmap_portstatus(oval)
 238                 else:
 239                         values['portstatus'] = None
 240
 241
 242                 ######  DRY RUN  ############################
 243                 if 'node_ids' in values['plc_pcu_stats'] and len(values['plc_pcu_stats']['node_ids']) > 0:
 244                         rb_ret = reboot.reboot_test(values['plc_pcu_stats']['nodenames'][0], values, continue_probe, 1, True)
 245                 else:
 246                         rb_ret = "Not_Run" # No nodes to test"
 247
 248                 values['reboot'] = rb_ret
 249
 250         except:
 251                 print "____________________________________"
 252                 print values
 253                 errors = values
 254                 print "____________________________________"
 255                 errors['traceback'] = traceback.format_exc()
 256                 print errors['traceback']
 257
 258         values['date_checked'] = time.time()
 259         return (pcuname, values, errors)
 260
 261 def recordPingAndSSH(request, result):
 262         global errorState
 263         global count
 264         global global_round
 265         (nodename, values, errors) = result
 266
 267         if values is not None:
 268                 pcu_id = int(nodename)
 269                 fbsync = FindbadPCURecordSync.findby_or_create(plc_pcuid=0,
 270                                                                                         if_new_set={'round': global_round})
 271                 global_round = fbsync.round
 272                 fbnodesync = FindbadPCURecordSync.findby_or_create(plc_pcuid=pcu_id,
 273                                                                                         if_new_set={'round' : global_round})
 274
 275                 fbrec = FindbadPCURecord(
 276                                         date_checked=datetime.fromtimestamp(values['date_checked']),
 277                                         record=fbsync.round,
 278                                         plc_pcuid=pcu_id,
 279                                         plc_pcu_stats=values['plc_pcu_stats'],
 280                                         dns_status=values['dnsmatch'],
 281                                         port_status=values['portstatus'],
 282                                         entry_complete=" ".join(values['complete_entry']),
 283                                         reboot_trial_status="%s" % values['reboot'],
 284                                 )
 285                 fbnodesync.round = global_round
 286                 count += 1
 287                 print "%d %s %s" % (count, nodename, values)
 288
 289         if errors is not None:
 290                 pcu_id = "id_%s" % nodename
 291                 errorState[pcu_id] = errors
 292                 database.dbDump("findbadpcu_errors", errorState)
 293
 294 # this will be called when an exception occurs within a thread
 295 def handle_exception(request, result):
 296         print "Exception occured in request %s" % request.requestID
 297         for i in result:
 298                 print "Result: %s" % i
 299
 300
 301 def checkAndRecordState(l_pcus, cohash):
 302         global global_round
 303         global count
 304
 305         tp = threadpool.ThreadPool(10)
 306
 307         # CREATE all the work requests
 308         for pcuname in l_pcus:
 309                 pcu_id = int(pcuname)
 310                 fbnodesync = FindbadPCURecordSync.findby_or_create(plc_pcuid=pcu_id, if_new_set={'round' : 0})
 311
 312                 node_round   = fbnodesync.round
 313                 if node_round < global_round:
 314                         # recreate node stats when refreshed
 315                         #print "%s" % nodename
 316                         req = threadpool.WorkRequest(collectPingAndSSH, [pcuname, cohash], {},
 317                                                                                  None, recordPingAndSSH, handle_exception)
 318                         tp.putRequest(req)
 319                 else:
 320                         # We just skip it, since it's "up to date"
 321                         count += 1
 322                         print "%d %s %s" % (count, pcu_id, node_round)
 323
 324         # WAIT while all the work requests are processed.
 325         begin = time.time()
 326         while 1:
 327                 try:
 328                         time.sleep(1)
 329                         tp.poll()
 330                         # if more than two hours
 331                         if time.time() - begin > (60*60*1):
 332                                 print "findbadpcus.py has run out of time!!!!!!"
 333                                 os._exit(1)
 334                 except KeyboardInterrupt:
 335                         print "Interrupted!"
 336                         break
 337                 except threadpool.NoResultsPending:
 338                         print "All results collected."
 339                         break
 340
 341         print FindbadPCURecordSync.query.count()
 342         print FindbadPCURecord.query.count()
 343
 344
 345 def main():
 346         global global_round
 347
 348         #  monitor.database.if_cached_else_refresh(1, config.refresh, "pculist", lambda : plc.GetPCUs())
 349         l_pcus = plccache.l_pcus
 350         cohash = {}
 351
 352         fbsync = FindbadPCURecordSync.findby_or_create(plc_pcuid=0, if_new_set={'round' : global_round})
 353
 354         global_round = fbsync.round
 355
 356         if config.increment:
 357                 # update global round number to force refreshes across all nodes
 358                 global_round += 1
 359                 fbsync.round = global_round
 360
 361         if config.site is not None:
 362                 api = plc.getAuthAPI()
 363                 site = api.GetSites(config.site)
 364                 l_nodes = api.GetNodes(site[0]['node_ids'], ['pcu_ids'])
 365                 pcus = []
 366                 for node in l_nodes:
 367                         pcus += node['pcu_ids']
 368                 # clear out dups.
 369                 l_pcus = [pcu for pcu in sets.Set(pcus)]
 370         elif config.pcuselect is not None:
 371                 n, pcus = pcu_select(config.pcuselect)
 372                 # clear out dups.
 373                 l_pcus = [pcu for pcu in sets.Set(pcus)]
 374
 375         elif config.nodelist == None and config.pcuid == None:
 376                 print "Calling API GetPCUs() : refresh(%s)" % config.refresh
 377                 l_pcus  = [pcu['pcu_id'] for pcu in l_pcus]
 378         elif config.nodelist is not None:
 379                 l_pcus = util.file.getListFromFile(config.nodelist)
 380                 l_pcus = [int(pcu) for pcu in l_pcus]
 381         elif config.pcuid is not None:
 382                 l_pcus = [ config.pcuid ]
 383                 l_pcus = [int(pcu) for pcu in l_pcus]
 384
 385         checkAndRecordState(l_pcus, cohash)
 386
 387         return 0
 388
 389
 390 if __name__ == '__main__':
 391         import logging
 392         logger = logging.getLogger("monitor")
 393         logger.setLevel(logging.DEBUG)
 394         fh = logging.FileHandler("monitor.log", mode = 'a')
 395         fh.setLevel(logging.DEBUG)
 396         formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
 397         fh.setFormatter(formatter)
 398         logger.addHandler(fh)
 399         from monitor import parser as parsermodule
 400         parser = parsermodule.getParser()
 401         parser.set_defaults(nodelist=None,
 402                                                 increment=False,
 403                                                 pcuid=None,
 404                                                 pcuselect=None,
 405                                                 site=None,
 406                                                 dbname="findbadpcus",
 407                                                 cachenodes=False,
 408                                                 refresh=False,
 409                                                 )
 410         parser.add_option("-f", "--nodelist", dest="nodelist", metavar="FILE",
 411                                                 help="Provide the input file for the node list")
 412         parser.add_option("", "--site", dest="site", metavar="FILE",
 413                                                 help="Get all pcus associated with the given site's nodes")
 414         parser.add_option("", "--pcuselect", dest="pcuselect", metavar="FILE",
 415                                                 help="Query string to apply to the findbad pcus")
 416         parser.add_option("", "--pcuid", dest="pcuid", metavar="id",
 417                                                 help="Provide the id for a single pcu")
 418
 419         parser.add_option("", "--cachenodes", action="store_true",
 420                                                 help="Cache node lookup from PLC")
 421         parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
 422                                                 help="Specify the name of the database to which the information is saved")
 423         parser.add_option("", "--refresh", action="store_true", dest="refresh",
 424                                                 help="Refresh the cached values")
 425         parser.add_option("-i", "--increment", action="store_true", dest="increment",
 426                                                 help="Increment round number to force refresh or retry")
 427         parser = parsermodule.getParser(['defaults'], parser)
 428         config = parsermodule.parse_args(parser)
 429         try:
 430                 # NOTE: evidently, there is a bizarre interaction between iLO and ssh
 431                 # when LANG is set... Do not know why.  Unsetting LANG, fixes the problem.
 432                 if 'LANG' in os.environ:
 433                         del os.environ['LANG']
 434                 main()
 435                 time.sleep(1)
 436         except Exception, err:
 437                 traceback.print_exc()
 438                 print "Exception: %s" % err
 439                 print "Saving data... exitting."
 440                 sys.exit(0)