findbadpcu.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import sys
   5 import string
   6 import time
   7 import socket
   8 import sets
   9 import signal
  10 import traceback
  11 from datetime import datetime,timedelta
  12 import threadpool
  13 import threading
  14
  15 import monitor
  16 from monitor.pcu import reboot
  17 from monitor import config
  18 from monitor.database import FindbadPCURecordSync, FindbadPCURecord
  19 from monitor import util
  20 from monitor.wrapper import plc
  21 import syncplcdb
  22 from nodequery import pcu_select
  23
  24 plc_lock = threading.Lock()
  25 global_round = 1
  26 errorState = {}
  27 count = 0
  28
  29 def nmap_portstatus(status):
  30         ps = {}
  31         l_nmap = status.split()
  32         ports = l_nmap[4:]
  33
  34         continue_probe = False
  35         for port in ports:
  36                 results = port.split('/')
  37                 ps[results[0]] = results[1]
  38                 if results[1] == "open":
  39                         continue_probe = True
  40         return (ps, continue_probe)
  41
  42 def get_pcu(pcuname):
  43         plc_lock.acquire()
  44         try:
  45                 #print "GetPCU from PLC %s" % pcuname
  46                 l_pcu  = plc.GetPCUs({'pcu_id' : pcuname})
  47                 #print l_pcu
  48                 if len(l_pcu) > 0:
  49                         l_pcu = l_pcu[0]
  50         except:
  51                 try:
  52                         #print "GetPCU from file %s" % pcuname
  53                         l_pcus = database.dbLoad("pculist")
  54                         for i in l_pcus:
  55                                 if i['pcu_id'] == pcuname:
  56                                         l_pcu = i
  57                 except:
  58                         traceback.print_exc()
  59                         l_pcu = None
  60
  61         plc_lock.release()
  62         return l_pcu
  63
  64 def get_nodes(node_ids):
  65         plc_lock.acquire()
  66         l_node = []
  67         try:
  68                 l_node = plc.getNodes(node_ids, ['hostname', 'last_contact', 'node_id', 'ports'])
  69         except:
  70                 try:
  71                         plc_nodes = database.dbLoad("l_plcnodes")
  72                         for n in plc_nodes:
  73                                 if n['node_id'] in node_ids:
  74                                         l_node.append(n)
  75                 except:
  76                         traceback.print_exc()
  77                         l_node = None
  78
  79         plc_lock.release()
  80         if l_node == []:
  81                 l_node = None
  82         return l_node
  83
  84
  85 def get_plc_pcu_values(pcuname):
  86         """
  87                 Try to contact PLC to get the PCU info.
  88                 If that fails, try a backup copy from the last run.
  89                 If that fails, return None
  90         """
  91         values = {}
  92
  93         l_pcu = get_pcu(pcuname)
  94
  95         if l_pcu is not None:
  96                 site_id = l_pcu['site_id']
  97                 node_ids = l_pcu['node_ids']
  98                 l_node = get_nodes(node_ids)
  99
 100                 if l_node is not None:
 101                         for node in l_node:
 102                                 values[node['hostname']] = node['ports'][0]
 103
 104                         values['nodenames'] = [node['hostname'] for node in l_node]
 105
 106                         # NOTE: this is for a dry run later. It doesn't matter which node.
 107                         values['node_id'] = l_node[0]['node_id']
 108
 109                 values.update(l_pcu)
 110         else:
 111                 values = None
 112
 113         return values
 114
 115 def get_plc_site_values(site_id):
 116         ### GET PLC SITE ######################
 117         plc_lock.acquire()
 118         values = {}
 119         d_site = None
 120
 121         try:
 122                 d_site = plc.getSites({'site_id': site_id}, ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
 123                 if len(d_site) > 0:
 124                         d_site = d_site[0]
 125         except:
 126                 try:
 127                         plc_sites = database.dbLoad("l_plcsites")
 128                         for site in plc_sites:
 129                                 if site['site_id'] == site_id:
 130                                         d_site = site
 131                                         break
 132                 except:
 133                         traceback.print_exc()
 134                         values = None
 135
 136         plc_lock.release()
 137
 138         if d_site is not None:
 139                 max_slices = d_site['max_slices']
 140                 num_slices = len(d_site['slice_ids'])
 141                 num_nodes = len(d_site['node_ids'])
 142                 loginbase = d_site['login_base']
 143                 values['plcsite'] = {'num_nodes' : num_nodes,
 144                                                         'max_slices' : max_slices,
 145                                                         'num_slices' : num_slices,
 146                                                         'login_base' : loginbase,
 147                                                         'status'     : 'SUCCESS'}
 148         else:
 149                 values = None
 150
 151
 152         return values
 153
 154
 155 def collectPingAndSSH(pcuname, cohash):
 156
 157         continue_probe = True
 158         errors = None
 159         values = {'reboot' : 'novalue'}
 160         ### GET PCU ######################
 161         try:
 162                 b_except = False
 163                 try:
 164                         v = get_plc_pcu_values(pcuname)
 165                         if v['hostname'] is not None: v['hostname'] = v['hostname'].strip()
 166                         if v['ip'] is not None: v['ip'] = v['ip'].strip()
 167
 168                         if v is not None:
 169                                 values['plc_pcu_stats'] = v
 170                         else:
 171                                 continue_probe = False
 172                 except:
 173                         b_except = True
 174                         traceback.print_exc()
 175                         continue_probe = False
 176
 177                 if b_except or not continue_probe: return (None, None, None)
 178
 179
 180                 #### COMPLETE ENTRY   #######################
 181
 182                 values['complete_entry'] = []
 183                 #if values['protocol'] is None or values['protocol'] is "":
 184                 #       values['complete_entry'] += ["protocol"]
 185                 if values['plc_pcu_stats']['model'] is None or values['plc_pcu_stats']['model'] is "":
 186                         values['complete_entry'] += ["model"]
 187                         # Cannot continue due to this condition
 188                         continue_probe = False
 189
 190                 if values['plc_pcu_stats']['password'] is None or values['plc_pcu_stats']['password'] is "":
 191                         values['complete_entry'] += ["password"]
 192                         # Cannot continue due to this condition
 193                         continue_probe = False
 194
 195                 if len(values['complete_entry']) > 0:
 196                         continue_probe = False
 197
 198                 if values['plc_pcu_stats']['hostname'] is None or values['plc_pcu_stats']['hostname'] is "":
 199                         values['complete_entry'] += ["hostname"]
 200                 if values['plc_pcu_stats']['ip'] is None or values['plc_pcu_stats']['ip'] is "":
 201                         values['complete_entry'] += ["ip"]
 202
 203                 # If there are no nodes associated with this PCU, then we cannot continue.
 204                 if len(values['plc_pcu_stats']['node_ids']) == 0:
 205                         continue_probe = False
 206                         values['complete_entry'] += ['NoNodeIds']
 207
 208                 #### DNS and IP MATCH #######################
 209                 if values['plc_pcu_stats']['hostname'] is not None and values['plc_pcu_stats']['hostname'] is not "" and \
 210                    values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
 211                         #print "Calling socket.gethostbyname(%s)" % values['hostname']
 212                         try:
 213                                 ipaddr = socket.gethostbyname(values['plc_pcu_stats']['hostname'])
 214                                 if ipaddr == values['plc_pcu_stats']['ip']:
 215                                         values['dnsmatch'] = "DNS-OK"
 216                                 else:
 217                                         values['dnsmatch'] = "DNS-MISMATCH"
 218                                         continue_probe = False
 219
 220                         except Exception, err:
 221                                 values['dnsmatch'] = "DNS-NOENTRY"
 222                                 values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
 223                                 #print err
 224                 else:
 225                         if values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
 226                                 values['dnsmatch'] = "NOHOSTNAME"
 227                                 values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
 228                         else:
 229                                 values['dnsmatch'] = "NO-DNS-OR-IP"
 230                                 values['plc_pcu_stats']['hostname'] = "No_entry_in_DB"
 231                                 continue_probe = False
 232
 233                 #### RUN NMAP ###############################
 234                 if continue_probe:
 235                         nmap = util.command.CMD()
 236                         (oval,eval) = nmap.run_noexcept("nmap -oG - -P0 -p22,23,80,443,5869,9100,16992 %s | grep Host:" % reboot.pcu_name(values['plc_pcu_stats']))
 237                         # NOTE: an empty / error value for oval, will still work.
 238                         (values['portstatus'], continue_probe) = nmap_portstatus(oval)
 239                 else:
 240                         values['portstatus'] = None
 241
 242
 243                 ######  DRY RUN  ############################
 244                 if 'node_ids' in values['plc_pcu_stats'] and len(values['plc_pcu_stats']['node_ids']) > 0:
 245                         rb_ret = reboot.reboot_test(values['plc_pcu_stats']['nodenames'][0], values, continue_probe, 1, True)
 246                 else:
 247                         rb_ret = "Not_Run" # No nodes to test"
 248
 249                 values['reboot'] = rb_ret
 250
 251         except:
 252                 print "____________________________________"
 253                 print values
 254                 errors = values
 255                 print "____________________________________"
 256                 errors['traceback'] = traceback.format_exc()
 257                 print errors['traceback']
 258
 259         values['date_checked'] = time.time()
 260         return (pcuname, values, errors)
 261
 262 def recordPingAndSSH(request, result):
 263         global errorState
 264         global count
 265         global global_round
 266         (nodename, values, errors) = result
 267
 268         if values is not None:
 269                 pcu_id = int(nodename)
 270                 fbsync = FindbadPCURecordSync.findby_or_create(plc_pcuid=0,
 271                                                                                         if_new_set={'round': global_round})
 272                 global_round = fbsync.round
 273                 fbnodesync = FindbadPCURecordSync.findby_or_create(plc_pcuid=pcu_id,
 274                                                                                         if_new_set={'round' : global_round})
 275
 276                 fbrec = FindbadPCURecord(
 277                                         date_checked=datetime.fromtimestamp(values['date_checked']),
 278                                         plc_pcuid=pcu_id,
 279                                         plc_pcu_stats=values['plc_pcu_stats'],
 280                                         dns_status=values['dnsmatch'],
 281                                         port_status=values['portstatus'],
 282                                         entry_complete=" ".join(values['complete_entry']),
 283                                         reboot_trial_status="%s" % values['reboot'],
 284                                 )
 285                 fbnodesync.round = global_round
 286                 count += 1
 287                 print "%d %s %s" % (count, nodename, values)
 288
 289         if errors is not None:
 290                 pcu_id = "id_%s" % nodename
 291                 errorState[pcu_id] = errors
 292                 database.dbDump("findbadpcu_errors", errorState)
 293
 294 # this will be called when an exception occurs within a thread
 295 def handle_exception(request, result):
 296         print "Exception occured in request %s" % request.requestID
 297         for i in result:
 298                 print "Result: %s" % i
 299
 300
 301 def checkAndRecordState(l_pcus, cohash):
 302         global global_round
 303         global count
 304
 305         tp = threadpool.ThreadPool(10)
 306
 307         # CREATE all the work requests
 308         for pcuname in l_pcus:
 309                 pcu_id = int(pcuname)
 310                 fbnodesync = FindbadPCURecordSync.findby_or_create(plc_pcuid=pcu_id, if_new_set={'round' : 0})
 311
 312                 node_round   = fbnodesync.round
 313                 if node_round < global_round:
 314                         # recreate node stats when refreshed
 315                         #print "%s" % nodename
 316                         req = threadpool.WorkRequest(collectPingAndSSH, [pcuname, cohash], {},
 317                                                                                  None, recordPingAndSSH, handle_exception)
 318                         tp.putRequest(req)
 319                 else:
 320                         # We just skip it, since it's "up to date"
 321                         count += 1
 322                         print "%d %s %s" % (count, pcu_id, node_round)
 323
 324         # WAIT while all the work requests are processed.
 325         begin = time.time()
 326         while 1:
 327                 try:
 328                         time.sleep(1)
 329                         tp.poll()
 330                         # if more than two hours
 331                         if time.time() - begin > (60*60*1):
 332                                 print "findbadpcus.py has run out of time!!!!!!"
 333                                 os._exit(1)
 334                 except KeyboardInterrupt:
 335                         print "Interrupted!"
 336                         break
 337                 except threadpool.NoResultsPending:
 338                         print "All results collected."
 339                         break
 340
 341         print FindbadPCURecordSync.query.count()
 342         print FindbadPCURecord.query.count()
 343
 344
 345 def main():
 346         global global_round
 347
 348         l_pcus = monitor.database.if_cached_else_refresh(1, config.refresh, "pculist", lambda : plc.GetPCUs())
 349         cohash = {}
 350
 351         fbsync = FindbadPCURecordSync.findby_or_create(plc_pcuid=0, if_new_set={'round' : global_round})
 352
 353         global_round = fbsync.round
 354
 355         if config.increment:
 356                 # update global round number to force refreshes across all nodes
 357                 global_round += 1
 358                 fbsync.round = global_round
 359
 360         if config.site is not None:
 361                 api = plc.getAuthAPI()
 362                 site = api.GetSites(config.site)
 363                 l_nodes = api.GetNodes(site[0]['node_ids'], ['pcu_ids'])
 364                 pcus = []
 365                 for node in l_nodes:
 366                         pcus += node['pcu_ids']
 367                 # clear out dups.
 368                 l_pcus = [pcu for pcu in sets.Set(pcus)]
 369         elif config.pcuselect is not None:
 370                 n, pcus = pcu_select(config.pcuselect)
 371                 # clear out dups.
 372                 l_pcus = [pcu for pcu in sets.Set(pcus)]
 373
 374         elif config.nodelist == None and config.pcuid == None:
 375                 print "Calling API GetPCUs() : refresh(%s)" % config.refresh
 376                 l_pcus  = [pcu['pcu_id'] for pcu in l_pcus]
 377         elif config.nodelist is not None:
 378                 l_pcus = util.file.getListFromFile(config.nodelist)
 379                 l_pcus = [int(pcu) for pcu in l_pcus]
 380         elif config.pcuid is not None:
 381                 l_pcus = [ config.pcuid ]
 382                 l_pcus = [int(pcu) for pcu in l_pcus]
 383
 384         checkAndRecordState(l_pcus, cohash)
 385
 386         return 0
 387
 388
 389 if __name__ == '__main__':
 390         import logging
 391         logger = logging.getLogger("monitor")
 392         logger.setLevel(logging.DEBUG)
 393         fh = logging.FileHandler("monitor.log", mode = 'a')
 394         fh.setLevel(logging.DEBUG)
 395         formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
 396         fh.setFormatter(formatter)
 397         logger.addHandler(fh)
 398         from monitor import parser as parsermodule
 399         parser = parsermodule.getParser()
 400         parser.set_defaults(nodelist=None,
 401                                                 increment=False,
 402                                                 pcuid=None,
 403                                                 pcuselect=None,
 404                                                 site=None,
 405                                                 dbname="findbadpcus",
 406                                                 cachenodes=False,
 407                                                 refresh=False,
 408                                                 )
 409         parser.add_option("-f", "--nodelist", dest="nodelist", metavar="FILE",
 410                                                 help="Provide the input file for the node list")
 411         parser.add_option("", "--site", dest="site", metavar="FILE",
 412                                                 help="Get all pcus associated with the given site's nodes")
 413         parser.add_option("", "--pcuselect", dest="pcuselect", metavar="FILE",
 414                                                 help="Query string to apply to the findbad pcus")
 415         parser.add_option("", "--pcuid", dest="pcuid", metavar="id",
 416                                                 help="Provide the id for a single pcu")
 417
 418         parser.add_option("", "--cachenodes", action="store_true",
 419                                                 help="Cache node lookup from PLC")
 420         parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
 421                                                 help="Specify the name of the database to which the information is saved")
 422         parser.add_option("", "--refresh", action="store_true", dest="refresh",
 423                                                 help="Refresh the cached values")
 424         parser.add_option("-i", "--increment", action="store_true", dest="increment",
 425                                                 help="Increment round number to force refresh or retry")
 426         parser = parsermodule.getParser(['defaults'], parser)
 427         config = parsermodule.parse_args(parser)
 428         try:
 429                 # NOTE: evidently, there is a bizarre interaction between iLO and ssh
 430                 # when LANG is set... Do not know why.  Unsetting LANG, fixes the problem.
 431                 if 'LANG' in os.environ:
 432                         del os.environ['LANG']
 433                 main()
 434                 time.sleep(1)
 435         except Exception, err:
 436                 traceback.print_exc()
 437                 print "Exception: %s" % err
 438                 print "Saving data... exitting."
 439                 sys.exit(0)