www/printbadnodes.py

   1 #!/usr/bin/python
   2 from monitor import database
   3 from monitor import config
   4 import string
   5 import sys
   6
   7 categories = {}
   8 ssherror = False
   9 fb = {}
  10
  11 def sec2days(sec):
  12         if sec == "null":
  13                 sec = -(60*60*24)
  14         sec = int(sec)
  15         return sec/(60*60*24)
  16
  17 def array_to_priority_map(array):
  18         """ Create a mapping where each entry of array is given a priority equal
  19         to its position in the array.  This is useful for subsequent use in the
  20         cmpMap() function."""
  21         map = {}
  22         count = 0
  23         for i in array:
  24                 map[i] = count
  25                 count += 1
  26         return map
  27
  28 def cmpValMap(v1, v2, map):
  29         if v1 in map and v2 in map and map[v1] < map[v2]:
  30                 return 1
  31         elif v1 in map and v2 in map and map[v1] > map[v2]:
  32                 return -1
  33         elif v1 in map and v2 in map:
  34                 return 0
  35         else:
  36                 raise Exception("No index %s or %s in map" % (v1, v2))
  37
  38 def cmpMap(l1, l2, index, map):
  39         if index in l1 and index in l2:
  40                 if map[l1[index]] < map[l2[index]]:
  41                         return -1
  42                 elif map[l1[index]] > map[l2[index]]:
  43                         return 1
  44                 else:
  45                         return 0
  46         else:
  47                 return 0
  48
  49 def cmpLoginBase(l1, l2):
  50         #print "'" + l1['loginbase'] + "'"  + " < " + "'" + l2['loginbase'] + "'" + "<BR>"
  51         if l1['loginbase'] == l2['loginbase']:
  52                 return 0
  53         elif l1['loginbase'] < l2['loginbase']:
  54                 return -1
  55         elif l1['loginbase'] > l2['loginbase']:
  56                 return 1
  57         else:
  58                 return 0
  59
  60 def cmpState(l1, l2):
  61         map = array_to_priority_map([ 'BOOT', 'DEBUG', 'DOWN' ])
  62         return cmpMap(l1,l2,'state', map)
  63
  64 def cmpCategoryVal(v1, v2):
  65         map = array_to_priority_map([ None, 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
  66         return cmpValMap(v1,v2,map)
  67
  68 def cmpCategory(l1, l2):
  69         map = array_to_priority_map([ 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'ERROR', ])
  70         return cmpMap(l1,l2,'category', map)
  71
  72 def cmpPCU(l1, l2):
  73         """ Either PCU or NOPCU"""
  74         map = array_to_priority_map([ 'PCU', 'NOPCU', 'UNKNOWN'])
  75         return cmpMap(l1, l2, 'pcu', map)
  76
  77 def cmpSSH(l1, l2):
  78         """ Either SSH or NOSSH """
  79         map = array_to_priority_map([ 'SSH', 'NOSSH'])
  80         return cmpMap(l1, l2, 'ssh', map)
  81
  82 def cmpDNS(l1,l2):
  83         """ Compare DNS states """
  84         map = array_to_priority_map([ 'OK', 'NOHOSTNAME', 'NOENTRY', 'MISMATCH'])
  85         return cmpMap(l1, l2, 'dnsmatch', map)
  86
  87 def cmpPing(l1,l2):
  88         """ Either PING or NOPING """
  89         map = array_to_priority_map([ 'PING', 'NOPING'])
  90         return cmpMap(l1, l2, 'ping', map)
  91
  92 def cmpUname(l1, l2):
  93         # Extract the kernel version from kernel -a string
  94         l_k1 = l1['kernel'].split()
  95         if len(l_k1) > 2:
  96                 k1 = l_k1[2]
  97         else:
  98                 return 1
  99
 100         l_k2 = l2['kernel'].split()
 101         if len(l_k2) > 2:
 102                 k2 = l_k2[2]
 103         else:
 104                 return -1
 105
 106         return cmp(k1, k2)
 107
 108 def cmpDays(l1, l2):
 109         if l1['comonstats'][config.comon] == "null":
 110                 l1['comonstats'][config.comon] = -1
 111         if l2['comonstats'][config.comon] == "null":
 112                 l2['comonstats'][config.comon] = -1
 113
 114         if int(l1['comonstats'][config.comon]) > int(l2['comonstats'][config.comon]):
 115                 return -1
 116         elif int(l1['comonstats'][config.comon]) < int(l2['comonstats'][config.comon]):
 117                 return 1
 118         else:
 119                 return 0
 120
 121 def ssh_error_to_str(str):
 122         ssh_error = ""
 123         if "Connection timed out" in str:
 124                 ssh_error = "Timeout"
 125         elif "Connection closed by remote host" in str:
 126                 ssh_error = "Closed by remote host"
 127         elif "Connection refused" in str:
 128                 ssh_error = "Connection refused"
 129         elif "Temporary failure in name resolution" in str:
 130                 ssh_error = "Could not resolve name"
 131         elif "Name or service not known" in str:
 132                 ssh_error = "Name not known"
 133         elif "Too many authentication failures" in str:
 134                 ssh_error = "Disconnect: root auth failure"
 135         elif "Network is unreachable" in str:
 136                 ssh_error = "Network is unreachable"
 137         elif "Connection reset by peer" in str:
 138                 ssh_error = "Connection reset by peer"
 139         elif "WARNING" in str:
 140                 ssh_error = "WARNING ssh key updated"
 141         else:
 142                 ssh_error = str
 143
 144         return ssh_error
 145
 146 def pcu_state(pcu_id):
 147         global fb
 148
 149         if 'nodes' in fb and "id_%s" % pcu_id in fb['nodes'] \
 150                 and 'values' in fb['nodes']["id_%s" % pcu_id]:
 151                 rec = fb['nodes']["id_%s" % pcu_id]['values']
 152                 if 'reboot' in rec:
 153                         rb = rec['reboot']
 154                         if rb == 0 or rb == "0":
 155                                 return 0
 156                         elif "NetDown" == rb  or "Not_Run" == rb:
 157                                 return 1
 158                         else:
 159                                 return -1
 160                 else:
 161                         return -1
 162         else:
 163                 return -1
 164
 165 def fields_to_html(fields, vals):
 166         global categories
 167         global ssherror
 168         pcu_colorMap = { -1 : 'indianred',
 169                                           0 : 'darkseagreen',
 170                                           1 : 'gold', }
 171
 172         colorMap = { 'PING'  : 'darkseagreen',
 173                                  'NOPING': 'darksalmon',
 174                                  'SSH': 'darkseagreen',
 175                                  'NOSSH': 'indianred',
 176                                  'PCU': 'darkseagreen',
 177                                  'NOPCU': 'lightgrey',
 178                                  'OLDBOOTCD': 'crimson',
 179                                  'DOWN': 'indianred',
 180                                  'ALPHA': 'gold',
 181                                  'ERROR': 'crimson',
 182                                  'PROD': 'darkseagreen',
 183                                  'DEBUG': 'darksalmon',
 184                                  'DEBUG': 'darksalmon',
 185                                  'BOOT': 'lightgreen'}
 186         r_str = ""
 187         f_prev = ""
 188         f_2prev = ""
 189         #print 'inside--------------'
 190         for f in fields:
 191                 f = f.strip()
 192                 #print f
 193
 194                 if f in ['DOWN', 'BOOT', 'DEBUG']:
 195                         #key = "%s-%s-%s" % (f,f_prev,f_2prev)
 196                         key = "%s-%s" % (f,f_prev)
 197                         if key not in categories:
 198                                 categories[key] = 1
 199                         else:
 200                                 categories[key] += 1
 201
 202                 #print "<pre>%s</pre><br>" % f
 203
 204                 if f in colorMap:
 205                         bgcolor="bgcolor='%s'" % colorMap[f]
 206                 else:
 207                         bgcolor=""
 208
 209                 if f == 'NOSSH':
 210                         if ssherror:
 211                                 if 'ssherror' in vals:
 212                                         str_ssh_error = ssh_error_to_str(vals['ssherror'])
 213                                 else:
 214                                         str_ssh_error = "NO SSHERROR in VALS"
 215                                 if str_ssh_error != "Timeout":
 216                                         r_str += """<td nowrap %s>%s<br><b><font size="-2">%s</font></b></td>""" % \
 217                                                                 (bgcolor,f,str_ssh_error)
 218                                 else:
 219                                         r_str += "<td %s>%s</td>" % (bgcolor, f)
 220                         else:
 221                                 r_str += "<td %s>%s</td>" % (bgcolor, f)
 222                 elif f == 'PCU':
 223                         if len(vals['plcnode']['pcu_ids']) > 0:
 224                                 #print "pcu_id: %s<br>" % vals['plcnode']['pcu_ids'][0]
 225                                 #print "state: %s<br>" % pcu_state(vals['plcnode']['pcu_ids'][0])
 226                                 #print "color: %s<br>" % pcu_colorMap[pcu_state(vals['plcnode']['pcu_ids'][0])]
 227                                 bgcolor = "bgcolor='%s'" % pcu_colorMap[pcu_state(vals['plcnode']['pcu_ids'][0])]
 228                                 url = "<a href='/cgi-bin/printbadpcus.php#id%s'>PCU</a>" % vals['plcnode']['pcu_ids'][0]
 229                                 r_str += "<td nowrap %s>%s</td>" % (bgcolor, url)
 230                 else:
 231                         r_str += "<td nowrap %s>%s</td>" % (bgcolor, f)
 232                 f_2prev = f_prev
 233                 f_prev  = f
 234
 235         return r_str
 236
 237
 238
 239 def main(sitefilter, catfilter, statefilter, comonfilter, nodeonlyfilter):
 240         global fb
 241
 242         db = database.dbLoad(config.dbname)
 243         fb = database.dbLoad("findbadpcus")
 244
 245         ## Field widths used for printing
 246         maxFieldLengths = { 'nodename' : -45,
 247                                                 'ping' : 6,
 248                                                 'ssh' : 6,
 249                                                 'pcu' : 7,
 250                                                 'category' : 9,
 251                                                 'state' : 5,
 252                                                 'kernel' : 10.65,
 253                                                 'comonstats' : 5,
 254                                                 'plcsite' : 12,
 255                                                 'bootcd' : 10.65}
 256         ## create format string based on config.fields
 257         fields = {}
 258         format = ""
 259         format_fields = []
 260         for f in config.fields.split(','):
 261                 fields[f] = "%%(%s)s" % f
 262                 #if f in maxFieldLengths:
 263                 #       fields[f] = "%%(%s)%ds" % (f, maxFieldLengths[f])
 264                 #else:
 265                 #       fields[f] = "%%(%s)%ds" % (f, 10)
 266
 267                 format_fields.append(fields[f])
 268         #print fields
 269         for f in config.fields.split(','):
 270                 format += fields[f] + " "
 271         #print format
 272
 273         d_n = db['nodes']
 274         l_nodes = d_n.keys()
 275
 276         # category by site
 277         #bysite = {}
 278         #for nodename in l_nodes:
 279         #       if 'plcsite' in d_n[nodename]['values'] and \
 280         #       'login_base' in d_n[nodename]['values']['plcsite']:
 281         #               loginbase = d_n[nodename]['values']['plcsite']['login_base']
 282         #               if loginbase not in bysite:
 283         #                       bysite[loginbase] = []
 284         #               d_n[nodename]['values']['nodename'] = nodename
 285         #               bysite[loginbase].append(d_n[nodename]['values'])
 286
 287         # d2 was an array of [{node}, {}, ...]
 288         # the bysite is a loginbase dict of [{node}, {node}]
 289         d2 = []
 290         import re
 291         if sitefilter != None:
 292                 sf = re.compile(sitefilter)
 293         else:
 294                 sf = None
 295         for nodename in l_nodes:
 296                 vals=d_n[nodename]['values']
 297                 v = {}
 298                 v.update(vals)
 299                 v['nodename'] = nodename
 300                 if  'plcsite' in vals and  \
 301                         'status' in vals['plcsite'] and  \
 302                         vals['plcsite']['status'] == "SUCCESS":
 303
 304                         url = "<a href='printbadnodes.py?site=%s'>%s</a>" % ( vals['plcsite']['login_base'],
 305                                                                                                                          vals['plcsite']['login_base'])
 306
 307                         site_string = "%s %2s nodes :: %2s of %4s slices" % ( \
 308                                                                                                                 url,
 309                                                                                                                 vals['plcsite']['num_nodes'],
 310                                                                                                                 vals['plcsite']['num_slices'],
 311                                                                                                                 vals['plcsite']['max_slices'])
 312                         loginbase = d_n[nodename]['values']['plcsite']['login_base']
 313                 else:
 314                         #print "ERROR: ", nodename, vals, "<br>"
 315                         site_string = "<b>UNKNOWN</b>"
 316                         loginbase = ""
 317
 318                 v['site_string'] = site_string
 319                 v['loginbase'] = loginbase
 320                 if (sitefilter != None and sf.match(loginbase) != None) or sitefilter == None:
 321                         d2.append(v)
 322
 323
 324         if sitefilter != None:
 325                 config.cmpcategory = True
 326         else:
 327                 config.cmploginbase = True
 328
 329
 330         if config.cmploginbase:
 331                 d2.sort(cmp=cmpLoginBase)
 332         elif config.cmpping:
 333                 d2.sort(cmp=cmpPing)
 334         elif config.cmpdns:
 335                 d2.sort(cmp=cmpDNS)
 336         elif config.cmpssh:
 337                 d2.sort(cmp=cmpSSH)
 338         elif config.cmpcategory:
 339                 d2.sort(cmp=cmpCategory)
 340         elif config.cmpstate:
 341                 d2.sort(cmp=cmpState)
 342         elif config.cmpdays:
 343                 d2.sort(cmp=cmpDays)
 344         elif config.cmpkernel:
 345                 d2.sort(cmp=cmpUname)
 346         else:
 347                 d2.sort(cmp=cmpCategory)
 348
 349
 350         if catfilter != None:   cf = re.compile(catfilter)
 351         else:                                   cf = None
 352
 353         if statefilter != None: stf = re.compile(statefilter)
 354         else:                                   stf = None
 355
 356         if comonfilter != None: cmf = re.compile(comonfilter)
 357         else:                                   cmf = None
 358
 359         #l_loginbase = bysite.keys()
 360         #l_loginbase.sort()
 361         if nodeonlyfilter == None:
 362                 print "<table width=80% border=1>"
 363
 364         prev_sitestring = ""
 365         for row in d2:
 366
 367                 vals = row
 368
 369                 if (catfilter != None and cf.match(vals['category']) == None):
 370                         continue
 371
 372                 if (statefilter != None and stf.match(vals['state']) == None):
 373                         continue
 374
 375                 if (comonfilter != None and comonfilter in vals['comonstats'] and vals['comonstats'][comonfilter] != 'null'):
 376                         continue
 377
 378                 if nodeonlyfilter != None:
 379                         print vals['nodename']
 380                         continue
 381
 382                 site_string = row['site_string']
 383                 if site_string != prev_sitestring:
 384                         print "<tr><td bgcolor=lightblue nowrap>"
 385                         print site_string
 386                         print "</td>"
 387                 else:
 388                         print "<tr><td>&nbsp;</td>"
 389
 390                 prev_sitestring = site_string
 391
 392
 393                 # convert uname values into a single kernel version string
 394                 if 'kernel' in vals:
 395                         kernel = vals['kernel'].split()
 396                         if len(kernel) > 0:
 397                                 if kernel[0] == "Linux":
 398                                         vals['kernel'] = kernel[2]
 399                                 else:
 400                                         vals['ssherror'] = vals['kernel']
 401                                         vals['kernel'] = ""
 402                 else:
 403                         vals['ssherror'] = ""
 404                         vals['kernel'] = ""
 405 #                       continue
 406                 if 'model' in vals or 'protocol' in vals or 'portstatus' in vals:
 407                         #vals['model'] = string.replace(vals['model']," ", "&nbsp;")
 408                         #vals['protocol'] = vals['protocol'].replace(" ", "&nbsp;")
 409                         if vals['model'] == None:
 410                                 vals['model'] = " "
 411                         vals['model'] = string.replace(vals['model']," ", "_")
 412                         vals['protocol'] = vals['protocol'].replace(" ", "_")
 413                         ps = ""
 414                         ports = vals['portstatus']
 415                         lports = ports.keys()
 416                         lports.sort()
 417                         for port in lports:
 418                                 t = ports[port]
 419                                 if t != "closed":
 420                                         ps += "%s:&nbsp;%s<br>" % (port, ports[port])
 421                         if ps == "":
 422                                 ps = "All_closed"
 423
 424                         vals['portstatus'] = ps
 425
 426                 if 'reboot' in vals:
 427                         vals['reboot'] = "%s" % vals['reboot']
 428                         vals['reboot'] = vals['reboot'].replace(" ", "_")
 429
 430                 if 'nodename' in vals:
 431                         url = "<a href='https://%s/db/nodes/index.php?nodepattern=%s'>%s</a>" % (config.MONITOR_HOSTNAME, vals['nodename'], vals['nodename'])
 432                         vals['nodename'] = url
 433
 434                 try:
 435                         str_fields = []
 436                         count = 0
 437                         for f in format_fields:
 438                                 str_fields.append(f % vals)
 439                                 count += 1
 440                 except:
 441                         print >>sys.stderr, vals
 442
 443                 s = fields_to_html(str_fields, vals)
 444                 print s
 445
 446                 print "\n</tr>"
 447
 448         if nodeonlyfilter == None:
 449                 print "</table>"
 450                 print "<table>"
 451         keys = categories.keys()
 452         keys.sort()
 453         for cat in keys:
 454                 print "<tr>"
 455                 print "<th nowrap align=left>Total %s</th>" % cat
 456                 print "<td align=left>%s</td>" % categories[cat]
 457                 print "</tr>"
 458         if nodeonlyfilter == None:
 459                 print "</table>"
 460
 461
 462
 463 if __name__ == '__main__':
 464         import cgi
 465         import cgitb;
 466         cgitb.enable()
 467         import sys
 468
 469         form = cgi.FieldStorage()
 470         myfilter = None
 471
 472         if form.has_key('site'):
 473                 myfilter = form.getvalue("site")
 474         else:
 475                 myfilter = None
 476
 477         if form.has_key('category'):
 478                 mycategory = form.getvalue("category")
 479         else:
 480                 mycategory = None
 481
 482         if form.has_key('state'):
 483                 mystate = form.getvalue("state")
 484         else:
 485                 mystate = None
 486
 487         if form.has_key('comon'):
 488                 mycomon = form.getvalue("comon")
 489         else:
 490                 mycomon = None
 491
 492         if form.has_key('nodeonly'):
 493                 mynodeonly = form.getvalue("nodeonly")
 494         else:
 495                 mynodeonly = None
 496
 497         config.cmpdays=False
 498         config.comon="sshstatus"
 499         config.fields="nodename,ping,ssh,pcu,category,state,comonstats,kernel,bootcd"
 500         config.dbname="findbad"
 501         config.cmpping=False
 502         config.cmpdns=False
 503         config.cmploginbase=False
 504         config.cmpssh=False
 505         config.cmpcategory=False
 506
 507         print "Content-Type: text/html\r\n"
 508         if mynodeonly == None:
 509                 print "<html><body>\n"
 510         if len(sys.argv) > 1:
 511                 if sys.argv[1] == "ssherror":
 512                         ssherror = True
 513         main(myfilter, mycategory, mystate, mycomon,mynodeonly)
 514         if mynodeonly == None:
 515                 print "</body></html>\n"