www/printbadnodes.py

   1 #!/usr/bin/python
   2 import soltesz
   3 from config import config
   4 from optparse import OptionParser
   5 import string
   6
   7 import sys
   8
   9 categories = {}
  10 ssherror = False
  11
  12 def sec2days(sec):
  13         if sec == "null":
  14                 sec = -(60*60*24)
  15         sec = int(sec)
  16         return sec/(60*60*24)
  17
  18 def array_to_priority_map(array):
  19         """ Create a mapping where each entry of array is given a priority equal
  20         to its position in the array.  This is useful for subsequent use in the
  21         cmpMap() function."""
  22         map = {}
  23         count = 0
  24         for i in array:
  25                 map[i] = count
  26                 count += 1
  27         return map
  28
  29 def cmpValMap(v1, v2, map):
  30         if v1 in map and v2 in map and map[v1] < map[v2]:
  31                 return 1
  32         elif v1 in map and v2 in map and map[v1] > map[v2]:
  33                 return -1
  34         elif v1 in map and v2 in map:
  35                 return 0
  36         else:
  37                 raise Exception("No index %s or %s in map" % (v1, v2))
  38
  39 def cmpMap(l1, l2, index, map):
  40         if index in l1 and index in l2:
  41                 if map[l1[index]] < map[l2[index]]:
  42                         return -1
  43                 elif map[l1[index]] > map[l2[index]]:
  44                         return 1
  45                 else:
  46                         return 0
  47         else:
  48                 return 0
  49
  50 def cmpLoginBase(l1, l2):
  51         #print "'" + l1['loginbase'] + "'"  + " < " + "'" + l2['loginbase'] + "'" + "<BR>"
  52         if l1['loginbase'] == l2['loginbase']:
  53                 return 0
  54         elif l1['loginbase'] < l2['loginbase']:
  55                 return -1
  56         elif l1['loginbase'] > l2['loginbase']:
  57                 return 1
  58         else:
  59                 return 0
  60
  61 def cmpState(l1, l2):
  62         map = array_to_priority_map([ 'BOOT', 'DEBUG', 'DOWN' ])
  63         return cmpMap(l1,l2,'state', map)
  64
  65 def cmpCategoryVal(v1, v2):
  66         map = array_to_priority_map([ None, 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'ERROR', ])
  67         return cmpValMap(v1,v2,map)
  68
  69 def cmpCategory(l1, l2):
  70         map = array_to_priority_map([ 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'ERROR', ])
  71         return cmpMap(l1,l2,'category', map)
  72
  73 def cmpPCU(l1, l2):
  74         """ Either PCU or NOPCU"""
  75         map = array_to_priority_map([ 'PCU', 'NOPCU', 'UNKNOWN'])
  76         return cmpMap(l1, l2, 'pcu', map)
  77
  78 def cmpSSH(l1, l2):
  79         """ Either SSH or NOSSH """
  80         map = array_to_priority_map([ 'SSH', 'NOSSH'])
  81         return cmpMap(l1, l2, 'ssh', map)
  82
  83 def cmpDNS(l1,l2):
  84         """ Compare DNS states """
  85         map = array_to_priority_map([ 'OK', 'NOHOSTNAME', 'NOENTRY', 'MISMATCH'])
  86         return cmpMap(l1, l2, 'dnsmatch', map)
  87
  88 def cmpPing(l1,l2):
  89         """ Either PING or NOPING """
  90         map = array_to_priority_map([ 'PING', 'NOPING'])
  91         return cmpMap(l1, l2, 'ping', map)
  92
  93 def cmpUname(l1, l2):
  94         # Extract the kernel version from kernel -a string
  95         l_k1 = l1['kernel'].split()
  96         if len(l_k1) > 2:
  97                 k1 = l_k1[2]
  98         else:
  99                 return 1
 100
 101         l_k2 = l2['kernel'].split()
 102         if len(l_k2) > 2:
 103                 k2 = l_k2[2]
 104         else:
 105                 return -1
 106
 107         return cmp(k1, k2)
 108
 109 def cmpDays(l1, l2):
 110         if l1['comonstats'][config.comon] == "null":
 111                 l1['comonstats'][config.comon] = -1
 112         if l2['comonstats'][config.comon] == "null":
 113                 l2['comonstats'][config.comon] = -1
 114
 115         if int(l1['comonstats'][config.comon]) > int(l2['comonstats'][config.comon]):
 116                 return -1
 117         elif int(l1['comonstats'][config.comon]) < int(l2['comonstats'][config.comon]):
 118                 return 1
 119         else:
 120                 return 0
 121
 122 def ssh_error_to_str(str):
 123         ssh_error = ""
 124         if "Connection timed out" in str:
 125                 ssh_error = "Timeout"
 126         elif "Connection closed by remote host" in str:
 127                 ssh_error = "Closed by remote host"
 128         elif "Connection refused" in str:
 129                 ssh_error = "Connection refused"
 130         elif "Temporary failure in name resolution" in str:
 131                 ssh_error = "Could not resolve name"
 132         elif "Name or service not known" in str:
 133                 ssh_error = "Name not known"
 134         elif "Too many authentication failures" in str:
 135                 ssh_error = "Disconnect: root auth failure"
 136         elif "Network is unreachable" in str:
 137                 ssh_error = "Network is unreachable"
 138         elif "Connection reset by peer" in str:
 139                 ssh_error = "Connection reset by peer"
 140         elif "WARNING" in str:
 141                 ssh_error = "WARNING ssh key updated"
 142         else:
 143                 ssh_error = str
 144
 145         return ssh_error
 146
 147 def fields_to_html(fields, vals):
 148         global categories
 149         global ssherror
 150         colorMap = { 'PING'  : 'darkseagreen',
 151                                  'NOPING': 'darksalmon',
 152                                  'SSH': 'darkseagreen',
 153                                  'NOSSH': 'indianred',
 154                                  'PCU': 'darkseagreen',
 155                                  'NOPCU': 'lightgrey',
 156                                  'OLDBOOTCD': 'crimson',
 157                                  'DOWN': 'indianred',
 158                                  'ALPHA': 'gold',
 159                                  'ERROR': 'crimson',
 160                                  'PROD': 'darkseagreen',
 161                                  'DEBUG': 'darksalmon',
 162                                  'DEBUG': 'darksalmon',
 163                                  'BOOT': 'lightgreen'}
 164         r_str = ""
 165         f_prev = ""
 166         f_2prev = ""
 167         #print 'inside--------------'
 168         for f in fields:
 169                 f = f.strip()
 170                 #print f
 171
 172                 if f in ['DOWN', 'BOOT', 'DEBUG']:
 173                         #key = "%s-%s-%s" % (f,f_prev,f_2prev)
 174                         key = "%s-%s" % (f,f_prev)
 175                         if key not in categories:
 176                                 categories[key] = 1
 177                         else:
 178                                 categories[key] += 1
 179
 180                 #print "<pre>%s</pre><br>" % f
 181
 182                 if f in colorMap:
 183                         bgcolor="bgcolor='%s'" % colorMap[f]
 184                 else:
 185                         bgcolor=""
 186
 187                 if f == 'NOSSH':
 188                         if ssherror:
 189                                 if 'ssherror' in vals:
 190                                         str_ssh_error = ssh_error_to_str(vals['ssherror'])
 191                                 else:
 192                                         str_ssh_error = "NO SSHERROR in VALS"
 193                                 if str_ssh_error != "Timeout":
 194                                         r_str += """<td nowrap %s>%s<br><b><font size="-2">%s</font></b></td>""" % \
 195                                                                 (bgcolor,f,str_ssh_error)
 196                                 else:
 197                                         r_str += "<td %s>%s</td>" % (bgcolor, f)
 198                         else:
 199                                 r_str += "<td %s>%s</td>" % (bgcolor, f)
 200                 elif f == 'PCU':
 201                         if len(vals['plcnode']['pcu_ids']) > 0:
 202                                 url = "<a href='/cgi-bin/printbadpcus.php#id%s'>PCU</a>" % vals['plcnode']['pcu_ids'][0]
 203                                 r_str += "<td nowrap %s>%s</td>" % (bgcolor, url)
 204                 else:
 205                         r_str += "<td nowrap %s>%s</td>" % (bgcolor, f)
 206                 f_2prev = f_prev
 207                 f_prev  = f
 208
 209         return r_str
 210
 211
 212
 213 def main(sitefilter):
 214         db = soltesz.dbLoad(config.dbname)
 215
 216         ## Field widths used for printing
 217         maxFieldLengths = { 'nodename' : -45,
 218                                                 'ping' : 6,
 219                                                 'ssh' : 6,
 220                                                 'pcu' : 7,
 221                                                 'category' : 9,
 222                                                 'state' : 5,
 223                                                 'kernel' : 10.65,
 224                                                 'comonstats' : 5,
 225                                                 'plcsite' : 12,
 226                                                 'bootcd' : 10.65}
 227         ## create format string based on config.fields
 228         fields = {}
 229         format = ""
 230         format_fields = []
 231         for f in config.fields.split(','):
 232                 fields[f] = "%%(%s)s" % f
 233                 #if f in maxFieldLengths:
 234                 #       fields[f] = "%%(%s)%ds" % (f, maxFieldLengths[f])
 235                 #else:
 236                 #       fields[f] = "%%(%s)%ds" % (f, 10)
 237
 238                 format_fields.append(fields[f])
 239         #print fields
 240         for f in config.fields.split(','):
 241                 format += fields[f] + " "
 242         #print format
 243
 244         d_n = db['nodes']
 245         l_nodes = d_n.keys()
 246
 247         # category by site
 248         #bysite = {}
 249         #for nodename in l_nodes:
 250         #       if 'plcsite' in d_n[nodename]['values'] and \
 251         #       'login_base' in d_n[nodename]['values']['plcsite']:
 252         #               loginbase = d_n[nodename]['values']['plcsite']['login_base']
 253         #               if loginbase not in bysite:
 254         #                       bysite[loginbase] = []
 255         #               d_n[nodename]['values']['nodename'] = nodename
 256         #               bysite[loginbase].append(d_n[nodename]['values'])
 257
 258         # d2 was an array of [{node}, {}, ...]
 259         # the bysite is a loginbase dict of [{node}, {node}]
 260         d2 = []
 261         for nodename in l_nodes:
 262                 vals=d_n[nodename]['values']
 263                 v = {}
 264                 v.update(vals)
 265                 v['nodename'] = nodename
 266                 if  'plcsite' in vals and  \
 267                         'status' in vals['plcsite'] and  \
 268                         vals['plcsite']['status'] == "SUCCESS":
 269
 270                         url = "<a href='printbadnodes.py?site=%s'>%s</a>" % ( vals['plcsite']['login_base'],
 271                                                                                                                          vals['plcsite']['login_base'])
 272
 273                         site_string = "%s %2s nodes :: %2s of %4s slices" % ( \
 274                                                                                                                 url,
 275                                                                                                                 vals['plcsite']['num_nodes'],
 276                                                                                                                 vals['plcsite']['num_slices'],
 277                                                                                                                 vals['plcsite']['max_slices'])
 278                         loginbase = d_n[nodename]['values']['plcsite']['login_base']
 279                 else:
 280                         #print "ERROR: ", nodename, vals, "<br>"
 281                         site_string = "<b>UNKNOWN</b>"
 282                         loginbase = ""
 283
 284                 v['site_string'] = site_string
 285                 v['loginbase'] = loginbase
 286                 if (sitefilter != None and loginbase == sitefilter) or sitefilter == None:
 287                         d2.append(v)
 288
 289
 290         if sitefilter != None:
 291                 config.cmpcategory = True
 292         else:
 293                 config.cmploginbase = True
 294
 295
 296         if config.cmploginbase:
 297                 d2.sort(cmp=cmpLoginBase)
 298         elif config.cmpping:
 299                 d2.sort(cmp=cmpPing)
 300         elif config.cmpdns:
 301                 d2.sort(cmp=cmpDNS)
 302         elif config.cmpssh:
 303                 d2.sort(cmp=cmpSSH)
 304         elif config.cmpcategory:
 305                 d2.sort(cmp=cmpCategory)
 306         elif config.cmpstate:
 307                 d2.sort(cmp=cmpState)
 308         elif config.cmpdays:
 309                 d2.sort(cmp=cmpDays)
 310         elif config.cmpkernel:
 311                 d2.sort(cmp=cmpUname)
 312         else:
 313                 d2.sort(cmp=cmpCategory)
 314
 315
 316         #l_loginbase = bysite.keys()
 317         #l_loginbase.sort()
 318         print "<table width=80% border=1>"
 319         prev_sitestring = ""
 320         for row in d2:
 321
 322                 site_string = row['site_string']
 323                 if site_string != prev_sitestring:
 324                         print "<tr><td bgcolor=lightblue nowrap>"
 325                         print site_string
 326                         print "</td>"
 327                 else:
 328                         print "<tr><td>&nbsp;</td>"
 329
 330                 prev_sitestring = site_string
 331
 332                 vals = row
 333                 # convert uname values into a single kernel version string
 334                 if 'kernel' in vals:
 335                         kernel = vals['kernel'].split()
 336                         if len(kernel) > 0:
 337                                 if kernel[0] == "Linux":
 338                                         vals['kernel'] = kernel[2]
 339                                 else:
 340                                         vals['ssherror'] = vals['kernel']
 341                                         vals['kernel'] = ""
 342                 else:
 343                         vals['ssherror'] = ""
 344                         vals['kernel'] = ""
 345 #                       continue
 346                 if 'model' in vals or 'protocol' in vals or 'portstatus' in vals:
 347                         #vals['model'] = string.replace(vals['model']," ", "&nbsp;")
 348                         #vals['protocol'] = vals['protocol'].replace(" ", "&nbsp;")
 349                         if vals['model'] == None:
 350                                 vals['model'] = " "
 351                         vals['model'] = string.replace(vals['model']," ", "_")
 352                         vals['protocol'] = vals['protocol'].replace(" ", "_")
 353                         ps = ""
 354                         ports = vals['portstatus']
 355                         lports = ports.keys()
 356                         lports.sort()
 357                         for port in lports:
 358                                 t = ports[port]
 359                                 if t != "closed":
 360                                         ps += "%s:&nbsp;%s<br>" % (port, ports[port])
 361                         if ps == "":
 362                                 ps = "All_closed"
 363
 364                         vals['portstatus'] = ps
 365
 366                 if 'reboot' in vals:
 367                         vals['reboot'] = "%s" % vals['reboot']
 368                         vals['reboot'] = vals['reboot'].replace(" ", "_")
 369
 370                 try:
 371                         str_fields = []
 372                         count = 0
 373                         for f in format_fields:
 374                                 str_fields.append(f % vals)
 375                                 count += 1
 376                 except:
 377                         print >>sys.stderr, vals
 378
 379                 s = fields_to_html(str_fields, vals)
 380                 print s
 381
 382                 print "\n</tr>"
 383
 384         print "</table>"
 385         print "<table>"
 386         keys = categories.keys()
 387         keys.sort()
 388         for cat in keys:
 389                 print "<tr>"
 390                 print "<th nowrap align=left>Total %s</th>" % cat
 391                 print "<td align=left>%s</td>" % categories[cat]
 392                 print "</tr>"
 393         print "</table>"
 394
 395 import cgi
 396 import cgitb;
 397 cgitb.enable()
 398 import sys
 399
 400 form = cgi.FieldStorage()
 401 myfilter = None
 402
 403 if form.has_key('site'):
 404         myfilter = form.getvalue("site")
 405 else:
 406         myfilter = None
 407
 408
 409 if __name__ == '__main__':
 410         parser = OptionParser()
 411         parser.set_defaults(cmpdays=False,
 412                                                 comon="sshstatus",
 413                                                 fields="nodename,ping,ssh,pcu,category,state,kernel,bootcd",
 414                                                 dbname="findbad", # -070724-1",
 415                                                 cmpping=False,
 416                                                 cmpdns=False,
 417                                                 cmploginbase=False,
 418                                                 cmpssh=False,
 419                                                 cmpcategory=False,
 420                                                 cmpstate=False)
 421         parser.add_option("", "--fields",       dest="fields", help="")
 422         parser.add_option("", "--dbname",       dest="dbname", help="")
 423         parser.add_option("", "--days",         dest="cmpdays", action="store_true", help="")
 424         parser.add_option("", "--ping",         dest="cmpping", action="store_true", help="")
 425         parser.add_option("", "--dns",          dest="cmpdns", action="store_true", help="")
 426         parser.add_option("", "--ssh",          dest="cmpssh",  action="store_true", help="")
 427         parser.add_option("", "--loginbase",dest="cmploginbase",action="store_true", help="")
 428         parser.add_option("", "--category",     dest="cmpcategory", action="store_true", help="")
 429         parser.add_option("", "--kernel",       dest="cmpkernel", action="store_true", help="")
 430         parser.add_option("", "--state",        dest="cmpstate", action="store_true", help="")
 431         parser.add_option("", "--comon",        dest="comon",   help="")
 432         config = config(parser)
 433         config.parse_args()
 434         print "Content-Type: text/html\r\n"
 435         print "<html><body>\n"
 436         if len(sys.argv) > 1:
 437                 if sys.argv[1] == "ssherror":
 438                         ssherror = True
 439         main(myfilter)
 440         print "</body></html>\n"