www/printbadnodes.py

   1 #!/usr/bin/python
   2 import soltesz
   3 from config import config
   4 from optparse import OptionParser
   5 import string
   6 #from HyperText.HTML import A, BR, IMG, TABLE, TR, TH, TD, EM, quote_body
   7 #from HyperText.Documents import Document
   8
   9 import sys
  10
  11 categories = {}
  12 ssherror = False
  13 fb = {}
  14
  15 def sec2days(sec):
  16         if sec == "null":
  17                 sec = -(60*60*24)
  18         sec = int(sec)
  19         return sec/(60*60*24)
  20
  21 def array_to_priority_map(array):
  22         """ Create a mapping where each entry of array is given a priority equal
  23         to its position in the array.  This is useful for subsequent use in the
  24         cmpMap() function."""
  25         map = {}
  26         count = 0
  27         for i in array:
  28                 map[i] = count
  29                 count += 1
  30         return map
  31
  32 def cmpValMap(v1, v2, map):
  33         if v1 in map and v2 in map and map[v1] < map[v2]:
  34                 return 1
  35         elif v1 in map and v2 in map and map[v1] > map[v2]:
  36                 return -1
  37         elif v1 in map and v2 in map:
  38                 return 0
  39         else:
  40                 raise Exception("No index %s or %s in map" % (v1, v2))
  41
  42 def cmpMap(l1, l2, index, map):
  43         if index in l1 and index in l2:
  44                 if map[l1[index]] < map[l2[index]]:
  45                         return -1
  46                 elif map[l1[index]] > map[l2[index]]:
  47                         return 1
  48                 else:
  49                         return 0
  50         else:
  51                 return 0
  52
  53 def cmpLoginBase(l1, l2):
  54         #print "'" + l1['loginbase'] + "'"  + " < " + "'" + l2['loginbase'] + "'" + "<BR>"
  55         if l1['loginbase'] == l2['loginbase']:
  56                 return 0
  57         elif l1['loginbase'] < l2['loginbase']:
  58                 return -1
  59         elif l1['loginbase'] > l2['loginbase']:
  60                 return 1
  61         else:
  62                 return 0
  63
  64 def cmpState(l1, l2):
  65         map = array_to_priority_map([ 'BOOT', 'DEBUG', 'DOWN' ])
  66         return cmpMap(l1,l2,'state', map)
  67
  68 def cmpCategoryVal(v1, v2):
  69         map = array_to_priority_map([ None, 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
  70         return cmpValMap(v1,v2,map)
  71
  72 def cmpCategory(l1, l2):
  73         map = array_to_priority_map([ 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'ERROR', ])
  74         return cmpMap(l1,l2,'category', map)
  75
  76 def cmpPCU(l1, l2):
  77         """ Either PCU or NOPCU"""
  78         map = array_to_priority_map([ 'PCU', 'NOPCU', 'UNKNOWN'])
  79         return cmpMap(l1, l2, 'pcu', map)
  80
  81 def cmpSSH(l1, l2):
  82         """ Either SSH or NOSSH """
  83         map = array_to_priority_map([ 'SSH', 'NOSSH'])
  84         return cmpMap(l1, l2, 'ssh', map)
  85
  86 def cmpDNS(l1,l2):
  87         """ Compare DNS states """
  88         map = array_to_priority_map([ 'OK', 'NOHOSTNAME', 'NOENTRY', 'MISMATCH'])
  89         return cmpMap(l1, l2, 'dnsmatch', map)
  90
  91 def cmpPing(l1,l2):
  92         """ Either PING or NOPING """
  93         map = array_to_priority_map([ 'PING', 'NOPING'])
  94         return cmpMap(l1, l2, 'ping', map)
  95
  96 def cmpUname(l1, l2):
  97         # Extract the kernel version from kernel -a string
  98         l_k1 = l1['kernel'].split()
  99         if len(l_k1) > 2:
 100                 k1 = l_k1[2]
 101         else:
 102                 return 1
 103
 104         l_k2 = l2['kernel'].split()
 105         if len(l_k2) > 2:
 106                 k2 = l_k2[2]
 107         else:
 108                 return -1
 109
 110         return cmp(k1, k2)
 111
 112 def cmpDays(l1, l2):
 113         if l1['comonstats'][config.comon] == "null":
 114                 l1['comonstats'][config.comon] = -1
 115         if l2['comonstats'][config.comon] == "null":
 116                 l2['comonstats'][config.comon] = -1
 117
 118         if int(l1['comonstats'][config.comon]) > int(l2['comonstats'][config.comon]):
 119                 return -1
 120         elif int(l1['comonstats'][config.comon]) < int(l2['comonstats'][config.comon]):
 121                 return 1
 122         else:
 123                 return 0
 124
 125 def ssh_error_to_str(str):
 126         ssh_error = ""
 127         if "Connection timed out" in str:
 128                 ssh_error = "Timeout"
 129         elif "Connection closed by remote host" in str:
 130                 ssh_error = "Closed by remote host"
 131         elif "Connection refused" in str:
 132                 ssh_error = "Connection refused"
 133         elif "Temporary failure in name resolution" in str:
 134                 ssh_error = "Could not resolve name"
 135         elif "Name or service not known" in str:
 136                 ssh_error = "Name not known"
 137         elif "Too many authentication failures" in str:
 138                 ssh_error = "Disconnect: root auth failure"
 139         elif "Network is unreachable" in str:
 140                 ssh_error = "Network is unreachable"
 141         elif "Connection reset by peer" in str:
 142                 ssh_error = "Connection reset by peer"
 143         elif "WARNING" in str:
 144                 ssh_error = "WARNING ssh key updated"
 145         else:
 146                 ssh_error = str
 147
 148         return ssh_error
 149
 150 def pcu_state(pcu_id):
 151         global fb
 152
 153         if 'nodes' in fb and "id_%s" % pcu_id in fb['nodes'] \
 154                 and 'values' in fb['nodes']["id_%s" % pcu_id]:
 155                 rec = fb['nodes']["id_%s" % pcu_id]['values']
 156                 if 'reboot' in rec:
 157                         rb = rec['reboot']
 158                         if rb == 0 or rb == "0":
 159                                 return 0
 160                         elif "NetDown" == rb  or "Not_Run" == rb:
 161                                 return 1
 162                         else:
 163                                 return -1
 164                 else:
 165                         return -1
 166         else:
 167                 return -1
 168
 169 def fields_to_html(fields, vals):
 170         global categories
 171         global ssherror
 172         pcu_colorMap = { -1 : 'indianred',
 173                                           0 : 'darkseagreen',
 174                                           1 : 'gold', }
 175
 176         colorMap = { 'PING'  : 'darkseagreen',
 177                                  'NOPING': 'darksalmon',
 178                                  'SSH': 'darkseagreen',
 179                                  'NOSSH': 'indianred',
 180                                  'PCU': 'darkseagreen',
 181                                  'NOPCU': 'lightgrey',
 182                                  'OLDBOOTCD': 'crimson',
 183                                  'DOWN': 'indianred',
 184                                  'ALPHA': 'gold',
 185                                  'ERROR': 'crimson',
 186                                  'PROD': 'darkseagreen',
 187                                  'DEBUG': 'darksalmon',
 188                                  'DEBUG': 'darksalmon',
 189                                  'BOOT': 'lightgreen'}
 190         r_str = ""
 191         f_prev = ""
 192         f_2prev = ""
 193         #print 'inside--------------'
 194         for f in fields:
 195                 f = f.strip()
 196                 #print f
 197
 198                 if f in ['DOWN', 'BOOT', 'DEBUG']:
 199                         #key = "%s-%s-%s" % (f,f_prev,f_2prev)
 200                         key = "%s-%s" % (f,f_prev)
 201                         if key not in categories:
 202                                 categories[key] = 1
 203                         else:
 204                                 categories[key] += 1
 205
 206                 #print "<pre>%s</pre><br>" % f
 207
 208                 if f in colorMap:
 209                         bgcolor="bgcolor='%s'" % colorMap[f]
 210                 else:
 211                         bgcolor=""
 212
 213                 if f == 'NOSSH':
 214                         if ssherror:
 215                                 if 'ssherror' in vals:
 216                                         str_ssh_error = ssh_error_to_str(vals['ssherror'])
 217                                 else:
 218                                         str_ssh_error = "NO SSHERROR in VALS"
 219                                 if str_ssh_error != "Timeout":
 220                                         r_str += """<td nowrap %s>%s<br><b><font size="-2">%s</font></b></td>""" % \
 221                                                                 (bgcolor,f,str_ssh_error)
 222                                 else:
 223                                         r_str += "<td %s>%s</td>" % (bgcolor, f)
 224                         else:
 225                                 r_str += "<td %s>%s</td>" % (bgcolor, f)
 226                 elif f == 'PCU':
 227                         if len(vals['plcnode']['pcu_ids']) > 0:
 228                                 #print "pcu_id: %s<br>" % vals['plcnode']['pcu_ids'][0]
 229                                 #print "state: %s<br>" % pcu_state(vals['plcnode']['pcu_ids'][0])
 230                                 #print "color: %s<br>" % pcu_colorMap[pcu_state(vals['plcnode']['pcu_ids'][0])]
 231                                 bgcolor = "bgcolor='%s'" % pcu_colorMap[pcu_state(vals['plcnode']['pcu_ids'][0])]
 232                                 url = "<a href='/cgi-bin/printbadpcus.php#id%s'>PCU</a>" % vals['plcnode']['pcu_ids'][0]
 233                                 r_str += "<td nowrap %s>%s</td>" % (bgcolor, url)
 234                 else:
 235                         r_str += "<td nowrap %s>%s</td>" % (bgcolor, f)
 236                 f_2prev = f_prev
 237                 f_prev  = f
 238
 239         return r_str
 240
 241
 242
 243 def main(sitefilter, catfilter, statefilter, comonfilter, nodeonlyfilter):
 244         global fb
 245
 246         db = soltesz.dbLoad(config.dbname)
 247         fb = soltesz.dbLoad("findbadpcus")
 248
 249         ## Field widths used for printing
 250         maxFieldLengths = { 'nodename' : -45,
 251                                                 'ping' : 6,
 252                                                 'ssh' : 6,
 253                                                 'pcu' : 7,
 254                                                 'category' : 9,
 255                                                 'state' : 5,
 256                                                 'kernel' : 10.65,
 257                                                 'comonstats' : 5,
 258                                                 'plcsite' : 12,
 259                                                 'bootcd' : 10.65}
 260         ## create format string based on config.fields
 261         fields = {}
 262         format = ""
 263         format_fields = []
 264         for f in config.fields.split(','):
 265                 fields[f] = "%%(%s)s" % f
 266                 #if f in maxFieldLengths:
 267                 #       fields[f] = "%%(%s)%ds" % (f, maxFieldLengths[f])
 268                 #else:
 269                 #       fields[f] = "%%(%s)%ds" % (f, 10)
 270
 271                 format_fields.append(fields[f])
 272         #print fields
 273         for f in config.fields.split(','):
 274                 format += fields[f] + " "
 275         #print format
 276
 277         d_n = db['nodes']
 278         l_nodes = d_n.keys()
 279
 280         # category by site
 281         #bysite = {}
 282         #for nodename in l_nodes:
 283         #       if 'plcsite' in d_n[nodename]['values'] and \
 284         #       'login_base' in d_n[nodename]['values']['plcsite']:
 285         #               loginbase = d_n[nodename]['values']['plcsite']['login_base']
 286         #               if loginbase not in bysite:
 287         #                       bysite[loginbase] = []
 288         #               d_n[nodename]['values']['nodename'] = nodename
 289         #               bysite[loginbase].append(d_n[nodename]['values'])
 290
 291         # d2 was an array of [{node}, {}, ...]
 292         # the bysite is a loginbase dict of [{node}, {node}]
 293         d2 = []
 294         import re
 295         if sitefilter != None:
 296                 sf = re.compile(sitefilter)
 297         else:
 298                 sf = None
 299         for nodename in l_nodes:
 300                 vals=d_n[nodename]['values']
 301                 v = {}
 302                 v.update(vals)
 303                 v['nodename'] = nodename
 304                 if  'plcsite' in vals and  \
 305                         'status' in vals['plcsite'] and  \
 306                         vals['plcsite']['status'] == "SUCCESS":
 307
 308                         url = "<a href='printbadnodes.py?site=%s'>%s</a>" % ( vals['plcsite']['login_base'],
 309                                                                                                                          vals['plcsite']['login_base'])
 310
 311                         site_string = "%s %2s nodes :: %2s of %4s slices" % ( \
 312                                                                                                                 url,
 313                                                                                                                 vals['plcsite']['num_nodes'],
 314                                                                                                                 vals['plcsite']['num_slices'],
 315                                                                                                                 vals['plcsite']['max_slices'])
 316                         loginbase = d_n[nodename]['values']['plcsite']['login_base']
 317                 else:
 318                         #print "ERROR: ", nodename, vals, "<br>"
 319                         site_string = "<b>UNKNOWN</b>"
 320                         loginbase = ""
 321
 322                 v['site_string'] = site_string
 323                 v['loginbase'] = loginbase
 324                 if (sitefilter != None and sf.match(loginbase) != None) or sitefilter == None:
 325                         d2.append(v)
 326
 327
 328         if sitefilter != None:
 329                 config.cmpcategory = True
 330         else:
 331                 config.cmploginbase = True
 332
 333
 334         if config.cmploginbase:
 335                 d2.sort(cmp=cmpLoginBase)
 336         elif config.cmpping:
 337                 d2.sort(cmp=cmpPing)
 338         elif config.cmpdns:
 339                 d2.sort(cmp=cmpDNS)
 340         elif config.cmpssh:
 341                 d2.sort(cmp=cmpSSH)
 342         elif config.cmpcategory:
 343                 d2.sort(cmp=cmpCategory)
 344         elif config.cmpstate:
 345                 d2.sort(cmp=cmpState)
 346         elif config.cmpdays:
 347                 d2.sort(cmp=cmpDays)
 348         elif config.cmpkernel:
 349                 d2.sort(cmp=cmpUname)
 350         else:
 351                 d2.sort(cmp=cmpCategory)
 352
 353
 354         if catfilter != None:   cf = re.compile(catfilter)
 355         else:                                   cf = None
 356
 357         if statefilter != None: stf = re.compile(statefilter)
 358         else:                                   stf = None
 359
 360         if comonfilter != None: cmf = re.compile(comonfilter)
 361         else:                                   cmf = None
 362
 363         #l_loginbase = bysite.keys()
 364         #l_loginbase.sort()
 365         if nodeonlyfilter == None:
 366                 print "<table width=80% border=1>"
 367
 368         prev_sitestring = ""
 369         for row in d2:
 370
 371                 vals = row
 372
 373                 if (catfilter != None and cf.match(vals['category']) == None):
 374                         continue
 375
 376                 if (statefilter != None and stf.match(vals['state']) == None):
 377                         continue
 378
 379                 if (comonfilter != None and comonfilter in vals['comonstats'] and vals['comonstats'][comonfilter] != 'null'):
 380                         continue
 381
 382                 if nodeonlyfilter != None:
 383                         print vals['nodename']
 384                         continue
 385
 386                 site_string = row['site_string']
 387                 if site_string != prev_sitestring:
 388                         print "<tr><td bgcolor=lightblue nowrap>"
 389                         print site_string
 390                         print "</td>"
 391                 else:
 392                         print "<tr><td>&nbsp;</td>"
 393
 394                 prev_sitestring = site_string
 395
 396
 397                 # convert uname values into a single kernel version string
 398                 if 'kernel' in vals:
 399                         kernel = vals['kernel'].split()
 400                         if len(kernel) > 0:
 401                                 if kernel[0] == "Linux":
 402                                         vals['kernel'] = kernel[2]
 403                                 else:
 404                                         vals['ssherror'] = vals['kernel']
 405                                         vals['kernel'] = ""
 406                 else:
 407                         vals['ssherror'] = ""
 408                         vals['kernel'] = ""
 409 #                       continue
 410                 if 'model' in vals or 'protocol' in vals or 'portstatus' in vals:
 411                         #vals['model'] = string.replace(vals['model']," ", "&nbsp;")
 412                         #vals['protocol'] = vals['protocol'].replace(" ", "&nbsp;")
 413                         if vals['model'] == None:
 414                                 vals['model'] = " "
 415                         vals['model'] = string.replace(vals['model']," ", "_")
 416                         vals['protocol'] = vals['protocol'].replace(" ", "_")
 417                         ps = ""
 418                         ports = vals['portstatus']
 419                         lports = ports.keys()
 420                         lports.sort()
 421                         for port in lports:
 422                                 t = ports[port]
 423                                 if t != "closed":
 424                                         ps += "%s:&nbsp;%s<br>" % (port, ports[port])
 425                         if ps == "":
 426                                 ps = "All_closed"
 427
 428                         vals['portstatus'] = ps
 429
 430                 if 'reboot' in vals:
 431                         vals['reboot'] = "%s" % vals['reboot']
 432                         vals['reboot'] = vals['reboot'].replace(" ", "_")
 433
 434                 if 'nodename' in vals:
 435                         url = "<a href='https://www.planet-lab.org/db/nodes/index.php?nodepattern=%s'>%s</a>" % (vals['nodename'], vals['nodename'])
 436                         vals['nodename'] = url
 437
 438                 try:
 439                         str_fields = []
 440                         count = 0
 441                         for f in format_fields:
 442                                 str_fields.append(f % vals)
 443                                 count += 1
 444                 except:
 445                         print >>sys.stderr, vals
 446
 447                 s = fields_to_html(str_fields, vals)
 448                 print s
 449
 450                 print "\n</tr>"
 451
 452         if nodeonlyfilter == None:
 453                 print "</table>"
 454                 print "<table>"
 455         keys = categories.keys()
 456         keys.sort()
 457         for cat in keys:
 458                 print "<tr>"
 459                 print "<th nowrap align=left>Total %s</th>" % cat
 460                 print "<td align=left>%s</td>" % categories[cat]
 461                 print "</tr>"
 462         if nodeonlyfilter == None:
 463                 print "</table>"
 464
 465
 466
 467 if __name__ == '__main__':
 468         import cgi
 469         import cgitb;
 470         cgitb.enable()
 471         import sys
 472
 473         form = cgi.FieldStorage()
 474         myfilter = None
 475
 476         if form.has_key('site'):
 477                 myfilter = form.getvalue("site")
 478         else:
 479                 myfilter = None
 480
 481         if form.has_key('category'):
 482                 mycategory = form.getvalue("category")
 483         else:
 484                 mycategory = None
 485
 486         if form.has_key('state'):
 487                 mystate = form.getvalue("state")
 488         else:
 489                 mystate = None
 490
 491         if form.has_key('comon'):
 492                 mycomon = form.getvalue("comon")
 493         else:
 494                 mycomon = None
 495
 496         if form.has_key('nodeonly'):
 497                 mynodeonly = form.getvalue("nodeonly")
 498         else:
 499                 mynodeonly = None
 500
 501         parser = OptionParser()
 502         parser.set_defaults(cmpdays=False,
 503                                                 comon="sshstatus",
 504                                                 fields="nodename,ping,ssh,pcu,category,state,comonstats,kernel,bootcd",
 505                                                 dbname="findbad", # -070724-1",
 506                                                 cmpping=False,
 507                                                 cmpdns=False,
 508                                                 cmploginbase=False,
 509                                                 cmpssh=False,
 510                                                 cmpcategory=False,
 511                                                 cmpstate=False)
 512         parser.add_option("", "--fields",       dest="fields", help="")
 513         parser.add_option("", "--dbname",       dest="dbname", help="")
 514         parser.add_option("", "--days",         dest="cmpdays", action="store_true", help="")
 515         parser.add_option("", "--ping",         dest="cmpping", action="store_true", help="")
 516         parser.add_option("", "--dns",          dest="cmpdns", action="store_true", help="")
 517         parser.add_option("", "--ssh",          dest="cmpssh",  action="store_true", help="")
 518         parser.add_option("", "--loginbase",dest="cmploginbase",action="store_true", help="")
 519         parser.add_option("", "--category",     dest="cmpcategory", action="store_true", help="")
 520         parser.add_option("", "--kernel",       dest="cmpkernel", action="store_true", help="")
 521         parser.add_option("", "--state",        dest="cmpstate", action="store_true", help="")
 522         parser.add_option("", "--comon",        dest="comon",   help="")
 523         config = config(parser)
 524         config.parse_args()
 525         print "Content-Type: text/html\r\n"
 526         if mynodeonly == None:
 527                 print "<html><body>\n"
 528         if len(sys.argv) > 1:
 529                 if sys.argv[1] == "ssherror":
 530                         ssherror = True
 531         main(myfilter, mycategory, mystate, mycomon,mynodeonly)
 532         if mynodeonly == None:
 533                 print "</body></html>\n"