Added 'FORCED' to handle some special actions
[monitor.git] / www / printbadnodes.py
1 #!/usr/bin/python
2 import soltesz
3 from config import config
4 from optparse import OptionParser
5 import string
6
7 import sys
8
9 categories = {}
10 ssherror = False
11
12 def sec2days(sec):
13         if sec == "null":
14                 sec = -(60*60*24)
15         sec = int(sec)
16         return sec/(60*60*24)
17
18 def array_to_priority_map(array):
19         """ Create a mapping where each entry of array is given a priority equal
20         to its position in the array.  This is useful for subsequent use in the
21         cmpMap() function."""
22         map = {}
23         count = 0
24         for i in array:
25                 map[i] = count
26                 count += 1
27         return map
28
29 def cmpValMap(v1, v2, map):
30         if v1 in map and v2 in map and map[v1] < map[v2]:
31                 return 1
32         elif v1 in map and v2 in map and map[v1] > map[v2]:
33                 return -1
34         elif v1 in map and v2 in map:
35                 return 0
36         else:
37                 raise Exception("No index %s or %s in map" % (v1, v2))
38
39 def cmpMap(l1, l2, index, map):
40         if index in l1 and index in l2:
41                 if map[l1[index]] < map[l2[index]]:
42                         return -1
43                 elif map[l1[index]] > map[l2[index]]:
44                         return 1
45                 else:
46                         return 0
47         else:
48                 return 0
49
50 def cmpLoginBase(l1, l2):
51         #print "'" + l1['loginbase'] + "'"  + " < " + "'" + l2['loginbase'] + "'" + "<BR>"
52         if l1['loginbase'] == l2['loginbase']:
53                 return 0
54         elif l1['loginbase'] < l2['loginbase']:
55                 return -1
56         elif l1['loginbase'] > l2['loginbase']:
57                 return 1
58         else:
59                 return 0
60
61 def cmpState(l1, l2):
62         map = array_to_priority_map([ 'BOOT', 'DEBUG', 'DOWN' ])
63         return cmpMap(l1,l2,'state', map)
64
65 def cmpCategoryVal(v1, v2):
66         map = array_to_priority_map([ None, 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
67         return cmpValMap(v1,v2,map)
68
69 def cmpCategory(l1, l2):
70         map = array_to_priority_map([ 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'ERROR', ])
71         return cmpMap(l1,l2,'category', map)
72
73 def cmpPCU(l1, l2):
74         """ Either PCU or NOPCU"""
75         map = array_to_priority_map([ 'PCU', 'NOPCU', 'UNKNOWN'])
76         return cmpMap(l1, l2, 'pcu', map)
77
78 def cmpSSH(l1, l2):
79         """ Either SSH or NOSSH """
80         map = array_to_priority_map([ 'SSH', 'NOSSH'])
81         return cmpMap(l1, l2, 'ssh', map)
82
83 def cmpDNS(l1,l2):
84         """ Compare DNS states """
85         map = array_to_priority_map([ 'OK', 'NOHOSTNAME', 'NOENTRY', 'MISMATCH'])
86         return cmpMap(l1, l2, 'dnsmatch', map)
87         
88 def cmpPing(l1,l2):
89         """ Either PING or NOPING """
90         map = array_to_priority_map([ 'PING', 'NOPING'])
91         return cmpMap(l1, l2, 'ping', map)
92
93 def cmpUname(l1, l2):
94         # Extract the kernel version from kernel -a string
95         l_k1 = l1['kernel'].split()
96         if len(l_k1) > 2:
97                 k1 = l_k1[2]
98         else:
99                 return 1
100
101         l_k2 = l2['kernel'].split()
102         if len(l_k2) > 2:
103                 k2 = l_k2[2]
104         else:
105                 return -1
106
107         return cmp(k1, k2)
108
109 def cmpDays(l1, l2):
110         if l1['comonstats'][config.comon] == "null":
111                 l1['comonstats'][config.comon] = -1
112         if l2['comonstats'][config.comon] == "null":
113                 l2['comonstats'][config.comon] = -1
114                 
115         if int(l1['comonstats'][config.comon]) > int(l2['comonstats'][config.comon]):
116                 return -1
117         elif int(l1['comonstats'][config.comon]) < int(l2['comonstats'][config.comon]):
118                 return 1
119         else:
120                 return 0
121
122 def ssh_error_to_str(str):
123         ssh_error = ""
124         if "Connection timed out" in str:
125                 ssh_error = "Timeout" 
126         elif "Connection closed by remote host" in str:
127                 ssh_error = "Closed by remote host"
128         elif "Connection refused" in str:
129                 ssh_error = "Connection refused"
130         elif "Temporary failure in name resolution" in str:
131                 ssh_error = "Could not resolve name"
132         elif "Name or service not known" in str:
133                 ssh_error = "Name not known"
134         elif "Too many authentication failures" in str:
135                 ssh_error = "Disconnect: root auth failure"
136         elif "Network is unreachable" in str:
137                 ssh_error = "Network is unreachable"
138         elif "Connection reset by peer" in str:
139                 ssh_error = "Connection reset by peer"
140         elif "WARNING" in str:
141                 ssh_error = "WARNING ssh key updated"
142         else:
143                 ssh_error = str
144
145         return ssh_error
146
147 def fields_to_html(fields, vals):
148         global categories
149         global ssherror
150         colorMap = { 'PING'  : 'darkseagreen',
151                                  'NOPING': 'darksalmon',
152                                  'SSH': 'darkseagreen',
153                                  'NOSSH': 'indianred',
154                                  'PCU': 'darkseagreen',
155                                  'NOPCU': 'lightgrey',
156                                  'OLDBOOTCD': 'crimson',
157                                  'DOWN': 'indianred',
158                                  'ALPHA': 'gold',
159                                  'ERROR': 'crimson',
160                                  'PROD': 'darkseagreen',
161                                  'DEBUG': 'darksalmon',
162                                  'DEBUG': 'darksalmon',
163                                  'BOOT': 'lightgreen'}
164         r_str = ""
165         f_prev = ""
166         f_2prev = ""
167         #print 'inside--------------'
168         for f in fields:
169                 f = f.strip()
170                 #print f
171
172                 if f in ['DOWN', 'BOOT', 'DEBUG']:
173                         #key = "%s-%s-%s" % (f,f_prev,f_2prev)
174                         key = "%s-%s" % (f,f_prev)
175                         if key not in categories:
176                                 categories[key] = 1
177                         else:
178                                 categories[key] += 1
179
180                 #print "<pre>%s</pre><br>" % f
181                                 
182                 if f in colorMap:
183                         bgcolor="bgcolor='%s'" % colorMap[f]
184                 else:
185                         bgcolor=""
186
187                 if f == 'NOSSH':
188                         if ssherror:
189                                 if 'ssherror' in vals:
190                                         str_ssh_error = ssh_error_to_str(vals['ssherror'])
191                                 else:
192                                         str_ssh_error = "NO SSHERROR in VALS"
193                                 if str_ssh_error != "Timeout":
194                                         r_str += """<td nowrap %s>%s<br><b><font size="-2">%s</font></b></td>""" % \
195                                                                 (bgcolor,f,str_ssh_error)
196                                 else:
197                                         r_str += "<td %s>%s</td>" % (bgcolor, f)
198                         else:
199                                 r_str += "<td %s>%s</td>" % (bgcolor, f)
200                 elif f == 'PCU':
201                         if len(vals['plcnode']['pcu_ids']) > 0:
202                                 url = "<a href='/cgi-bin/printbadpcus.php#id%s'>PCU</a>" % vals['plcnode']['pcu_ids'][0]
203                                 r_str += "<td nowrap %s>%s</td>" % (bgcolor, url)
204                 else:
205                         r_str += "<td nowrap %s>%s</td>" % (bgcolor, f)
206                 f_2prev = f_prev
207                 f_prev  = f
208         
209         return r_str
210
211
212
213 def main(sitefilter):
214         db = soltesz.dbLoad(config.dbname)
215
216         ## Field widths used for printing
217         maxFieldLengths = { 'nodename' : -45,
218                                                 'ping' : 6, 
219                                                 'ssh' : 6, 
220                                                 'pcu' : 7, 
221                                                 'category' : 9, 
222                                                 'state' : 5, 
223                                                 'kernel' : 10.65, 
224                                                 'comonstats' : 5, 
225                                                 'plcsite' : 12,
226                                                 'bootcd' : 10.65}
227         ## create format string based on config.fields
228         fields = {}
229         format = ""
230         format_fields = []
231         for f in config.fields.split(','):
232                 fields[f] = "%%(%s)s" % f
233                 #if f in maxFieldLengths:
234                 #       fields[f] = "%%(%s)%ds" % (f, maxFieldLengths[f])
235                 #else:
236                 #       fields[f] = "%%(%s)%ds" % (f, 10)
237
238                 format_fields.append(fields[f])
239         #print fields
240         for f in config.fields.split(','):
241                 format += fields[f] + " "
242         #print format
243
244         d_n = db['nodes']
245         l_nodes = d_n.keys()
246
247         # category by site
248         #bysite = {}
249         #for nodename in l_nodes:
250         #       if 'plcsite' in d_n[nodename]['values'] and \
251         #       'login_base' in d_n[nodename]['values']['plcsite']:
252         #               loginbase = d_n[nodename]['values']['plcsite']['login_base']
253         #               if loginbase not in bysite:
254         #                       bysite[loginbase] = []
255         #               d_n[nodename]['values']['nodename'] = nodename
256         #               bysite[loginbase].append(d_n[nodename]['values'])
257
258         # d2 was an array of [{node}, {}, ...]
259         # the bysite is a loginbase dict of [{node}, {node}]
260         d2 = []
261         import re
262         if sitefilter != None:
263                 sf = re.compile(sitefilter)
264         else:
265                 sf = None
266         for nodename in l_nodes: 
267                 vals=d_n[nodename]['values'] 
268                 v = {}
269                 v.update(vals)
270                 v['nodename'] = nodename 
271                 if  'plcsite' in vals and  \
272                         'status' in vals['plcsite'] and  \
273                         vals['plcsite']['status'] == "SUCCESS":
274
275                         url = "<a href='printbadnodes.py?site=%s'>%s</a>" % ( vals['plcsite']['login_base'],
276                                                                                                                          vals['plcsite']['login_base'])
277
278                         site_string = "%s %2s nodes :: %2s of %4s slices" % ( \
279                                                                                                                 url,
280                                                                                                                 vals['plcsite']['num_nodes'], 
281                                                                                                                 vals['plcsite']['num_slices'], 
282                                                                                                                 vals['plcsite']['max_slices'])
283                         loginbase = d_n[nodename]['values']['plcsite']['login_base']
284                 else:
285                         #print "ERROR: ", nodename, vals, "<br>"
286                         site_string = "<b>UNKNOWN</b>"
287                         loginbase = ""
288
289                 v['site_string'] = site_string
290                 v['loginbase'] = loginbase
291                 if (sitefilter != None and sf.match(loginbase) != None) or sitefilter == None:
292                         d2.append(v)
293                         
294
295         if sitefilter != None:
296                 config.cmpcategory = True
297         else:
298                 config.cmploginbase = True
299                 
300
301         if config.cmploginbase:
302                 d2.sort(cmp=cmpLoginBase)
303         elif config.cmpping:
304                 d2.sort(cmp=cmpPing)
305         elif config.cmpdns:
306                 d2.sort(cmp=cmpDNS)
307         elif config.cmpssh:
308                 d2.sort(cmp=cmpSSH)
309         elif config.cmpcategory:
310                 d2.sort(cmp=cmpCategory)
311         elif config.cmpstate:
312                 d2.sort(cmp=cmpState)
313         elif config.cmpdays:
314                 d2.sort(cmp=cmpDays)
315         elif config.cmpkernel:
316                 d2.sort(cmp=cmpUname)
317         else:
318                 d2.sort(cmp=cmpCategory)
319         
320
321         #l_loginbase = bysite.keys()
322         #l_loginbase.sort()
323         print "<table width=80% border=1>"
324         prev_sitestring = ""
325         for row in d2:
326
327                 site_string = row['site_string']
328                 if site_string != prev_sitestring:
329                         print "<tr><td bgcolor=lightblue nowrap>" 
330                         print site_string
331                         print "</td>"
332                 else:
333                         print "<tr><td>&nbsp;</td>"
334
335                 prev_sitestring = site_string
336
337                 vals = row
338                 # convert uname values into a single kernel version string
339                 if 'kernel' in vals:
340                         kernel = vals['kernel'].split()
341                         if len(kernel) > 0:
342                                 if kernel[0] == "Linux":
343                                         vals['kernel'] = kernel[2]
344                                 else:
345                                         vals['ssherror'] = vals['kernel']
346                                         vals['kernel'] = ""
347                 else:
348                         vals['ssherror'] = ""
349                         vals['kernel'] = ""
350 #                       continue
351                 if 'model' in vals or 'protocol' in vals or 'portstatus' in vals:
352                         #vals['model'] = string.replace(vals['model']," ", "&nbsp;")
353                         #vals['protocol'] = vals['protocol'].replace(" ", "&nbsp;")
354                         if vals['model'] == None:
355                                 vals['model'] = " "
356                         vals['model'] = string.replace(vals['model']," ", "_")
357                         vals['protocol'] = vals['protocol'].replace(" ", "_")
358                         ps = ""
359                         ports = vals['portstatus']
360                         lports = ports.keys()
361                         lports.sort()
362                         for port in lports:
363                                 t = ports[port]
364                                 if t != "closed":
365                                         ps += "%s:&nbsp;%s<br>" % (port, ports[port])
366                         if ps == "":
367                                 ps = "All_closed"
368                                 
369                         vals['portstatus'] = ps
370
371                 if 'reboot' in vals:
372                         vals['reboot'] = "%s" % vals['reboot']
373                         vals['reboot'] = vals['reboot'].replace(" ", "_")
374
375                 try:
376                         str_fields = []
377                         count = 0
378                         for f in format_fields:
379                                 str_fields.append(f % vals)
380                                 count += 1
381                 except:
382                         print >>sys.stderr, vals
383
384                 s = fields_to_html(str_fields, vals)
385                 print s
386                         
387                 print "\n</tr>"
388
389         print "</table>"
390         print "<table>"
391         keys = categories.keys()
392         keys.sort()
393         for cat in keys:
394                 print "<tr>"
395                 print "<th nowrap align=left>Total %s</th>" % cat
396                 print "<td align=left>%s</td>" % categories[cat]
397                 print "</tr>"
398         print "</table>"
399
400
401
402 if __name__ == '__main__':
403         import cgi
404         import cgitb; 
405         cgitb.enable()
406         import sys
407
408         form = cgi.FieldStorage()
409         myfilter = None
410
411         if form.has_key('site'):
412                 myfilter = form.getvalue("site")
413         else:
414                 myfilter = None
415         parser = OptionParser()
416         parser.set_defaults(cmpdays=False, 
417                                                 comon="sshstatus", 
418                                                 fields="nodename,ping,ssh,pcu,category,state,kernel,bootcd", 
419                                                 dbname="findbad", # -070724-1", 
420                                                 cmpping=False, 
421                                                 cmpdns=False, 
422                                                 cmploginbase=False, 
423                                                 cmpssh=False, 
424                                                 cmpcategory=False,
425                                                 cmpstate=False)
426         parser.add_option("", "--fields",       dest="fields", help="")
427         parser.add_option("", "--dbname",       dest="dbname", help="")
428         parser.add_option("", "--days",         dest="cmpdays", action="store_true", help="")
429         parser.add_option("", "--ping",         dest="cmpping", action="store_true", help="")
430         parser.add_option("", "--dns",          dest="cmpdns", action="store_true", help="")
431         parser.add_option("", "--ssh",          dest="cmpssh",  action="store_true", help="")
432         parser.add_option("", "--loginbase",dest="cmploginbase",action="store_true", help="")
433         parser.add_option("", "--category",     dest="cmpcategory", action="store_true", help="")
434         parser.add_option("", "--kernel",       dest="cmpkernel", action="store_true", help="")
435         parser.add_option("", "--state",        dest="cmpstate", action="store_true", help="")
436         parser.add_option("", "--comon",        dest="comon",   help="")
437         config = config(parser)
438         config.parse_args()
439         print "Content-Type: text/html\r\n"
440         print "<html><body>\n"
441         if len(sys.argv) > 1:
442                 if sys.argv[1] == "ssherror":
443                         ssherror = True
444         main(myfilter)
445         print "</body></html>\n"