setup default values if they're not already set.
[monitor.git] / monitor / scanapi.py
1 #!/usr/bin/python
2
3 import os
4 import sys
5 import string
6 import time
7 from datetime import datetime,timedelta
8 import threadpool
9 import threading
10
11 import socket
12 from pcucontrol import reboot
13
14 from pcucontrol.util import command
15 from monitor import config
16
17 from monitor.database.info.model import *
18
19 from monitor.sources import comon
20 from monitor.wrapper import plc, plccache
21
22 import traceback
23 from monitor.common import nmap_port_status, email_exception
24
25 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
26                         "table=table_nodeview&" + \
27                         "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
28                         "formatcsv"
29
30 api = plc.getAuthAPI()
31 plc_lock = threading.Lock()
32 round = 1
33 global_round = round
34 count = 0
35
36
37 def get_pcu(pcuname):
38         plc_lock.acquire()
39         try:
40                 #print "GetPCU from PLC %s" % pcuname
41                 l_pcu  = plc.GetPCUs({'pcu_id' : pcuname})
42                 #print l_pcu
43                 if len(l_pcu) > 0:
44                         l_pcu = l_pcu[0]
45         except:
46                 try:
47                         #print "GetPCU from file %s" % pcuname
48                         l_pcus = plccache.l_pcus
49                         for i in l_pcus:
50                                 if i['pcu_id'] == pcuname:
51                                         l_pcu = i
52                 except:
53                         traceback.print_exc()
54                         l_pcu = None
55
56         plc_lock.release()
57         return l_pcu
58
59 def get_nodes(node_ids):
60         plc_lock.acquire()
61         l_node = []
62         try:
63                 l_node = plc.getNodes(node_ids, ['hostname', 'last_contact', 'node_id', 'ports'])
64         except:
65                 try:
66                         plc_nodes = plccache.l_nodes
67                         for n in plc_nodes:
68                                 if n['node_id'] in node_ids:
69                                         l_node.append(n)
70                 except:
71                         traceback.print_exc()
72                         l_node = None
73
74         plc_lock.release()
75         if l_node == []:
76                 l_node = None
77         return l_node
78         
79
80 def get_plc_pcu_values(pcuname):
81         """
82                 Try to contact PLC to get the PCU info.
83                 If that fails, try a backup copy from the last run.
84                 If that fails, return None
85         """
86         values = {}
87
88         l_pcu = get_pcu(pcuname)
89         
90         if l_pcu is not None:
91                 site_id = l_pcu['site_id']
92                 node_ids = l_pcu['node_ids']
93                 l_node = get_nodes(node_ids) 
94                                 
95                 if l_node is not None:
96                         for node in l_node:
97                                 values[node['hostname']] = node['ports'][0]
98
99                         values['nodenames'] = [node['hostname'] for node in l_node]
100
101                         # NOTE: this is for a dry run later. It doesn't matter which node.
102                         values['node_id'] = l_node[0]['node_id']
103
104                 values.update(l_pcu)
105         else:
106                 values = None
107         
108         return values
109
110 class ScanInterface(object):
111         recordclass = None
112         syncclass = None
113         primarykey = 'hostname'
114
115         def __init__(self, round=1):
116                 self.round = round
117                 self.count = 1
118
119         def __getattr__(self, name):
120                 if 'collect' in name or 'record' in name:
121                         method = getattr(self, name, None)
122                         if method is None:
123                                 raise Exception("No such method %s" % name)
124                         return method
125                 else:
126                         raise Exception("No such method %s" % name)
127
128         def collect(self, nodename, data):
129                 pass
130
131         def record(self, request, (nodename, values) ):
132
133                 try:
134                         if values is None:
135                                 return
136                         
137                         if self.syncclass:
138                                 fbnodesync = self.syncclass.findby_or_create(
139                                                                                                 #if_new_set={'round' : self.round},
140                                                                                                 **{ self.primarykey : nodename})
141                         # NOTE: This code will either add a new record for the new self.round, 
142                         #       OR it will find the previous value, and update it with new information.
143                         #       The data that is 'lost' is not that important, b/c older
144                         #       history still exists.  
145                         fbrec = self.recordclass.findby_or_create(
146                                                 **{ self.primarykey:nodename})
147
148                         fbrec.set( **values ) 
149
150                         fbrec.flush()
151                         if self.syncclass:
152                                 fbnodesync.round = self.round
153                                 fbnodesync.flush()
154
155                         print "%d %s %s" % (self.count, nodename, values)
156                         self.count += 1
157
158                 except:
159                         print "ERROR:"
160                         email_exception(str(nodename))
161                         print traceback.print_exc()
162                         pass
163
164 class ScanNodeInternal(ScanInterface):
165         recordclass = FindbadNodeRecord
166         #syncclass = FindbadNodeRecordSync
167         syncclass = None
168         primarykey = 'hostname'
169
170         def collectNMAP(self, nodename, cohash):
171                 #### RUN NMAP ###############################
172                 values = {}
173                 nmap = command.CMD()
174                 print "nmap -oG - -P0 -p22,80,806 %s | grep Host:" % nodename
175                 (oval,eval) = nmap.run_noexcept("nmap -oG - -P0 -p22,80,806 %s | grep Host:" % nodename)
176                 # NOTE: an empty / error value for oval, will still work.
177                 (values['port_status'], continue_probe) = nmap_port_status(oval)
178
179                 values['date_checked'] = datetime.now()
180                                 
181                 return (nodename, values)
182
183         def collectInternal(self, nodename, cohash):
184                 ### RUN PING ######################
185                 ping = command.CMD()
186                 (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
187
188                 try:
189                         values = {}
190
191                         if oval == "":
192                                 # An error occurred
193                                 values['ping_status'] = False
194                         else:
195                                 values['ping_status'] = True
196
197                         try:
198                                 for port in [22, 806]: 
199                                         ssh = command.SSH('root', nodename, port)
200
201                                         (oval, errval) = ssh.run_noexcept2(""" <<\EOF
202                                                 echo "{"
203                                                 echo '  "kernel_version":"'`uname -a`'",'
204                                                 echo '  "bmlog":"'`ls /tmp/bm.log`'",'
205                                                 echo '  "bootcd_version":"'`cat /mnt/cdrom/bootme/ID`'",'
206                                                 echo '  "nm_status":"'`ps ax | grep nm.py | grep -v grep`'",'
207                                                 echo '  "dns_status":"'`host boot.planet-lab.org 2>&1`'",'
208                                                 echo '  "princeton_comon_dir":"'`ls -d /vservers/princeton_comon`'",'
209                                                 echo '  "uptime":"'`uptime`'",'
210
211                                                 ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'` 
212                                                 echo '  "princeton_comon_running":"'`ls -d /proc/virtual/$ID`'",'
213                                                 echo '  "princeton_comon_procs":"'`vps ax | grep $ID | grep -v grep | wc -l`'",'
214                                                 echo '  "fs_status":"'`grep proc /proc/mounts | grep ro, ; if [ -x /usr/bin/timeout.pl ] ; then timeout.pl 20 touch /var/log/monitor 2>&1 ; if [ -d /vservers/ ] ; then timeout.pl 20 touch /vservers/monitor.log 2>&1  ; fi ; fi`'",'
215                                                 echo '  "rpm_version":"'`if [ -x /usr/bin/timeout.pl ] ; then timeout.pl 30 rpm -q NodeManager ; fi`'",'
216                                                 echo '  "rpm_versions":"'`if [ -x /usr/bin/timeout.pl ] ; then timeout.pl 45 rpm -q -a ; fi`'",'
217                                                 echo "}"
218 EOF                             """)
219
220                                         values['ssh_error'] = errval
221                                         if len(oval) > 0:
222                                                 #print "OVAL: %s" % oval
223                                                 values.update(eval(oval))
224                                                 values['ssh_portused'] = port
225                                                 break
226                                         else:
227                                                 values.update({'kernel_version': "", 'bmlog' : "", 'bootcd_version' : '', 
228                                                                                 'nm_status' : '', 
229                                                                                 'fs_status' : '',
230                                                                                 'uptime' : '',
231                                                                                 'dns_status' : '',
232                                                                                 'rpm_version' : '',
233                                                                                 'rpm_versions' : '',
234                                                                                 'princeton_comon_dir' : "", 
235                                                                                 'princeton_comon_running' : "", 
236                                                                                 'princeton_comon_procs' : "", 'ssh_portused' : None})
237                         except:
238                                 print traceback.print_exc()
239                                 sys.exit(1)
240
241                         values['fs_status'] = ""
242                         print "ALLVERSIONS: %s %s" % (nodename, values['rpm_versions'])
243
244                         print "RPMVERSION: %s %s" % (nodename, values['rpm_version'])
245                         print "UPTIME: %s %s" % (nodename, values['uptime'])
246                         ### RUN SSH ######################
247                         b_getbootcd_id = True
248
249                         oval = values['kernel_version']
250                         if "2.6.17" in oval or "2.6.2" in oval:
251                                 values['ssh_status'] = True
252                                 values['observed_category'] = 'PROD'
253                                 if "bm.log" in values['bmlog']:
254                                         values['observed_status'] = 'DEBUG'
255                                 else:
256                                         values['observed_status'] = 'BOOT'
257                         elif "2.6.12" in oval or "2.6.10" in oval:
258                                 values['ssh_status'] = True
259                                 values['observed_category'] = 'OLDPROD'
260                                 if "bm.log" in values['bmlog']:
261                                         values['observed_status'] = 'DEBUG'
262                                 else:
263                                         values['observed_status'] = 'BOOT'
264                         
265                         # NOTE: on 2.6.8 kernels, with 4.2 bootstrapfs, the chroot 
266                         #       command fails.  I have no idea why.
267                         elif "2.4" in oval or "2.6.8" in oval:
268                                 b_getbootcd_id = False
269                                 values['ssh_status'] = True
270                                 values['observed_category'] = 'OLDBOOTCD'
271                                 values['observed_status'] = 'DEBUG'
272                         elif oval != "":
273                                 values['ssh_status'] = True
274                                 values['observed_category'] = 'UNKNOWN'
275                                 if "bm.log" in values['bmlog']:
276                                         values['observed_status'] = 'DEBUG'
277                                 else:
278                                         values['observed_status'] = 'BOOT'
279                         else:
280                                 # An error occurred.
281                                 b_getbootcd_id = False
282                                 values['ssh_status'] = False
283                                 values['observed_category'] = 'ERROR'
284                                 values['observed_status'] = 'DOWN'
285                                 val = errval.strip()
286                                 values['ssh_error'] = val
287                                 values['kernel_version'] = ""
288
289                         if b_getbootcd_id:
290                                 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
291                                 oval = values['bootcd_version']
292                                 if "BootCD" in oval:
293                                         values['bootcd_version'] = oval
294                                         if "v2" in oval and \
295                                                 ( nodename is not "planetlab1.cs.unc.edu" and \
296                                                   nodename is not "planetlab2.cs.unc.edu" ):
297                                                 values['observed_category'] = 'OLDBOOTCD'
298                                 else:
299                                         values['bootcd_version'] = ""
300                         else:
301                                 values['bootcd_version'] = ""
302
303                         oval = values['nm_status']
304                         if "nm.py" in oval:
305                                 values['nm_status'] = "Y"
306                         else:
307                                 values['nm_status'] = "N"
308
309                         continue_slice_check = True
310                         oval = values['princeton_comon_dir']
311                         if "princeton_comon_dir" in oval:
312                                 values['princeton_comon_dir'] = True
313                         else:
314                                 values['princeton_comon_dir'] = False
315                                 continue_slice_check = False
316
317                         if continue_slice_check:
318                                 oval = values['princeton_comon_running']
319                                 if len(oval) > len('/proc/virtual/'):
320                                         values['princeton_comon_running'] = True
321                                 else:
322                                         values['princeton_comon_running'] = False
323                                         continue_slice_check = False
324                         else:
325                                 values['princeton_comon_running'] = False
326                                 
327                         if continue_slice_check:
328                                 oval = values['princeton_comon_procs']
329                                 values['princeton_comon_procs'] = int(oval)
330                         else:
331                                 values['princeton_comon_procs'] = None
332
333                                 
334                         if nodename in cohash: 
335                                 values['comon_stats'] = cohash[nodename]
336                         else:
337                                 values['comon_stats'] = {'resptime':  '-1', 
338                                                                                 'uptime':    '-1',
339                                                                                 'sshstatus': '-1', 
340                                                                                 'lastcotop': '-1',
341                                                                                 'cpuspeed' : "null",
342                                                                                 'disksize' : 'null',
343                                                                                 'memsize'  : 'null'}
344                         # include output value
345                         ### GET PLC NODE ######################
346                         d_node = plccache.GetNodeByName(nodename)
347                         values['plc_node_stats'] = d_node
348
349                         ##### NMAP  ###################
350                         (n, v) = self.collectNMAP(nodename, None)
351                         values.update(v)
352
353                         ### GET PLC PCU ######################
354                         site_id = -1
355                         d_pcu = None
356                         if d_node:
357                                 pcu = d_node['pcu_ids']
358                                 if len(pcu) > 0:
359                                         d_pcu = pcu[0]
360
361                                 site_id = d_node['site_id']
362
363                         values['plc_pcuid'] = d_pcu
364
365                         ### GET PLC SITE ######################
366                         plc_lock.acquire()
367                         d_site = None
368                         values['loginbase'] = ""
369                         try:
370                                 d_site = plccache.GetSitesById([ site_id ])[0]
371                                 #d_site = plc.getSites({'site_id': site_id}, 
372                                 #                                       ['max_slices', 'slice_ids', 'node_ids', 'login_base'])[0]
373                                 values['loginbase'] = d_site['login_base']
374                         except:
375                                 traceback.print_exc()
376                         plc_lock.release()
377
378                         values['plc_site_stats'] = d_site 
379                         values['date_checked'] = datetime.now()
380                 except:
381                         print traceback.print_exc()
382
383                 return (nodename, values)
384
385 def internalprobe(hostname):
386         #fbsync = FindbadNodeRecordSync.findby_or_create(hostname="global", 
387         #                                                                                               if_new_set={'round' : 1})
388         scannode = ScanNodeInternal() # fbsync.round)
389         try:
390                 (nodename, values) = scannode.collectInternal(hostname, {})
391                 scannode.record(None, (nodename, values))
392                 session.flush()
393                 return True
394         except:
395                 print traceback.print_exc()
396                 return False
397
398 def externalprobe(hostname):
399         #fbsync = FindbadNodeRecordSync.findby_or_create(hostname="global", 
400         #                                                                                               if_new_set={'round' : 1})
401         scannode = ScanNodeInternal() # fbsync.round)
402         try:
403                 (nodename, values) = scannode.collectNMAP(hostname, {})
404                 scannode.record(None, (nodename, values))
405                 session.flush()
406                 return True
407         except:
408                 print traceback.print_exc()
409                 return False
410
411 class ScanPCU(ScanInterface):
412         recordclass = FindbadPCURecord
413         syncclass = None
414         primarykey = 'plc_pcuid'
415
416         def collectInternal(self, pcuname, cohash):
417
418                 continue_probe = True
419                 errors = None
420                 values = {'reboot_trial_status' : 'novalue'}
421                 ### GET PCU ######################
422                 try:
423                         b_except = False
424                         try:
425                                 v = get_plc_pcu_values(pcuname)
426                                 if v['hostname'] is not None: v['hostname'] = v['hostname'].strip()
427                                 if v['ip'] is not None: v['ip'] = v['ip'].strip()
428
429                                 if v is not None:
430                                         values['plc_pcu_stats'] = v
431                                 else:
432                                         continue_probe = False
433                         except:
434                                 b_except = True
435                                 traceback.print_exc()
436                                 continue_probe = False
437
438                         if b_except or not continue_probe: return (None, None, None)
439
440                         #### RUN NMAP ###############################
441                         if continue_probe:
442                                 nmap = command.CMD()
443                                 print "nmap -oG - -P0 -p22,23,80,443,5869,9100,16992 %s | grep Host:" % reboot.pcu_name(values['plc_pcu_stats'])
444                                 (oval,eval) = nmap.run_noexcept("nmap -oG - -P0 -p22,23,80,443,5869,9100,16992 %s | grep Host:" % reboot.pcu_name(values['plc_pcu_stats']))
445                                 # NOTE: an empty / error value for oval, will still work.
446                                 (values['port_status'], continue_probe) = nmap_port_status(oval)
447                         else:
448                                 values['port_status'] = None
449                                 
450                         #### COMPLETE ENTRY   #######################
451
452                         values['entry_complete'] = []
453                         #if values['protocol'] is None or values['protocol'] is "":
454                         #       values['entry_complete'] += ["protocol"]
455                         if values['plc_pcu_stats']['model'] is None or values['plc_pcu_stats']['model'] is "":
456                                 values['entry_complete'] += ["model"]
457                                 # Cannot continue due to this condition
458                                 continue_probe = False
459
460                         if values['plc_pcu_stats']['password'] is None or values['plc_pcu_stats']['password'] is "":
461                                 values['entry_complete'] += ["password"]
462                                 # Cannot continue due to this condition
463                                 continue_probe = False
464
465                         if len(values['entry_complete']) > 0:
466                                 continue_probe = False
467
468                         if values['plc_pcu_stats']['hostname'] is None or values['plc_pcu_stats']['hostname'] is "":
469                                 values['entry_complete'] += ["hostname"]
470                         if values['plc_pcu_stats']['ip'] is None or values['plc_pcu_stats']['ip'] is "":
471                                 values['entry_complete'] += ["ip"]
472
473                         # If there are no nodes associated with this PCU, then we cannot continue.
474                         if len(values['plc_pcu_stats']['node_ids']) == 0:
475                                 continue_probe = False
476                                 values['entry_complete'] += ['nodeids']
477
478
479                         #### DNS and IP MATCH #######################
480                         if values['plc_pcu_stats']['hostname'] is not None and values['plc_pcu_stats']['hostname'] is not "" and \
481                            values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
482                                 try:
483                                         ipaddr = socket.gethostbyname(values['plc_pcu_stats']['hostname'])
484                                         if ipaddr == values['plc_pcu_stats']['ip']:
485                                                 values['dns_status'] = "DNS-OK"
486                                         else:
487                                                 values['dns_status'] = "DNS-MISMATCH"
488                                                 values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
489
490                                 except Exception, err:
491                                         values['dns_status'] = "DNS-NOENTRY"
492                                         values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
493                         else:
494                                 if values['plc_pcu_stats']['ip'] is not None and values['plc_pcu_stats']['ip'] is not "":
495                                         values['dns_status'] = "NOHOSTNAME"
496                                         values['plc_pcu_stats']['hostname'] = values['plc_pcu_stats']['ip']
497                                 else:
498                                         values['dns_status'] = "NO-DNS-OR-IP"
499                                         values['plc_pcu_stats']['hostname'] = "No_entry_in_DB"
500                                         continue_probe = False
501
502
503                         ######  DRY RUN  ############################
504                         if continue_probe and 'node_ids' in values['plc_pcu_stats'] and \
505                                 len(values['plc_pcu_stats']['node_ids']) > 0:
506                                 rb_ret = reboot.reboot_test_new(values['plc_pcu_stats']['nodenames'][0], 
507                                                                                                 values, 1, True)
508                         else:
509                                 rb_ret = "Not_Run" # No nodes to test"
510
511                         values['reboot_trial_status'] = rb_ret
512
513                 except:
514                         print "____________________________________"
515                         print values
516                         errors = values
517                         print "____________________________________"
518                         errors['traceback'] = traceback.format_exc()
519                         print errors['traceback']
520                         values['reboot_trial_status'] = str(errors['traceback'])
521                         print values
522
523                 values['entry_complete']=" ".join(values['entry_complete'])
524
525                 values['date_checked'] = datetime.now()
526                 return (pcuname, values)
527