12 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
13 "table=table_nodeview&" + \
14 "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
17 #"select='lastcotop!=0'"
20 plc_lock = threading.Lock()
22 externalState = {'round': round, 'nodes': {}}
31 from nodequery import verify,query_to_dict,node_select
34 api = plc.getAuthAPI()
36 def collectPingAndSSH(nodename, cohash):
37 ### RUN PING ######################
38 ping = moncommands.CMD()
39 (oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
45 values['ping'] = "NOPING"
47 values['ping'] = "PING"
50 for port in [22, 806]:
51 ssh = moncommands.SSH('root', nodename, port)
53 (oval, errval) = ssh.run_noexcept2(""" <<\EOF
55 echo ' "kernel":"'`uname -a`'",'
56 echo ' "bmlog":"'`ls /tmp/bm.log`'",'
57 echo ' "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",'
58 echo ' "nm":"'`ps ax | grep nm.py | grep -v grep`'",'
59 echo ' "readonlyfs":"'`touch /var/log/monitor 2>&1`'",'
60 echo ' "dns":"'`host boot.planet-lab.org 2>&1`'",'
61 echo ' "princeton_comon":"'`ls -d /vservers/princeton_comon`'",'
63 ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'`
65 echo ' "princeton_comon_running":"'`ls -d /proc/virtual/$ID`'",'
66 echo ' "princeton_comon_procs":"'`vps ax | grep $ID | grep -v grep | wc -l`'",'
71 values.update(eval(oval))
72 values['sshport'] = port
75 values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '',
79 'princeton_comon' : '',
80 'princeton_comon_running' : '',
81 'princeton_comon_procs' : '', 'sshport' : None})
83 print traceback.print_exc()
84 from nodecommon import email_exception
88 ### RUN SSH ######################
90 #ssh = moncommands.SSH('root', nodename)
93 #(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
95 oval = values['kernel']
96 if "2.6.17" in oval or "2.6.2" in oval:
98 values['category'] = 'PROD'
99 if "bm.log" in values['bmlog']:
100 values['state'] = 'DEBUG'
102 values['state'] = 'BOOT'
103 elif "2.6.12" in oval or "2.6.10" in oval:
104 values['ssh'] = 'SSH'
105 values['category'] = 'OLDPROD'
106 if "bm.log" in values['bmlog']:
107 values['state'] = 'DEBUG'
109 values['state'] = 'BOOT'
111 # NOTE: on 2.6.8 kernels, with 4.2 bootstrapfs, the chroot command fails. I have no idea why.
112 elif "2.4" in oval or "2.6.8" in oval:
113 b_getbootcd_id = False
114 values['ssh'] = 'SSH'
115 values['category'] = 'OLDBOOTCD'
116 values['state'] = 'DEBUG'
118 values['ssh'] = 'SSH'
119 values['category'] = 'UNKNOWN'
120 if "bm.log" in values['bmlog']:
121 values['state'] = 'DEBUG'
123 values['state'] = 'BOOT'
126 b_getbootcd_id = False
127 values['ssh'] = 'NOSSH'
128 values['category'] = 'ERROR'
129 values['state'] = 'DOWN'
131 values['kernel'] = val
133 #values['kernel'] = val
136 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
137 #(oval, errval) = ssh.run_noexcept('cat /mnt/cdrom/bootme/ID')
138 oval = values['bootcd']
140 values['bootcd'] = oval
141 if "v2" in oval and \
142 ( nodename is not "planetlab1.cs.unc.edu" and \
143 nodename is not "planetlab2.cs.unc.edu" ):
144 values['category'] = 'OLDBOOTCD'
146 values['bootcd'] = ""
148 values['bootcd'] = ""
150 # TODO: get bm.log for debug nodes.
153 #(oval, errval) = ssh.run_noexcept('ps ax | grep nm.py | grep -v grep')
160 continue_slice_check = True
161 #(oval, errval) = ssh.run_noexcept('ls -d /vservers/princeton_comon')
162 oval = values['princeton_comon']
163 if "princeton_comon" in oval:
164 values['princeton_comon'] = "Y"
166 values['princeton_comon'] = "N"
167 continue_slice_check = False
169 if continue_slice_check:
170 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; ls -d /proc/virtual/$ID')
171 oval = values['princeton_comon_running']
172 if len(oval) > len('/proc/virtual/'):
173 values['princeton_comon_running'] = "Y"
175 values['princeton_comon_running'] = "N"
176 continue_slice_check = False
178 values['princeton_comon_running'] = "-"
180 if continue_slice_check:
181 #(oval, errval) = ssh.run_noexcept('ID=`grep princeton_comon /etc/passwd | awk -F : "{if ( \\\$3 > 500 ) { print \\\$3}}"`; vps ax | grep $ID | grep -v grep | wc -l')
182 oval = values['princeton_comon_procs']
183 values['princeton_comon_procs'] = oval
185 values['princeton_comon_procs'] = "-"
188 if nodename in cohash:
189 values['comonstats'] = cohash[nodename]
191 values['comonstats'] = {'resptime': '-1',
198 # include output value
199 ### GET PLC NODE ######################
204 d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'date_created', 'last_updated', 'last_contact', 'boot_state', 'nodegroup_ids'])
207 traceback.print_exc()
208 from nodecommon import email_exception
212 if b_except: return (None, None)
215 if d_node and len(d_node) > 0:
216 pcu = d_node[0]['pcu_ids']
218 values['pcu'] = "PCU"
220 values['pcu'] = "NOPCU"
221 site_id = d_node[0]['site_id']
222 last_contact = d_node[0]['last_contact']
223 nodegroups = [ i['name'] for i in api.GetNodeGroups(d_node[0]['nodegroup_ids']) ]
224 values['plcnode'] = {'status' : 'SUCCESS',
226 'boot_state' : d_node[0]['boot_state'],
228 'nodegroups' : nodegroups,
229 'last_contact': last_contact,
230 'date_created': d_node[0]['date_created'],
231 'last_updated': d_node[0]['last_updated']}
233 values['pcu'] = "UNKNOWN"
234 values['plcnode'] = {'status' : "GN_FAILED"}
237 ### GET PLC SITE ######################
242 d_site = plc.getSites({'site_id': site_id},
243 ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
246 traceback.print_exc()
247 from nodecommon import email_exception
251 if b_except: return (None, None)
253 if d_site and len(d_site) > 0:
254 max_slices = d_site[0]['max_slices']
255 num_slices = len(d_site[0]['slice_ids'])
256 num_nodes = len(d_site[0]['node_ids'])
257 loginbase = d_site[0]['login_base']
258 values['plcsite'] = {'num_nodes' : num_nodes,
259 'max_slices' : max_slices,
260 'num_slices' : num_slices,
261 'login_base' : loginbase,
262 'status' : 'SUCCESS'}
264 values['plcsite'] = {'status' : "GS_FAILED"}
266 values['checked'] = time.time()
268 return (nodename, values)
270 def recordPingAndSSH(request, result):
273 (nodename, values) = result
275 if values is not None:
276 global_round = externalState['round']
277 externalState['nodes'][nodename]['values'] = values
278 externalState['nodes'][nodename]['round'] = global_round
281 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
283 database.dbDump(config.dbname, externalState)
285 # this will be called when an exception occurs within a thread
286 def handle_exception(request, result):
287 print "Exception occured in request %s" % request.requestID
289 print "Result: %s" % i
292 def checkAndRecordState(l_nodes, cohash):
295 global_round = externalState['round']
297 tp = threadpool.ThreadPool(20)
299 # CREATE all the work requests
300 for nodename in l_nodes:
301 if nodename not in externalState['nodes']:
302 externalState['nodes'][nodename] = {'round': 0, 'values': []}
304 node_round = externalState['nodes'][nodename]['round']
305 if node_round < global_round:
306 # recreate node stats when refreshed
307 #print "%s" % nodename
308 req = threadpool.WorkRequest(collectPingAndSSH, [nodename, cohash], {},
309 None, recordPingAndSSH, handle_exception)
312 # We just skip it, since it's "up to date"
314 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
317 # WAIT while all the work requests are processed.
323 # if more than two hours
324 if time.time() - begin > (60*60*1.5):
325 print "findbad.py has run out of time!!!!!!"
326 database.dbDump(config.dbname, externalState)
328 except KeyboardInterrupt:
331 except threadpool.NoResultsPending:
332 print "All results collected."
335 database.dbDump(config.dbname, externalState)
342 externalState = database.if_cached_else(1, config.dbname, lambda : externalState)
345 # update global round number to force refreshes across all nodes
346 externalState['round'] += 1
348 cotop = comon.Comon()
349 # lastcotop measures whether cotop is actually running. this is a better
350 # metric than sshstatus, or other values from CoMon
351 cotop_url = COMON_COTOPURL
353 # history information for all nodes
355 cohash = cotop.coget(cotop_url)
356 l_nodes = syncplcdb.create_plcdb()
358 f_nodes = util.file.getListFromFile(config.nodelist)
359 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
361 f_nodes = [config.node]
362 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
363 elif config.nodegroup:
364 ng = api.GetNodeGroups({'name' : config.nodegroup})
365 l_nodes = api.GetNodes(ng[0]['node_ids'])
367 site = api.GetSites(config.site)
368 l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
370 l_nodes = [node['hostname'] for node in l_nodes]
372 # perform this query after the above options, so that the filter above
374 if config.nodeselect:
375 fb = database.dbLoad("findbad")
376 l_nodes = node_select(config.nodeselect, fb['nodes'].keys(), fb)
378 print "fetching %s hosts" % len(l_nodes)
380 checkAndRecordState(l_nodes, cohash)
385 if __name__ == '__main__':
386 import parser as parsermodule
388 parser = parsermodule.getParser(['nodesets'])
390 parser.set_defaults( increment=False, dbname="findbad", cachenodes=False)
391 parser.add_option("", "--cachenodes", action="store_true",
392 help="Cache node lookup from PLC")
393 parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
394 help="Specify the name of the database to which the information is saved")
395 parser.add_option("-i", "--increment", action="store_true", dest="increment",
396 help="Increment round number to force refresh or retry")
398 parser = parsermodule.getParser(['defaults'], parser)
400 cfg = parsermodule.parse_args(parser)
404 except Exception, err:
405 print traceback.print_exc()
406 from nodecommon import email_exception
408 print "Exception: %s" % err
409 print "Saving data... exitting."
410 database.dbDump(config.dbname, externalState)