X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=findbad.py;h=6420f99c1efdafb498c67b68deb47c7193ba31a0;hb=refs%2Fheads%2F1.0;hp=141f9ac66ec2077b5baca1faa4e586e0ed35618d;hpb=77f84f1e8242cdc45eb091ab65eef940a23493a6;p=monitor.git

diff --git a/findbad.py b/findbad.py
index 141f9ac..6420f99 100755
--- a/findbad.py
+++ b/findbad.py
@@ -4,13 +4,15 @@ import os
 import sys
 import string
 import time
+import config
+import util.file
 
 
 # QUERY all nodes.
 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
-					"table=table_nodeview&" + \
-				    "dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
-				    "formatcsv"
+				"table=table_nodeview&" + \
+				"dumpcols='name,resptime,sshstatus,uptime,lastcotop,cpuspeed,memsize,disksize'&" + \
+				"formatcsv"
 				    #"formatcsv&" + \
 					#"select='lastcotop!=0'"
 
@@ -21,19 +23,19 @@ externalState = {'round': round, 'nodes': {}}
 count = 0
 
 
-import soltesz
+import database
+import moncommands 
 import comon
 import threadpool
 import syncplcdb
 from nodequery import verify,query_to_dict,node_select
-
+import traceback
 import plc
-import auth
-api = plc.PLC(auth.auth, auth.plc)
+api = plc.getAuthAPI()
 
 def collectPingAndSSH(nodename, cohash):
 	### RUN PING ######################
-	ping = soltesz.CMD()
+	ping = moncommands.CMD()
 	(oval,errval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
 
 	values = {}
@@ -46,7 +48,7 @@ def collectPingAndSSH(nodename, cohash):
 
 	try:
 		for port in [22, 806]: 
-			ssh = soltesz.SSH('root', nodename, port)
+			ssh = moncommands.SSH('root', nodename, port)
 
 			(oval, errval) = ssh.run_noexcept2(""" <<\EOF
 				echo "{"
@@ -54,6 +56,8 @@ def collectPingAndSSH(nodename, cohash):
 				echo '  "bmlog":"'`ls /tmp/bm.log`'",'
 				echo '  "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",'
 				echo '  "nm":"'`ps ax | grep nm.py | grep -v grep`'",'
+				echo '  "readonlyfs":"'`touch /var/log/monitor 2>&1 ; touch /vservers/monitor.log 2>&1`'",'
+				echo '  "dns":"'`host boot.planet-lab.org 2>&1`'",'
 				echo '  "princeton_comon":"'`ls -d /vservers/princeton_comon`'",'
 
 				ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'` 
@@ -68,16 +72,22 @@ EOF			""")
 				values['sshport'] = port
 				break
 			else:
-				values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', 'nm' :
-				'', 'princeton_comon' : '', 'princeton_comon_running' : '',
-				'princeton_comon_procs' : '', 'sshport' : None})
+				values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', 
+							 	'nm' : '', 
+								'readonlyfs' : '',
+								'dns' : '',
+								'princeton_comon' : '', 
+								'princeton_comon_running' : '', 
+								'princeton_comon_procs' : '', 'sshport' : None})
 	except:
-		import traceback; print traceback.print_exc()
+		print traceback.print_exc()
+		from nodecommon import email_exception
+		email_exception()
 		sys.exit(1)
 
 	### RUN SSH ######################
 	b_getbootcd_id = True
-	#ssh = soltesz.SSH('root', nodename)
+	#ssh = moncommands.SSH('root', nodename)
 	#oval = ""
 	#errval = ""
 	#(oval, errval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
@@ -85,14 +95,14 @@ EOF			""")
 	oval = values['kernel']
 	if "2.6.17" in oval or "2.6.2" in oval:
 		values['ssh'] = 'SSH'
-		values['category'] = 'ALPHA'
+		values['category'] = 'PROD'
 		if "bm.log" in values['bmlog']:
 			values['state'] = 'DEBUG'
 		else:
 			values['state'] = 'BOOT'
 	elif "2.6.12" in oval or "2.6.10" in oval:
 		values['ssh'] = 'SSH'
-		values['category'] = 'PROD'
+		values['category'] = 'OLDPROD'
 		if "bm.log" in values['bmlog']:
 			values['state'] = 'DEBUG'
 		else:
@@ -181,20 +191,22 @@ EOF			""")
 		values['comonstats'] = {'resptime':  '-1', 
 								'uptime':    '-1',
 								'sshstatus': '-1', 
-								'lastcotop': '-1'}
+								'lastcotop': '-1',
+								'cpuspeed' : "null",
+								'disksize' : 'null',
+								'memsize'  : 'null'}
 	# include output value
 	### GET PLC NODE ######################
 	b_except = False
 	plc_lock.acquire()
 
 	try:
-		d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'last_contact', 'boot_state', 'nodegroup_ids'])
+		d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'date_created', 'last_updated', 'last_contact', 'boot_state', 'nodegroup_ids'])
 	except:
 		b_except = True
-		import traceback
-		b_except = True
-		import traceback
 		traceback.print_exc()
+		from nodecommon import email_exception
+		email_exception()
 
 	plc_lock.release()
 	if b_except: return (None, None)
@@ -208,13 +220,15 @@ EOF			""")
 			values['pcu'] = "NOPCU"
 		site_id = d_node[0]['site_id']
 		last_contact = d_node[0]['last_contact']
-		nodegroups = d_node[0]['nodegroup_ids']
+		nodegroups = [ i['groupname'] for i in api.GetNodeGroups(d_node[0]['nodegroup_ids']) ]
 		values['plcnode'] = {'status' : 'SUCCESS', 
 							'pcu_ids': pcu, 
 							'boot_state' : d_node[0]['boot_state'],
 							'site_id': site_id,
 							'nodegroups' : nodegroups,
-							'last_contact': last_contact}
+							'last_contact': last_contact,
+							'date_created': d_node[0]['date_created'],
+							'last_updated': d_node[0]['last_updated']}
 	else:
 		values['pcu']     = "UNKNOWN"
 		values['plcnode'] = {'status' : "GN_FAILED"}
@@ -229,8 +243,9 @@ EOF			""")
 							['max_slices', 'slice_ids', 'node_ids', 'login_base'])
 	except:
 		b_except = True
-		import traceback
 		traceback.print_exc()
+		from nodecommon import email_exception
+		email_exception()
 
 	plc_lock.release()
 	if b_except: return (None, None)
@@ -265,7 +280,7 @@ def recordPingAndSSH(request, result):
 		count += 1
 		print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
 		if count % 20 == 0:
-			soltesz.dbDump(config.dbname, externalState)
+			database.dbDump(config.dbname, externalState)
 
 # this will be called when an exception occurs within a thread
 def handle_exception(request, result):
@@ -300,10 +315,16 @@ def checkAndRecordState(l_nodes, cohash):
 			pass
 
 	# WAIT while all the work requests are processed.
+	begin = time.time()
 	while 1:
 		try:
 			time.sleep(1)
 			tp.poll()
+			# if more than two hours
+			if time.time() - begin > (60*60*1.5):
+				print "findbad.py has run out of time!!!!!!"
+				database.dbDump(config.dbname, externalState)
+				os._exit(1)
 		except KeyboardInterrupt:
 			print "Interrupted!"
 			break
@@ -311,14 +332,14 @@ def checkAndRecordState(l_nodes, cohash):
 			print "All results collected."
 			break
 
-	soltesz.dbDump(config.dbname, externalState)
+	database.dbDump(config.dbname, externalState)
 
 
 
 def main():
 	global externalState
 
-	externalState = soltesz.if_cached_else(1, config.dbname, lambda : externalState) 
+	externalState = database.if_cached_else(1, config.dbname, lambda : externalState) 
 
 	if config.increment:
 		# update global round number to force refreshes across all nodes
@@ -333,14 +354,14 @@ def main():
 	#cohash = {}
 	cohash = cotop.coget(cotop_url)
 	l_nodes = syncplcdb.create_plcdb()
-	if config.filename:
-		f_nodes = config.getListFromFile(config.filename)
+	if config.nodelist:
+		f_nodes = util.file.getListFromFile(config.nodelist)
 		l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 	elif config.node:
 		f_nodes = [config.node]
 		l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 	elif config.nodegroup:
-		ng = api.GetNodeGroups({'name' : config.nodegroup})
+		ng = api.GetNodeGroups({'groupname' : config.nodegroup})
 		l_nodes = api.GetNodes(ng[0]['node_ids'])
 	elif config.site:
 		site = api.GetSites(config.site)
@@ -351,7 +372,8 @@ def main():
 	# perform this query after the above options, so that the filter above
 	# does not break.
 	if config.nodeselect:
-		l_nodes = node_select(config.nodeselect)
+		fb = database.dbLoad("findbad")
+		l_nodes = node_select(config.nodeselect, fb['nodes'].keys(), fb)
 
 	print "fetching %s hosts" % len(l_nodes)
 
@@ -361,37 +383,29 @@ def main():
 
 
 if __name__ == '__main__':
-	from config import config
-	from optparse import OptionParser
-	parser = OptionParser()
-	parser.set_defaults(filename=None, node=None, site=None, nodeselect=False, nodegroup=None, 
-						increment=False, dbname="findbadnodes", cachenodes=False)
-	parser.add_option("", "--node", dest="node", metavar="hostname", 
-						help="Provide a single node to operate on")
-	parser.add_option("-f", "--nodelist", dest="filename", metavar="FILE", 
-						help="Provide the input file for the node list")
-	parser.add_option("", "--nodeselect", dest="nodeselect", metavar="query string", 
-						help="Provide a selection string to return a node list.")
-	parser.add_option("", "--nodegroup", dest="nodegroup", metavar="FILE", 
-						help="Provide the nodegroup for the list of nodes.")
-	parser.add_option("", "--site", dest="site", metavar="site name",
-						help="Specify a site to view node status")
+	import parser as parsermodule
+
+	parser = parsermodule.getParser(['nodesets'])
 
+	parser.set_defaults( increment=False, dbname="findbad", cachenodes=False)
 	parser.add_option("", "--cachenodes", action="store_true",
 						help="Cache node lookup from PLC")
 	parser.add_option("", "--dbname", dest="dbname", metavar="FILE", 
 						help="Specify the name of the database to which the information is saved")
 	parser.add_option("-i", "--increment", action="store_true", dest="increment", 
 						help="Increment round number to force refresh or retry")
-	config = config(parser)
-	config.parse_args()
+
+	parser = parsermodule.getParser(['defaults'], parser)
+	
+	cfg = parsermodule.parse_args(parser)
 
 	try:
 		main()
 	except Exception, err:
-		import traceback
 		print traceback.print_exc()
+		from nodecommon import email_exception
+		email_exception()
 		print "Exception: %s" % err
 		print "Saving data... exitting."
-		soltesz.dbDump(config.dbname, externalState)
+		database.dbDump(config.dbname, externalState)
 		sys.exit(0)