X-Git-Url: http://git.onelab.eu/?p=monitor.git;a=blobdiff_plain;f=monitor%2Fbootman.py;h=fdfadb207c504850cc5f511e84919532133fbefa;hp=4693315666106f14c583b8d87711d350c9c28060;hb=035a846d8617889c01cae12bc6d64eb7c48b64bd;hpb=3f2acbd8b7761dae4bb02fedd3557e1003cafb43

diff --git a/monitor/bootman.py b/monitor/bootman.py
index 4693315..fdfadb2 100755
--- a/monitor/bootman.py
+++ b/monitor/bootman.py
@@ -68,18 +68,34 @@ class NodeConnection:
 		return "unknown"
 
 	def get_dmesg(self):
+		t_stamp = time.strftime("%Y-%m-%d-%H:%M")
 		self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
-		download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node)
-		log = open("log/dmesg.%s.log" % self.node, 'r')
+		download(self.c, "/var/log/dmesg.bm.log", "%s/history/%s-dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node))
+		os.system("cp %s/history/%s-dmesg.%s.log %s/dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node, config.MONITOR_BOOTMANAGER_LOG, self.node))
+		log = open("%s/dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r')
 		return log
 
 	def get_bootmanager_log(self):
-		download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node)
-		#os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node))
-		os.system("cp log/bm.%s.log.gz log/bm.%s.log" % (self.node, self.node))
-		log = open("log/bm.%s.log" % self.node, 'r')
+		t_stamp = time.strftime("%Y-%m-%d-%H:%M")
+		download(self.c, "/tmp/bm.log", "%s/history/%s-bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node))
+		os.system("cp %s/history/%s-bm.%s.log %s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node, config.MONITOR_BOOTMANAGER_LOG, self.node))
+		log = open("%s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r')
 		return log
 
+
+#	def get_dmesg(self):
+#		self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
+#		download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node)
+#		log = open("log/dmesg.%s.log" % self.node, 'r')
+#		return log
+#
+#	def get_bootmanager_log(self):
+#		download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node)
+#		#os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node))
+#		os.system("cp log/bm.%s.log.gz log/bm.%s.log" % (self.node, self.node))
+#		log = open("log/bm.%s.log" % self.node, 'r')
+#		return log
+
 	def dump_plconf_file(self):
 		c = self.c
 		self.c.modules.sys.path.append("/tmp/source/")
@@ -471,6 +487,8 @@ class DebugInterface:
 		# update_node_config_email
 		for n in ["bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-done",
 				  "bminit-cfg-exception-update-bootupdatefail-nonode-debug-done",
+				  "bminit-cfg-exception-update-bootupdatefail-nonode-debug-validate-exception-done",
+				  "bminit-cfg-exception-nocfg-update-bootupdatefail-nonode-debug-validate-exception-done",
 				  "bminit-cfg-auth-bootcheckfail-nonode-exception-update-bootupdatefail-nonode-debug-done",
 				]:
 			sequences.update({n : "update_node_config_email"})
@@ -626,8 +644,12 @@ class DebugInterface:
 
 		return sequence
 		
-
 def restore(sitehist, hostname, config=None, forced_action=None):
+	ret = restore_basic(sitehist, hostname, config, forced_action)
+	session.flush()
+	return ret
+
+def restore_basic(sitehist, hostname, config=None, forced_action=None):
 
 	# NOTE: Nothing works if the bootcd is REALLY old.
 	#       So, this is the first step.
@@ -638,27 +660,19 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 	if fbnode['observed_category'] == "OLDBOOTCD":
 		print "\t...Notify owner to update BootImage!!!"
 
-		if not found_within(recent_actions, 'newbootcd_notice', 3):
+		if not found_within(recent_actions, 'newbootcd_notice', 3.5):
 			sitehist.sendMessage('newbootcd_notice', hostname=hostname)
 
 			print "\tDisabling %s due to out-of-date BootImage" % hostname
-			api.UpdateNode(hostname, {'boot_state' : 'disable'})
+			api.UpdateNode(hostname, {'boot_state' : 'disabled'})
 
 		# NOTE: nothing else is possible.
 		return True
 
 	debugnode = DebugInterface(hostname)
 	conn = debugnode.getConnection()
-	#print "conn: %s" % conn
-	#print "trying to use conn after returning it."
-	#print conn.c.modules.sys.path
-	#print conn.c.modules.os.path.exists('/tmp/source')
 	if type(conn) == type(False): return False
 
-	#if forced_action == "reboot":
-	#	conn.restart_node('reinstall')
-	#	return True
-
 	boot_state = conn.get_boot_state()
 	if boot_state != "debug":
 		print "... %s in %s state: skipping..." % (hostname , boot_state)
@@ -688,11 +702,12 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 			print "...Should investigate.  Skipping node."
 			# TODO: send message related to these errors.
 
-			if not found_within(recent_actions, 'newbootcd_notice', 3):
+			if not found_within(recent_actions, 'baddisk_notice', 7):
+				print "baddisk_notice not found recently"
 
 				log=conn.get_dmesg().read()
 				sitehist.sendMessage('baddisk_notice', hostname=hostname, log=log)
-				conn.set_nodestate('disable')
+				conn.set_nodestate('disabled')
 
 			return False
 
@@ -729,6 +744,8 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 		args['sequence'] = s
 		args['bmlog'] = conn.get_bootmanager_log().read()
 		args['viart'] = False
+		args['saveact'] = True
+		args['ccemail'] = True
 
 		sitehist.sendMessage('unknownsequence_notice', **args)
 
@@ -767,6 +784,8 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 			args['sequence'] = s
 			args['bmlog'] = conn.get_bootmanager_log().read()
 			args['viart'] = False
+			args['saveact'] = True
+			args['ccemail'] = True
 
 			sitehist.sendMessage('unknownsequence_notice', **args)
 			conn.restart_bootmanager('boot')
@@ -774,7 +793,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 		# TODO: differentiate this and the 'nodenetwork_email' actions.
 		elif sequences[s] == "update_node_config_email":
 
-			if not found_within(recent_actions, 'nodeconfig_notice', 3):
+			if not found_within(recent_actions, 'nodeconfig_notice', 3.5):
 				args = {}
 				args['hostname'] = hostname
 				sitehist.sendMessage('nodeconfig_notice', **args)
@@ -782,7 +801,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 
 		elif sequences[s] == "nodenetwork_email":
 
-			if not found_within(recent_actions, 'nodeconfig_notice', 3):
+			if not found_within(recent_actions, 'nodeconfig_notice', 3.5):
 				args = {}
 				args['hostname'] = hostname
 				args['bmlog'] = conn.get_bootmanager_log().read()
@@ -791,7 +810,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 
 		elif sequences[s] == "update_bootcd_email":
 
-			if not found_within(recent_actions, 'newalphacd_notice', 3):
+			if not found_within(recent_actions, 'newalphacd_notice', 3.5):
 				args = {}
 				args.update(getconf.getconf(hostname)) # NOTE: Generates boot images for the user:
 				args['hostname'] = hostname
@@ -805,17 +824,17 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 			# require either an exception "/minhw" or other manual intervention.
 			# Definitely need to send out some more EMAIL.
 			# TODO: email notice of broken hardware
-			if not found_within(recent_actions, 'baddisk_notice', 1):
+			if not found_within(recent_actions, 'baddisk_notice', 7):
 				print "...NOTIFYING OWNERS OF BROKEN HARDWARE on %s!!!" % hostname
 				args = {}
 				args['hostname'] = hostname
 				args['log'] = conn.get_dmesg().read()
 
 				sitehist.sendMessage('baddisk_notice', **args)
-				conn.set_nodestate('disable')
+				conn.set_nodestate('disabled')
 
 		elif sequences[s] == "update_hardware_email":
-			if not found_within(recent_actions, 'minimalhardware_notice', 1):
+			if not found_within(recent_actions, 'minimalhardware_notice', 7):
 				print "...NOTIFYING OWNERS OF MINIMAL HARDWARE FAILURE on %s!!!" % hostname
 				args = {}
 				args['hostname'] = hostname