X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=monitor%2Fbootman.py;h=8dda34116c827ee287bdb935a8e8553a216f9d30;hb=6a2f47ac05289fd850dc724e0da13599e7846294;hp=2afbbf6347823434bf7e4782e5db18f7b694caef;hpb=7028fdb5e122089f2e1fb1c6a9178ab8f8aa5fe1;p=monitor.git

diff --git a/monitor/bootman.py b/monitor/bootman.py
index 2afbbf6..8dda341 100755
--- a/monitor/bootman.py
+++ b/monitor/bootman.py
@@ -68,18 +68,34 @@ class NodeConnection:
 		return "unknown"
 
 	def get_dmesg(self):
+		t_stamp = time.strftime("%Y-%m-%d-%H:%M")
 		self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
-		download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node)
-		log = open("log/dmesg.%s.log" % self.node, 'r')
+		download(self.c, "/var/log/dmesg.bm.log", "%s/history/%s-dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node))
+		os.system("cp %s/history/%s-dmesg.%s.log %s/dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node, config.MONITOR_BOOTMANAGER_LOG, self.node))
+		log = open("%s/dmesg.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r')
 		return log
 
 	def get_bootmanager_log(self):
-		download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node)
-		#os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node))
-		os.system("cp log/bm.%s.log.gz log/bm.%s.log" % (self.node, self.node))
-		log = open("log/bm.%s.log" % self.node, 'r')
+		t_stamp = time.strftime("%Y-%m-%d-%H:%M")
+		download(self.c, "/tmp/bm.log", "%s/history/%s-bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node))
+		os.system("cp %s/history/%s-bm.%s.log %s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node, config.MONITOR_BOOTMANAGER_LOG, self.node))
+		log = open("%s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r')
 		return log
 
+
+#	def get_dmesg(self):
+#		self.c.modules.os.system("dmesg > /var/log/dmesg.bm.log")
+#		download(self.c, "/var/log/dmesg.bm.log", "log/dmesg.%s.log" % self.node)
+#		log = open("log/dmesg.%s.log" % self.node, 'r')
+#		return log
+#
+#	def get_bootmanager_log(self):
+#		download(self.c, "/tmp/bm.log", "log/bm.%s.log.gz" % self.node)
+#		#os.system("zcat log/bm.%s.log.gz > log/bm.%s.log" % (self.node, self.node))
+#		os.system("cp log/bm.%s.log.gz log/bm.%s.log" % (self.node, self.node))
+#		log = open("log/bm.%s.log" % self.node, 'r')
+#		return log
+
 	def dump_plconf_file(self):
 		c = self.c
 		self.c.modules.sys.path.append("/tmp/source/")
@@ -627,8 +643,12 @@ class DebugInterface:
 
 		return sequence
 		
-
 def restore(sitehist, hostname, config=None, forced_action=None):
+	ret = restore_basic(sitehist, hostname, config, forced_action)
+	session.flush()
+	return ret
+
+def restore_basic(sitehist, hostname, config=None, forced_action=None):
 
 	# NOTE: Nothing works if the bootcd is REALLY old.
 	#       So, this is the first step.
@@ -639,7 +659,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 	if fbnode['observed_category'] == "OLDBOOTCD":
 		print "\t...Notify owner to update BootImage!!!"
 
-		if not found_within(recent_actions, 'newbootcd_notice', 3):
+		if not found_within(recent_actions, 'newbootcd_notice', 3.5):
 			sitehist.sendMessage('newbootcd_notice', hostname=hostname)
 
 			print "\tDisabling %s due to out-of-date BootImage" % hostname
@@ -650,16 +670,8 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 
 	debugnode = DebugInterface(hostname)
 	conn = debugnode.getConnection()
-	#print "conn: %s" % conn
-	#print "trying to use conn after returning it."
-	#print conn.c.modules.sys.path
-	#print conn.c.modules.os.path.exists('/tmp/source')
 	if type(conn) == type(False): return False
 
-	#if forced_action == "reboot":
-	#	conn.restart_node('reinstall')
-	#	return True
-
 	boot_state = conn.get_boot_state()
 	if boot_state != "debug":
 		print "... %s in %s state: skipping..." % (hostname , boot_state)
@@ -689,7 +701,8 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 			print "...Should investigate.  Skipping node."
 			# TODO: send message related to these errors.
 
-			if not found_within(recent_actions, 'newbootcd_notice', 3):
+			if not found_within(recent_actions, 'baddisk_notice', 7):
+				print "baddisk_notice not found recently"
 
 				log=conn.get_dmesg().read()
 				sitehist.sendMessage('baddisk_notice', hostname=hostname, log=log)
@@ -730,6 +743,8 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 		args['sequence'] = s
 		args['bmlog'] = conn.get_bootmanager_log().read()
 		args['viart'] = False
+		args['saveact'] = True
+		args['ccemail'] = True
 
 		sitehist.sendMessage('unknownsequence_notice', **args)
 
@@ -768,6 +783,8 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 			args['sequence'] = s
 			args['bmlog'] = conn.get_bootmanager_log().read()
 			args['viart'] = False
+			args['saveact'] = True
+			args['ccemail'] = True
 
 			sitehist.sendMessage('unknownsequence_notice', **args)
 			conn.restart_bootmanager('boot')
@@ -775,7 +792,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 		# TODO: differentiate this and the 'nodenetwork_email' actions.
 		elif sequences[s] == "update_node_config_email":
 
-			if not found_within(recent_actions, 'nodeconfig_notice', 3):
+			if not found_within(recent_actions, 'nodeconfig_notice', 3.5):
 				args = {}
 				args['hostname'] = hostname
 				sitehist.sendMessage('nodeconfig_notice', **args)
@@ -783,7 +800,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 
 		elif sequences[s] == "nodenetwork_email":
 
-			if not found_within(recent_actions, 'nodeconfig_notice', 3):
+			if not found_within(recent_actions, 'nodeconfig_notice', 3.5):
 				args = {}
 				args['hostname'] = hostname
 				args['bmlog'] = conn.get_bootmanager_log().read()
@@ -792,7 +809,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 
 		elif sequences[s] == "update_bootcd_email":
 
-			if not found_within(recent_actions, 'newalphacd_notice', 3):
+			if not found_within(recent_actions, 'newalphacd_notice', 3.5):
 				args = {}
 				args.update(getconf.getconf(hostname)) # NOTE: Generates boot images for the user:
 				args['hostname'] = hostname
@@ -806,7 +823,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 			# require either an exception "/minhw" or other manual intervention.
 			# Definitely need to send out some more EMAIL.
 			# TODO: email notice of broken hardware
-			if not found_within(recent_actions, 'baddisk_notice', 1):
+			if not found_within(recent_actions, 'baddisk_notice', 7):
 				print "...NOTIFYING OWNERS OF BROKEN HARDWARE on %s!!!" % hostname
 				args = {}
 				args['hostname'] = hostname
@@ -816,7 +833,7 @@ def restore(sitehist, hostname, config=None, forced_action=None):
 				conn.set_nodestate('disabled')
 
 		elif sequences[s] == "update_hardware_email":
-			if not found_within(recent_actions, 'minimalhardware_notice', 1):
+			if not found_within(recent_actions, 'minimalhardware_notice', 7):
 				print "...NOTIFYING OWNERS OF MINIMAL HARDWARE FAILURE on %s!!!" % hostname
 				args = {}
 				args['hostname'] = hostname