+####################
+class TestInstance:
+ def __init__ (self, pid, buildname):
+ self.pids=[pid]
+ self.buildname=buildname
+ # latest trace line
+ self.trace=''
+ # has a KO test
+ self.broken_steps=[]
+ def add_pid (self,pid):
+ self.pids.append(pid)
+ def set_broken (self,plcindex, step):
+ self.broken_steps.append ( (plcindex, step,) )
+
+ def line (self):
+ msg = " == %s =="%self.buildname
+ msg += " (pid=%s)"%(self.pids)
+ if self.broken_steps:
+ msg += "\n BROKEN IN STEPS "
+ for (i,s) in self.broken_steps: msg += "step=%s,plc=%s"%(s,i)
+ return msg
+
+class TestBox (Box):
+ def __init__ (self,hostname):
+ Box.__init__(self,hostname)
+ self.starting_ips=[]
+ self.test_instances=[]
+
+ def reboot (self, options):
+ # can't reboot a vserver VM
+ self.run_ssh (['pkill','run_log'],"Terminating current runs",
+ dry_run=options.dry_run)
+ self.run_ssh (['rm','-f','/root/starting'],"Cleaning /root/starting",
+ dry_run=options.dry_run)
+
+ def get_test (self, buildname):
+ for i in self.test_instances:
+ if i.buildname==buildname: return i
+
+ def add_test (self, pid, buildname):
+ i=self.get_test(buildname)
+ if i:
+ print "WARNING: 2 concurrent tests run on same build %s"%buildname
+ i.add_pid (pid)
+ return
+ self.test_instances.append (TestInstance (pid,buildname))
+
+ matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
+ matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
+ def sense (self, options):
+ print 't',
+ self.sense_uptime()
+ self.starting_ips=self.backquote_ssh(['cat','/root/starting']).strip().split('\n')
+ pids = self.backquote_ssh (['pgrep','run_log'],trash_err=True)
+ if not pids: return
+ command=['ls','-ld'] + ["/proc/%s/cwd"%pid for pid in pids.split("\n") if pid]
+ ps_lines=self.backquote_ssh (command).split('\n')
+ for line in ps_lines:
+ if not line.strip(): continue
+ m=TestBox.matcher_proc.match(line)
+ if m:
+ pid=m.group('pid')
+ buildname=m.group('buildname')
+ self.add_test(pid, buildname)
+ else: header("command %r returned line that failed to match\n%s"%(command,line))
+ buildnames=[i.buildname for i in self.test_instances]
+ if not buildnames: return
+
+# messy - tail has different output if one or several args
+# command=['tail','-n','1'] + [ "/root/%s/logs/trace"%b for b in buildnames ]
+# trace_lines=self.backquote_ssh (command).split('\n\n')
+# header('TAIL')
+# for line in trace_lines:
+# if not line.strip(): continue
+# print 'line [[[%s]]]'%line
+ command=['grep','KO'] + [ "/root/%s/logs/trace"%b for b in buildnames ] + [ "/dev/null" ]
+ trace_lines=self.backquote_ssh (command).split('\n')
+ for line in trace_lines:
+ if not line.strip(): continue
+ m=TestBox.matcher_grep.match(line)
+ if m:
+ buildname=m.group('buildname')
+ plcindex=m.group('plcindex')
+ step=m.group('step')
+ self.get_test(buildname).set_broken (plcindex, step)
+ else: header("command %r returned line that failed to match\n%s"%(command,line))
+
+
+
+ def line (self):
+ return "%s (%s)"%(self.hostname,self.uptime())
+
+ def list (self):
+ if not self.starting_ips:
+ header ("No starting IP addresses on %s"%self.line())
+ else:
+ header ("IP addresses currently starting up on %s"%self.line())
+ self.starting_ips.sort()
+ for starting in self.starting_ips: print starting
+ if not self.test_instances:
+ header ("No running tests on %s"%self.line())
+ else:
+ header ("Running tests on %s"%self.line())
+ for i in self.test_instances: print i.line()
+