+####################
+class TestInstance:
+ def __init__ (self, buildname, pid=0):
+ self.pids=[]
+ if pid!=0: self.pid.append(pid)
+ self.buildname=buildname
+ # latest trace line
+ self.trace=''
+ # has a KO test
+ self.broken_steps=[]
+ self.timestamp = 0
+
+ def set_timestamp (self,timestamp): self.timestamp=timestamp
+ def set_now (self): self.timestamp=int(time.time())
+ def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
+
+ def is_running (self): return len(self.pids) != 0
+
+ def add_pid (self,pid):
+ self.pids.append(pid)
+ def set_broken (self, plcindex, step):
+ self.broken_steps.append ( (plcindex, step,) )
+
+ def second_letter (self):
+ if not self.broken_steps: return '='
+ else:
+ really_broken = [ step for (i,step) in self.broken_steps if '_ignore' not in step ]
+ # W is for warning like what's in the build mail
+ if len(really_broken)==0: return 'W'
+ else: return 'B'
+
+ def line (self):
+ # make up a 2-letter sign
+ # first letter : '=', unless build is running : '*'
+ double = '*' if self.pids else '='
+ # second letter : '=' if fine, 'W' for warnings (only ignored steps) 'B' for broken
+ letter2 = self.second_letter()
+ double += letter2
+ msg = " %s %s =="%(double,self.buildname)
+ if not self.pids: pass
+ elif len(self.pids)==1: msg += " (pid=%s)"%self.pids[0]
+ else: msg += " !!!pids=%s!!!"%self.pids
+ msg += " @%s"%self.pretty_timestamp()
+ if letter2 != '=':
+ msg2 = ( ' BROKEN' if letter2 == 'B' else ' WARNING' )
+ # sometimes we have an empty plcindex
+ msg += " [%s="%msg2 + " ".join( [ "%s@%s"%(s,i) if i else s for (i,s) in self.broken_steps ] ) + "]"
+ return msg
+
+class TestBox (Box):
+ def __init__ (self,hostname):
+ Box.__init__(self,hostname)
+ self.starting_ips=[]
+ self.test_instances=[]
+
+ def reboot (self, options):
+ # can't reboot a vserver VM
+ self.run_ssh (['pkill','run_log'],"Terminating current runs",
+ dry_run=options.dry_run)
+ self.run_ssh (['rm','-f',Starting.location],"Cleaning %s"%Starting.location,
+ dry_run=options.dry_run)
+
+ def get_test (self, buildname):
+ for i in self.test_instances:
+ if i.buildname==buildname: return i
+
+ # we scan ALL remaining test results, even the ones not running
+ def add_timestamp (self, buildname, timestamp):
+ i=self.get_test(buildname)
+ if i:
+ i.set_timestamp(timestamp)
+ else:
+ i=TestInstance(buildname,0)
+ i.set_timestamp(timestamp)
+ self.test_instances.append(i)
+
+ def add_running_test (self, pid, buildname):
+ i=self.get_test(buildname)
+ if not i:
+ self.test_instances.append (TestInstance (buildname,pid))
+ return
+ if i.pids:
+ print "WARNING: 2 concurrent tests run on same build %s"%buildname
+ i.add_pid (pid)
+
+ def add_broken (self, buildname, plcindex, step):
+ i=self.get_test(buildname)
+ if not i:
+ i=TestInstance(buildname)
+ self.test_instances.append(i)
+ i.set_broken(plcindex, step)
+
+ matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
+ matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace.*:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
+ matcher_grep_missing=re.compile ("grep: /root/(?P<buildname>[^/]+)/logs/trace: No such file or directory")
+ def sense (self, options):
+ print 'tm',
+ self.starting_ips=[x for x in self.backquote_ssh(['cat',Starting.location], trash_err=True).strip().split('\n') if x]
+
+ # scan timestamps on all tests
+ # this is likely to not invoke ssh so we need to be a bit smarter to get * expanded
+ # xxx would make sense above too
+ command=['bash','-c',"grep . /root/*/timestamp /dev/null"]
+ ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
+ for ts_line in ts_lines:
+ if not ts_line.strip(): continue
+ # expect /root/<buildname>/timestamp:<timestamp>
+ try:
+ (ts_file,timestamp)=ts_line.split(':')
+ ts_file=os.path.dirname(ts_file)
+ buildname=os.path.basename(ts_file)
+ timestamp=int(timestamp)
+ t=self.add_timestamp(buildname,timestamp)
+ except: print 'WARNING, could not parse ts line',ts_line
+
+ # let's try to be robust here -- tests that fail very early like e.g.
+ # "Cannot make space for a PLC instance: vplc IP pool exhausted", that occurs as part of provision
+ # will result in a 'trace' symlink to an inexisting 'trace-<>.txt' because no step has gone through
+ # simple 'trace' should exist though as it is created by run_log
+ command=['bash','-c',"grep KO /root/*/logs/trace /dev/null 2>&1" ]
+ trace_lines=self.backquote_ssh (command).split('\n')
+ for line in trace_lines:
+ if not line.strip(): continue
+ m=TestBox.matcher_grep_missing.match(line)
+ if m:
+ buildname=m.group('buildname')
+ self.add_broken(buildname,'','NO STEP DONE')
+ continue
+ m=TestBox.matcher_grep.match(line)
+ if m:
+ buildname=m.group('buildname')
+ plcindex=m.group('plcindex')
+ step=m.group('step')
+ self.add_broken(buildname,plcindex, step)
+ continue
+ header("TestBox.sense: command %r returned line that failed to match\n%s"%(command,line))
+ header(">>%s<<"%line)
+
+ pids = self.backquote_ssh (['pgrep','run_log'],trash_err=True)
+ if not pids: return
+ command=['ls','-ld'] + ["/proc/%s/cwd"%pid for pid in pids.split("\n") if pid]
+ ps_lines=self.backquote_ssh (command).split('\n')
+ for line in ps_lines:
+ if not line.strip(): continue
+ m=TestBox.matcher_proc.match(line)
+ if m:
+ pid=m.group('pid')
+ buildname=m.group('buildname')
+ self.add_running_test(pid, buildname)
+ continue
+ header("TestBox.sense: command %r returned line that failed to match\n%s"%(command,line))
+ header(">>%s<<"%line)
+
+
+ def line (self):
+ return self.hostname_fedora()
+
+ def list (self, verbose=False):
+ # verbose shows all tests
+ if verbose:
+ instances = self.test_instances
+ msg="tests"
+ else:
+ instances = [ i for i in self.test_instances if i.is_running() ]
+ msg="running tests"
+
+ if not instances:
+ header ("No %s on %s"%(msg,self.line()))
+ else:
+ header ("%s on %s"%(msg,self.line()))
+ instances.sort(timestamp_sort)
+ for i in instances: print i.line()
+ # show 'starting' regardless of verbose
+ if self.starting_ips:
+ header ("Starting IP addresses on %s"%self.line())
+ self.starting_ips.sort()
+ for starting in self.starting_ips: print starting
+ else:
+ header ("Empty 'starting' on %s"%self.line())
+