+ except:
+ print('WARNING, could not parse ts line',ts_line)
+
+####################
+class TestInstance:
+ def __init__(self, buildname, pid=0):
+ self.pids = []
+ if pid != 0:
+ self.pid.append(pid)
+ self.buildname = buildname
+ # latest trace line
+ self.trace = ''
+ # has a KO test
+ self.broken_steps = []
+ self.timestamp = 0
+
+ def set_timestamp(self, timestamp):
+ self.timestamp = timestamp
+ def set_now(self):
+ self.timestamp = int(time.time())
+ def pretty_timestamp(self):
+ return time.strftime("%Y-%m-%d:%H-%M", time.localtime(self.timestamp))
+ def is_running (self):
+ return len(self.pids) != 0
+ def add_pid(self, pid):
+ self.pids.append(pid)
+ def set_broken(self, plcindex, step):
+ self.broken_steps.append( (plcindex, step,) )
+
+ def second_letter(self):
+ if not self.broken_steps:
+ return '='
+ else:
+ really_broken = [ step for (i,step) in self.broken_steps if '_ignore' not in step ]
+ # W is for warning like what's in the build mail
+ if len(really_broken) == 0:
+ return 'W'
+ else:
+ return 'B'
+
+ def line (self):
+ # make up a 2-letter sign
+ # first letter : '=', unless build is running : '*'
+ double = '*' if self.pids else '='
+ # second letter : '=' if fine, 'W' for warnings (only ignored steps) 'B' for broken
+ letter2 = self.second_letter()
+ double += letter2
+ msg = " {} {} ==".format(double, self.buildname)
+ if not self.pids:
+ pass
+ elif len(self.pids)==1:
+ msg += " (pid={})".format(self.pids[0])
+ else:
+ msg += " !!!pids={}!!!".format(self.pids)
+ msg += " @{}".format(self.pretty_timestamp())
+ if letter2 != '=':
+ msg2 = ( ' BROKEN' if letter2 == 'B' else ' WARNING' )
+ # sometimes we have an empty plcindex
+ msg += " [{}=".format(msg2) \
+ + " ".join(["{}@{}".format(s, i) if i else s for (i, s) in self.broken_steps]) \
+ + "]"
+ return msg
+
+class TestBox(Box):
+ def __init__(self, hostname):
+ Box.__init__(self, hostname)
+ self.starting_ips = []
+ self.test_instances = []
+
+ def reboot(self, options):
+ # can't reboot a vserver VM
+ self.run_ssh(['pkill', 'run_log'], "Terminating current runs",
+ dry_run=options.dry_run)
+ self.run_ssh(['rm', '-f', Starting.location], "Cleaning {}".format(Starting.location),
+ dry_run=options.dry_run)
+
+ def get_test(self, buildname):
+ for i in self.test_instances:
+ if i.buildname == buildname:
+ return i
+
+ # we scan ALL remaining test results, even the ones not running
+ def add_timestamp(self, buildname, timestamp):
+ i = self.get_test(buildname)
+ if i:
+ i.set_timestamp(timestamp)
+ else:
+ i = TestInstance(buildname, 0)
+ i.set_timestamp(timestamp)
+ self.test_instances.append(i)
+
+ def add_running_test(self, pid, buildname):
+ i = self.get_test(buildname)
+ if not i:
+ self.test_instances.append(TestInstance(buildname, pid))
+ return
+ if i.pids:
+ print("WARNING: 2 concurrent tests run on same build {}".format(buildname))
+ i.add_pid(pid)
+
+ def add_broken(self, buildname, plcindex, step):
+ i = self.get_test(buildname)
+ if not i:
+ i = TestInstance(buildname)
+ self.test_instances.append(i)
+ i.set_broken(plcindex, step)
+
+ matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
+ matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace.*:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
+ matcher_grep_missing=re.compile ("grep: /root/(?P<buildname>[^/]+)/logs/trace: No such file or directory")
+
+ def sense(self, options):
+ print('tm', end=' ')
+ self.starting_ips = [x for x in self.backquote_ssh(['cat',Starting.location], trash_err=True).strip().split('\n') if x]
+
+ # scan timestamps on all tests
+ # this is likely to not invoke ssh so we need to be a bit smarter to get * expanded
+ # xxx would make sense above too
+ command = ['bash', '-c', "grep . /root/*/timestamp /dev/null"]
+ ts_lines = self.backquote_ssh(command, trash_err=True).split('\n')
+ for ts_line in ts_lines:
+ if not ts_line.strip():
+ continue
+ # expect /root/<buildname>/timestamp:<timestamp>
+ try:
+ (ts_file, timestamp) = ts_line.split(':')
+ ts_file = os.path.dirname(ts_file)
+ buildname = os.path.basename(ts_file)
+ timestamp = int(timestamp)
+ t = self.add_timestamp(buildname, timestamp)
+ except:
+ print('WARNING, could not parse ts line', ts_line)
+
+ # let's try to be robust here -- tests that fail very early like e.g.
+ # "Cannot make space for a PLC instance: vplc IP pool exhausted", that occurs as part of provision
+ # will result in a 'trace' symlink to an inexisting 'trace-<>.txt' because no step has gone through
+ # simple 'trace' should exist though as it is created by run_log
+ command = ['bash', '-c', "grep KO /root/*/logs/trace /dev/null 2>&1" ]
+ trace_lines = self.backquote_ssh(command).split('\n')
+ for line in trace_lines:
+ if not line.strip():
+ continue
+ m = TestBox.matcher_grep_missing.match(line)
+ if m:
+ buildname = m.group('buildname')
+ self.add_broken(buildname, '', 'NO STEP DONE')
+ continue
+ m = TestBox.matcher_grep.match(line)
+ if m:
+ buildname = m.group('buildname')
+ plcindex = m.group('plcindex')
+ step = m.group('step')
+ self.add_broken(buildname, plcindex, step)
+ continue
+ header("TestBox.sense: command {} returned line that failed to match\n{}".format(command, line))
+ header(">>{}<<".format(line))
+
+ pids = self.backquote_ssh (['pgrep', 'run_log'], trash_err=True)
+ if not pids:
+ return
+ command = ['ls','-ld'] + ["/proc/{}/cwd".format(pid) for pid in pids.split("\n") if pid]
+ ps_lines = self.backquote_ssh(command).split('\n')
+ for line in ps_lines:
+ if not line.strip():
+ continue
+ m = TestBox.matcher_proc.match(line)
+ if m:
+ pid = m.group('pid')
+ buildname = m.group('buildname')
+ self.add_running_test(pid, buildname)
+ continue
+ header("TestBox.sense: command {} returned line that failed to match\n{}".format(command, line))
+ header(">>{}<<".format(line))
+
+
+ def line (self):
+ return self.hostname_fedora()
+
+ def list (self, verbose=False):
+ # verbose shows all tests
+ if verbose:
+ instances = self.test_instances
+ msg="tests"
+ else:
+ instances = [ i for i in self.test_instances if i.is_running() ]
+ msg="running tests"
+
+ if not instances:
+ header ("No {} on {}".format(msg, self.line()))
+ else:
+ header ("{} on {}".format(msg, self.line()))
+ instances.sort(key=timestamp_key)
+ for i in instances:
+ print(i.line())
+ # show 'starting' regardless of verbose
+ if self.starting_ips:
+ header("Starting IP addresses on {}".format(self.line()))
+ self.starting_ips.sort()
+ for starting in self.starting_ips:
+ print(starting)
+ else:
+ header("Empty 'starting' on {}".format(self.line()))