X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=qaapi%2Fsystem-test.py;h=498c7f4369264f1653d32b7fae5372590afd7560;hb=9e5cba850b98b3b1fb5c9f51b94523eb29689d98;hp=46b288fa24c47ed69813ee37b89343b19b0e346a;hpb=cae623a51b09f180028580a8c7c4acf9564ecd00;p=tests.git diff --git a/qaapi/system-test.py b/qaapi/system-test.py index 46b288f..498c7f4 100755 --- a/qaapi/system-test.py +++ b/qaapi/system-test.py @@ -2,69 +2,58 @@ import commands import os import sys +import traceback from optparse import OptionParser from random import Random from time import localtime +from qa.sendmail import sendmail from qa.utils import commands +from qa.logger import Logfile from qa import utils from qa.Config import Config +from qa.Step import Step from qa.tests.vserver_create import vserver_create from qa.tests.vserver_delete import vserver_delete from qa.tests.plc_configure import plc_configure from qa.tests.plc_stop import plc_stop from qa.tests.plc_start import plc_start from qa.tests.add_test_data import add_test_data +from qa.tests.api_unit_test import api_unit_test from qa.tests.sync_person_key import sync_person_key from qa.tests.boot_node import boot_node -from qa.tests.access_slice import access_slice - -def run_system_tests(config, vserver_name, plc_name): - # configure the plc in this vserver - config.plcs[plc_name]['vserver'] = vserver_name - config.plcs[plc_name]['host'] = config.hostname - config.plcs[plc_name]['ip'] = config.ip - config.plcs[plc_name].update_ip() - - plc_configure(config)(plc_name) - options = {} - for service in ['API', 'WWW', 'BOOT',' DB']: - options['PLC_'+service+'_HOST'] = config.plcs[plc_name]['ip'] - options['PLC_'+service+'_IP'] = config.plcs[plc_name]['ip'] - plc_configure(config)(plc_name, options) - plc_start(config)(plc_name) - - # Add test site, node, person and slice data - # Adds slice to node and person to slice - add_test_data(config)(plc_name) - plc_stop(config)(plc_name) - plc_start(config)(plc_name) - person_email = config.persons.values()[0]['email'] - sync_person_key(config)(person_email) - - # Boot test node and confirm boot state - nodelist = ['vm1.paris.cs.princeton.edu'] - for node in nodelist: - if not node in config.nodes.keys(): - continue - node = config.nodes[node] - node['vserver'] = config.plcs[plc_name]['vserver'] - boot_node(config)(plc_name, node['hostname']) +from qa.tests.node_run_tests import node_run_tests def create_vserver(vserver_name, vserver_home, mailto): # create vserver for this system test if it doesnt already exist if not os.path.isdir('%(vserver_home)s/%(vserver_name)s' % locals()): vserver_create(config)(vserver_name, distro, mailto) +def stop_vservers(prefix = 'plc', exempt = []): + + # stop all running vservers + vserver_stat = "vserver-stat | grep %(prefix)s | awk '{print$8}'" % locals() + (stdin, stdout, stderr) = os.popen3(vserver_stat) + vservers = [line.strip() for line in stdout.readlines()] + vservers = filter(lambda x: x not in exempt, vservers) + for vserver in vservers: + try: + utils.header("Stopping %(vserver)s " % locals()) + stop_cmd = "vserver %(vserver)s stop" % locals() + (status, output) = commands("vserver %(vserver)s stop" % locals()) + except: + print "Failed to stop %(vserver)s" % locals() + utils.header("%s" % traceback.format_exc()) + def cleanup_vservers(max_vservers, vserver_home, vserver_basename): # only keep the newest MAX_VSERVERS vservers = os.listdir("%(vserver_home)s" % locals()) - valid_vservers = lambda vserver: vserver.startswith(vserver_basename) + valid_vservers = lambda vserver: vserver.startswith(vserver_basename) and os.path.isdir(vserver) vservers = filter(valid_vservers, vservers) vservers.sort() vservers.reverse() - expired_vservers = vservers[5:] + expired_vservers = vservers[max_vservers:] for vserver in expired_vservers: - utils.header("Deleting vserver: %(vserver)s" % locals()) + utils.header("Deleting vserver: %(vserver)s" % locals(), logfile = config.logfile) #vserver_delete()(vserver) usage=""" @@ -76,49 +65,153 @@ parser.add_option("-d", "--distro", help = "Fedora distro to use") parser.add_option("-p", "--plcname", help = "Which plc do we use (from config file)") parser.add_option("-m", "--mailto", help = "Vserver build mailto address") -(options, args) = parser.parse_args() +# Define globals +# Determine vserver name, distribution and mailto +# The distribution and current date will be part of of the vserver name +MAX_VSERVERS = 3 +VSERVER_HOME = '/vservers/' +VSERVER_BASENAME = 'plc' +distro = 'f8' +# use todays date and defaults to determine which vservers to run tests in +year, month, day = localtime()[:3] +YEAR, MONTH, DAY = [str(x) for x in [year,month,day]] +DATE = ".".join([YEAR, MONTH, DAY]) +vserver_name = "%(VSERVER_BASENAME)s-%(distro)s-%(DATE)s" % locals() +(options, args) = parser.parse_args() # choose which distros to use if options.distro is not None: distros = [options.distro] -else: distros = ['f7'] +else: distros = ['f8'] + # did the user specify a vserver or plc -vserver_name = options.vserver +vserver_names = [] +if options.vserver is not None: + vserver_name = options.vserver + vserver_names.append(vserver_name) + if options.plcname is not None: plc_name = options.plcname else: plc_name = 'TestPLC' + # who gets emailed if options.mailto is not None: mailto = options.mailto else: mailto = 'tmack@cs.princeton.edu' -# Define globals -# Determine vserver name, distribution and mailto -# The distribution and current date will be part of of the vserver name -MAX_VSERVERS = 5 -VSERVER_HOME = '/vservers/' -VSERVER_BASENAME = 'plc' - -# Setup configuration -config = Config() -config.load("qa/qa_config.py") - -if vserver_name: - # run tests in vserver specified by user - create_vserver(vserver_name, VSERVER_HOME, mailto) - run_system_tests(config, vserver_name, plc_name) -else: - - # use todays date and defaults to determine which vservers to run tests in - year, month, day = localtime()[:3] - YEAR, MONTH, DAY = [str(x) for x in [year,month,day]] - DATE = ".".join([YEAR, MONTH, DAY]) +if not options.vserver: for distro in distros: - vserver_name = "%(VSERVER_BASENAME)s-%(distro)s-%(DATE)s" % locals() - create_vserver(vserver_name, VSERVER_HOME, mailto) - try: - run_system_tests(config, vserver_name, plc_name) - except: - utils.header("ERROR %(vserver_name)s tests failed" % locals()) - raise + vserver_names.append("%(VSERVER_BASENAME)s-%(distro)s-%(DATE)s" % locals()) + +stop_vservers(exempt = vserver_names) + +for vserver_name in vserver_names: + try: + # Setup configuration + logfile_dir = "/var/log/qaapi/%(vserver_name)s" % locals() + config = Config(logdir = logfile_dir) + config.load("qa/qa_config.py") + + config.plcs[plc_name]['vserver'] = vserver_name + config.plcs[plc_name]['host'] = config.hostname + config.plcs[plc_name]['ip'] = config.ip + config.plcs[plc_name].update_ip() + person_email = config.persons.values()[0]['email'] + + # Set plc configuration options + config_options = {} + for service in ['API', 'WWW', 'BOOT',' DB']: + config_options['PLC_'+service+'_HOST'] = config.plcs[plc_name]['ip'] + config_options['PLC_'+service+'_IP'] = config.plcs[plc_name]['ip'] + config_options['PLC_ROOT_USER'] = 'root@localhost.localdomain' + config_options['PLC_ROOT_PASSWORD'] = 'root' + + # Set node configuration options + nodelist = ['vm1.paris.cs.princeton.edu'] + node_tests = ['node_cpu_sched.py', 'pf2test.pl' ] + slice = config.slices['ts_slice1'] + + steps = {} + steps[1] = Step("Create vserver %s" % vserver_name, create_vserver, + (vserver_name, VSERVER_HOME, mailto), config.logfile.filename) + steps[2] = Step("Mount plc %s " % plc_name, config.plcs[plc_name].commands, + ("/sbin/service plc mount",), config.logfile.filename) + steps[3] = Step("Configure plc %s" % plc_name, plc_configure(config), (plc_name, config_options,), + config.logfile.filename) + steps[4] = Step("Start plc %s " % plc_name, plc_start(config), (plc_name,), + config.logfile.filename) + steps[5] = Step("Add test data", add_test_data(config), (plc_name,), config.logfile.filename) + steps[6] = Step("Sync person public key", sync_person_key(config), (person_email,), + config.logfile.filename, False) + # XX fix logfile parameter + step_method = api_unit_test(config) + #steps[7] = Step("API unit test", step_method, (plc_name,), step_method.logfile.filename, False) + + for node in nodelist: + if not node in config.nodes.keys(): continue + node = config.nodes[node] + node['vserver'] = vserver_name + step_num = max(steps.keys()) + 1 + + # Boot node + steps[step_num] = Step("Boot node %s" % node['hostname'], boot_node(config), + (plc_name, node['hostname']), node.logfile.filename) + + # Check if node is fully booted + ready_step = Step("Check %s is ready" % node['hostname'], node.is_ready, (), node.logfile.filename) + steps[step_num].next_steps.append(ready_step) + + # Download test scripts + download_scripts = Step("Download test scripts onto %s" % node['hostname'], node.download_testscripts, + (), config.logfile.filename) + steps[step_num].next_steps.append(download_scripts) + + + # XX fix logfile parameter + # XX node_run_tests should only run the test, download test on node before calling node_run_test + for test in node_tests: + # Create a separate logfile for this script + log_filename = "%s/%s-%s.log" % (config.logfile.dir, test, node['hostname']) + test_logfile = Logfile(log_filename) + step_method = node_run_tests(config, test_logfile) + test_step = Step("%s test on node %s" % (test, node['hostname']), + step_method, (node['hostname'], plc_name, test), + step_method.logfile.filename, False) + steps[step_num].next_steps.append(test_step) + + # enter test slice + step_num = step_num + 1 + steps[step_num] = Step("Enter slice %s on %s" % (slice['name'], node['hostname']), + node.slice_commands, ("echo `whoami`@`hostname`", slice['name']), + node.logfile.filename, False) + + # Copy contents of /var/log + step_num = step_num + 1 + steps[step_num] = Step("Get %s logs" % node['hostname'], node.get_logs, (), + node.logfile.filename, False) + + # Now that all the steps are defined, run them + order = steps.keys() + order.sort() + results = {} + for num in order: + steps[num].run() + steps[num].notify_contacts() + if steps[num].fatal and not steps[num].passed: + break + + # Generate summary email + to = ["qa@planet-lab.org"] + subject = "PLC - %(distro)s - %(DATE)s QA summary" % locals() + body = """ + The following are the results of QA tests for %(DATE)s %(distro)s MyPLC.\n\n""" % locals() -# remove old vsevers -cleanup_vservers(MAX_VSERVERS, VSERVER_HOME, VSERVER_BASENAME) + utils.header("Sending summary email") + # add results to summary body + for num in order: + step = steps[num] + body += step.get_results() + + sendmail(to, subject, body) + sendmail(['tmack@cs.princeton.edu'], subject, body) + except: + utils.header("ERROR %(vserver_name)s tests failed" % locals(), logfile = config.logfile) + utils.header("%s" % traceback.format_exc(), logfile = config.logfile)