From: Claudio-Daniel Freire Date: Wed, 5 Oct 2011 17:06:46 +0000 (-0300) Subject: POPI experiment moved into nepi X-Git-Tag: nepi-3.0.0~171^2~6 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=d91be2d553803234fb22fcf2f3e8d060b80d625e;p=nepi.git POPI experiment moved into nepi --- diff --git a/examples/POPI/popi-0.9-rpy2.patch b/examples/POPI/popi-0.9-rpy2.patch new file mode 100644 index 00000000..269e1d22 --- /dev/null +++ b/examples/POPI/popi-0.9-rpy2.patch @@ -0,0 +1,37 @@ +diff -ur popi-0.9/rlranklib.py popi-0.9-rpy2/rlranklib.py +--- popi-0.9/rlranklib.py 2011-08-17 15:30:31.000000000 +0200 ++++ popi-0.9-rpy2/rlranklib.py 2011-09-16 05:28:16.547000068 +0200 +@@ -5,10 +5,13 @@ + # $Header: /data/cvsroot/plportest/anadata/rlranklib.py,v 1.11.2.1 2007/11/26 08:46:29 lgh Exp $ + + import random +-from rpy import * ++import rpy2.rpy_classic ++from rpy2.rpy_classic import * + from math import sqrt, log + from sets import Set + ++rpy2.rpy_classic.set_default_mode(rpy2.rpy_classic.BASIC_CONVERSION) ++ + safe_eff = [None,1.365,1.208,1.116,1.110,1.104,1.064,1.064] + + def calc_rlrank(lrs, opt_ranktype): +@@ -66,7 +69,7 @@ + riag = [ rlranks[i] for i in range(0, len(rlranks)) if partition[i] == igrp ] # rlranks in a group + if len(riag) == 1: + continue +- threshold = r.qtukey(alpha, len(riag), 1000)*sqrt(1.0/12/nb)*len(riag)/len(rlranks) ++ threshold = r.qtukey(alpha, len(riag), 1000)[0]*sqrt(1.0/12/nb)*len(riag)/len(rlranks) + range_ = max(riag) - min(riag) + if verbose: + print " ", igrp, len(riag), min(riag), max(riag), range_, "%.3f" % threshold, +@@ -99,7 +102,7 @@ + return True + + # t = r.qtukey(alpha, len(arrs), 1000)*sqrt(1.0/12/nb)*(len(arrs)+1)/ksum +- t = r.qtukey(alpha, km, 1000) * sqrt((km*km-1)/12.0/nb) / ksum ++ t = r.qtukey(alpha, km, 1000)[0] * sqrt((km*km-1)/12.0/nb) / ksum + if km <= len(safe_eff): + t = t * safe_eff[km-1] + if max(arrs) - min(arrs) > t: +Binary files popi-0.9/rlranklib.pyc and popi-0.9-rpy2/rlranklib.pyc differ diff --git a/examples/POPI/popi-0.9.tar.gz b/examples/POPI/popi-0.9.tar.gz new file mode 100644 index 00000000..9f117072 Binary files /dev/null and b/examples/POPI/popi-0.9.tar.gz differ diff --git a/examples/POPI/popi-tun-classfilter-2MB-q500-pl-hibw.xml b/examples/POPI/popi-tun-classfilter-2MB-q500-pl-hibw.xml new file mode 100644 index 00000000..976fb551 --- /dev/null +++ b/examples/POPI/popi-tun-classfilter-2MB-q500-pl-hibw.xml @@ -0,0 +1,259 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/POPI/popi-tun-classfilter-2MB-q500-pl.xml b/examples/POPI/popi-tun-classfilter-2MB-q500-pl.xml new file mode 100644 index 00000000..c73848d6 --- /dev/null +++ b/examples/POPI/popi-tun-classfilter-2MB-q500-pl.xml @@ -0,0 +1,241 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/POPI/run_one_experiment.py b/examples/POPI/run_one_experiment.py new file mode 100644 index 00000000..0b94995b --- /dev/null +++ b/examples/POPI/run_one_experiment.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from nepi.core.execute import ExperimentController +import os +import re +import time +import commands +import subprocess +import collections +import signal +import traceback +import shutil +import sys +import cPickle +import gzip + +class PopiExperiment(object): + def run(self, duration, xml_filepath, testset, results_dir, iteration): + app_guid = 8 + + testset_dir = os.path.join(results_dir, testset) + + # create test results file + test_dir = os.path.join(testset_dir, str(iteration)) + + # replace results values in xml + replacements = cPickle.loads(os.environ['POPI_REPLACEMENTS'].strip().decode("base64")) + file = open(xml_filepath) + xml2 = xml = file.read() + file.close() + + for key,value in replacements.iteritems(): + xml2 = xml2.replace("##%s##" % (key,), value) + + # launch experiment + controller = ExperimentController(xml2, results_dir) + + try: + controller.start() + + t0 = time.time() + t1 = t0 + while (t1-t0) < duration and not controller.is_finished(app_guid): + time.sleep(10) + + # download results + for testbed_guid, guids in controller.traces_info().iteritems(): + for guid, traces in guids.iteritems(): + for name, data in traces.iteritems(): + path = data["filepath"] + print >>sys.stderr, "Downloading trace", path + + filepath = os.path.join(test_dir, path) + + try: + trace = controller.trace(guid, name) + except: + traceback.print_exc(file=sys.stderr) + continue + try: + if not os.path.exists(os.path.dirname(filepath)): + os.makedirs(os.path.dirname(filepath)) + except: + traceback.print_exc(file=sys.stderr) + + try: + if len(trace) >= 2**20: + # Bigger than 1M, compress + tracefile = gzip.GzipFile(filepath+".gz", "wb") + else: + tracefile = open(filepath,"wb") + try: + tracefile.write(trace) + finally: + tracefile.close() + except: + traceback.print_exc(file=sys.stderr) + + finally: + # clean up + try: + controller.stop() + except: + pass + try: + controller.shutdown() + except: + pass + + def results_append(self, file, testset, sta_pcap, ap_pcap): + line = "%s %s %s\n" % (testset, sta_pcap, ap_pcap) + file.write(line) + +if __name__ == '__main__': + experiment = PopiExperiment() + duration = int(sys.argv[1]) + xml_filepath = sys.argv[2] + testset = sys.argv[3] + results_dir = sys.argv[4] + iteration = sys.argv[5] + experiment.run(duration, xml_filepath, testset, results_dir, iteration) + diff --git a/examples/POPI/run_popi_experiments.py b/examples/POPI/run_popi_experiments.py new file mode 100644 index 00000000..dcb07e5c --- /dev/null +++ b/examples/POPI/run_popi_experiments.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from nepi.core.execute import ExperimentController +from optparse import OptionParser, SUPPRESS_HELP +import collections +import commands +import os +import shutil +import signal +import subprocess +import sys +import time +import traceback +import getpass +import cPickle + +class PopiExample(object): + _testsets = dict({ + "popi": "./popi-tun-classfilter-2MB-q500-pl.xml", + "popi_hibw": "./popi-tun-classfilter-2MB-q500-pl-hibw.xml", + }) + + classes = { + 'tcpx4' : 'udp:tcp*4:icmp:', + 'icmpx4' : 'udp:tcp:icmp*4:', + 'udpx4' : 'udp*4:tcp:icmp:', + 'u1t4i16' : 'udp:tcp*4:icmp*16:', + 'u4t4i16' : 'udp*4:tcp*4:icmp*16:', + 'u1t16i16' : 'udp*4:tcp*16:icmp*16:', + 'u1t1t1' : 'udp:tcp:icmp:', + } + + bwlimits = { + '32K' : '32', + '64K' : '64', + '128K' : '128', + '256K' : '256', + '384K' : '384', + # '512K' : '512', + # '768K' : '768', + # '1M' : '1024', + # '2M' : '2048', + } + + testsets = dict([ + ("%s-%s-%s" % (tset,clsname,bwname), (xml, {'classes':cls, 'bwlimit':bw})) + for tset,xml in _testsets.iteritems() + for clsname,cls in classes.iteritems() + for bwname,bw in bwlimits.iteritems() + ]) + + def __init__(self): + usage = "usage: %prog -u user -t times -d results_dir -f remove -e experiment -s start" + parser = OptionParser(usage=usage) + parser.add_option("-u", "--user", dest="pluser", help="PlanetLab PLC user (email)", type="str") + parser.add_option("-p", "--pass", dest="plpass", help="PlanetLab PLC user (password) - leave empty for interactive prompt", type="str") + parser.add_option("-k", "--key", dest="plkey", help="PlanetLab PLC private key to use", type="str") + parser.add_option("-S", "--slice", dest="plslice", help="PlanetLab slice into which to deploy experiments", type="str") + parser.add_option("-t", "--times", dest="times", help="Number of times to run each scenario", type="int") + parser.add_option("-d", "--dir", dest="results_dir", help="Results directory", type="str") + parser.add_option("-f", "--remove", dest="remove", help="Remove previous results directory", action="store_true", default=False) + parser.add_option("-e", "--experiment", dest="experiment", help="Experiment to execute [%s]" % ('|'.join(self._testsets.keys()),), type="str") + parser.add_option("-s", "--start", dest="start", help="Start experiment at specific iteration", type="int") + (options, args) = parser.parse_args() + + if not options.pluser: + print >>sys.stderr, "Must specify --user" + sys.exit(1) + else: + self.pluser = options.pluser + + if not options.plslice: + print >>sys.stderr, "Must specify --slice" + sys.exit(1) + else: + self.plslice = options.plslice + + if not options.plkey: + print >>sys.stderr, "Must specify --key" + sys.exit(1) + else: + self.plkey = options.plkey + + if not options.plpass: + self.plpass = getpass.getpass("Password for %s: " % (self.pluser,)) + + self.times = options.times if options.times else 5 + self.results_dir = options.results_dir if options.results_dir else "results" + self.remove = options.remove + if options.experiment: + if ',' in options.experiment: + options.experiment = options.experiment.split(',') + else: + options.experiment = [ options.experiment ] + else: + options.experiment = self.testsets.keys() + self.experiments = [x for x in options.experiment if x in self.testsets] + self.start = options.start if options.start else 0 + + def run(self): + duration = 3600 + + if self.remove: + try: + shutil.rmtree(self.results_dir) + except: + traceback.print_exc(file=sys.stderr) + + try: + os.mkdir(self.results_dir) + except: + traceback.print_exc(file=sys.stderr) + + for j,testset in enumerate(self.experiments): + xml_filepath, replacements = self.testsets[testset] + replacements = dict(replacements) + replacements['pluser'] = self.pluser + replacements['plpass'] = self.plpass + replacements['plslice'] = self.plslice + replacements['plkey'] = self.plkey + + for i in xrange(self.start, self.times): + testset_dir = os.path.join(self.results_dir, testset, str(i)) + os.makedirs(testset_dir) + + print >>sys.stderr, "%3d%% - " % ((j+i*1.0/(self.times-self.start))*100/len(self.experiments),), testset, "...", + + # launch experiment + command = "python run_one_experiment.py %d '%s' '%s' '%s' %d" % \ + (duration, xml_filepath, testset, self.results_dir, i) + # send by environment, we don't want passwords in the commandline + env = dict(os.environ) + env['POPI_REPLACEMENTS'] = cPickle.dumps(replacements,2).encode("base64").strip() + + for trials in xrange(5): + logfile = open(os.path.join(testset_dir,"log"), "w") + p = subprocess.Popen( + command, + shell = True, + env = env, + stdout = logfile, + stderr = logfile, + stdin = open("/dev/null","rb") ) + + # we wait two time the estimated dirantion of the movie (120s) + for i in xrange(0, duration * 2, 10): + time.sleep(10) + returncode = p.poll() + if returncode is not None: + break + time.sleep(10) + try: + os.kill(p.pid, signal.SIGKILL) + except: + pass + + logfile.close() + retfile = open(os.path.join(testset_dir,"retcode"), "w") + if returncode: + rettext = "FAIL %s" % (returncode,) + else: + rettext = "SUCCESS" + retfile.write(rettext) + retfile.close() + + print >>sys.stderr, rettext, + + if not returncode: + print >>sys.stderr + break + else: + time.sleep(60) + else: + print >>sys.stderr, "Giving up" + +if __name__ == '__main__': + example = PopiExample() + example.run() + diff --git a/examples/POPI/run_popi_experiments.sh b/examples/POPI/run_popi_experiments.sh new file mode 100755 index 00000000..92ec590c --- /dev/null +++ b/examples/POPI/run_popi_experiments.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# +# description: run nef demo script +# + +PYTHONPATH="../../../nepi/src:../../../nepi/test/lib:../../../netns/src:../../../python-passfd/src:../../../python-unshare/src" python run_popi_experiments.py $* + +