X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=TestPeers.py;h=67478e7d0cf03c9bebf30b19065a69301adaf87d;hb=4e032a0867c7b8d99d0af92fdc2175918338f875;hp=a3b2e8b89d424864138aefb6aa5e7eac1030009b;hpb=e12b56d5b3d0bf32ad1d930f36276184dea09d45;p=plcapi.git diff --git a/TestPeers.py b/TestPeers.py index a3b2e8b..67478e7 100755 --- a/TestPeers.py +++ b/TestPeers.py @@ -25,32 +25,43 @@ import getopt import sys import time -## we use indexes 1 and 2 +from PLC.Shell import Shell +import PLC.Methods + +# when running locally, we might wish to run only our local stuff +dummy_print_methods = [ 'RefreshPeer' ] +class DummyShell: + class Callable: + def __init__(self,method,index): + self.method=method + self.index=index + self.printed=False + def __call__ (self, *args, **kwds): + if not self.printed or self.method in dummy_print_methods: + print "Dummy method %s on remote peer %d skipped"%(self.method,self.index) + self.printed=True + return 0 + def __init__(self,index): + self.index=index + for method in PLC.Methods.methods: + # ignore path-defined methods for now + if "." not in method: + setattr(self,method,DummyShell.Callable(method,self.index)) + +#################### +#import xmlrpclib +import os + +## try to support reload try: - dir(plc) + globals()['plc'] except: plc=[None,None,None] -## the server objects try: - dir(s) + globals()['s'] except: s=[None,None,None] -## the authentication objects -## our user -try: - dir(a) -except: - a=[None,None,None] -## the builtin root user for bootstrapping -try: - dir(aa) -except: - aa=[None,None,None] - -#################### -import xmlrpclib -import os - + #################### # predefined stuff # number of 'system' persons @@ -59,64 +70,102 @@ system_persons = 4 # among that, 1 gets refreshed - other ones have conflicting names system_persons_cross = 1 -system_slices_ids = (1,2) +system_slices_ids = (1,) def system_slices (): return len(system_slices_ids) def total_slices (): return number_slices+system_slices() -# temporary - the myplc I use doesnt know about 'system' yet def system_slivers (): -# return len(system_slices_ids) - return 0 + return len(system_slices_ids) + +# too tedious to do the maths : how many slices attached to node 1 +expected_slivers=None def total_slivers (): - return number_slices+system_slivers() + global expected_slivers + if expected_slivers is None: + expected_slivers=0 + actual_nodes_per_slice = min (number_nodes,number_nodes_per_slice) + for ns in myrange(number_slices): + slice_range = [ map_on_node (n+ns) for n in range(actual_nodes_per_slice)] + if 1 in slice_range: + expected_slivers += 1 + return expected_slivers+system_slivers() #################### # set initial conditions -def define_test (keys,sites,persons,nodes,slices,fast_mode=None): - global number_keys, number_sites, number_persons, number_nodes, number_slices, fast_flag - number_keys=keys +# actual persons_per_slice is min(number_persons,number_persons_per_slice) +# actual nodes_per_slice is min(number_nodes,number_nodes_per_slice) +# this is to prevent quadractic test times on big tests +def define_test (sites,persons,nodes,slices, + keys_per_person,nodes_per_slice,persons_per_slice,fast_mode=None): + global number_sites, number_persons, number_nodes, number_slices + global number_keys_per_person, number_nodes_per_slice, number_persons_per_slice, fast_flag number_sites = sites number_persons=persons number_nodes=nodes number_slices=slices + number_keys_per_person=keys_per_person + number_nodes_per_slice=nodes_per_slice + number_persons_per_slice=persons_per_slice if fast_mode is not None: fast_flag=fast_mode -local_index=None +# when we run locally on a given peer +local_peer=None def show_test(): - print '%d keys, %d sites, %d persons, %d nodes & %d slices'%( - number_keys, number_sites,number_persons,number_nodes,number_slices), + print '%d sites, %d persons, %d nodes & %d slices'%( + number_sites,number_persons,number_nodes,number_slices) + print '%d keys/person, %d nodes/slice & %d persons/slice'%( + number_keys_per_person,number_nodes_per_slice,number_persons_per_slice) print 'fast_flag',fast_flag - if local_index is not None: - print 'Running locally on index %d'%local_index + if local_peer is not None: + print 'Running locally on index %d'%local_peer def mini(): - define_test(1,1,1,1,1,True) + define_test(1,1,1,1,1,1,1,True) def normal(): - define_test (keys=2,sites=4,persons=4,nodes=5,slices=4,fast_mode=False) + define_test (sites=4,persons=4,nodes=5,slices=4, + keys_per_person=2,nodes_per_slice=3,persons_per_slice=6,fast_mode=False) + +def apply_factor (factor): + global number_sites, number_persons, number_nodes, number_slices + [number_sites, number_persons, number_nodes, number_slices] = \ + [factor*x for x in [number_sites, number_persons, number_nodes, number_slices]] + # use only 1 key in this case big_factor=4 def big(): - global number_keys, number_sites, number_persons, number_nodes, number_slices - normal() - (number_sites,number_persons,number_nodes,number_slices) = [ - big_factor * x for x in (number_sites,number_persons,number_nodes,number_slices)] - number_keys=1 - -huge_factor=50 + global number_sites, number_persons, number_nodes, number_slices + number_sites=200 + number_persons=500 + number_nodes=350 + number_slices=500 + global nodes_per_slice + nodes_per_slice=3 + global number_keys_per_person + number_keys_per_person=1 + global number_persons_per_slice + number_persons_per_slice=3 + +#huge_factor=1000 def huge(): - global number_keys, number_sites, number_persons, number_nodes, number_slices - normal() - (number_sites,number_persons,number_nodes,number_slices) = [ - huge_factor * x for x in (number_sites,number_persons,number_nodes,number_slices)] - number_keys=1 - -# use fast by default in interactive mode + global number_sites, number_persons, number_nodes, number_slices + number_sites=1000 + number_persons=2000 + number_nodes=3000 + number_slices=2000 + global nodes_per_slice + nodes_per_slice=3 + global number_keys_per_person + number_keys_per_person=1 + global number_persons_per_slice + number_persons_per_slice=3 + +# use mini test by default in interactive mode mini() #normal() @@ -126,33 +175,37 @@ plain_numbers=['zero','one','two','three','four','five','six','seven','eight','n 'eleven','twelve','thirteen','fourteen','fifteen','sixteen','seventeen','eighteen','nineteen','twenty'] plain_digits=['a','b','c','d','e','f','g','h','i','j'] #################### -plc1={ 'plcname':'plc1 in federation', - 'hostname':'lurch.cs.princeton.edu', - 'url-format':'https://%s:443/PLCAPI/', - 'builtin-admin-id':'root@plc1.org', - 'builtin-admin-password':'root', - 'peer-admin-name':'peer1@planet-lab.org', - 'peer-admin-password':'peer', - 'node-format':'n1%02d.plc1.org', - 'plainname' : 'one', - 'site-format':'one%s', - 'person-format' : 'user1-%d@plc1.org', - 'key-format':'ssh-rsa 11key4plc11 user%d-key%d', - 'person-password' : 'password1', +plc[1]={ 'plcname':'Thierry plc1', + 'hostname':'planetlab-devbox.inria.fr', + 'url-format':'https://%s:443/PLCAPI/', + 'builtin-admin-id':'root@plc1.org', + 'builtin-admin-password':'root', + 'peer-admin-name':'peer1@planet-lab.org', + 'peer-admin-password':'peer', + 'node-format':'n1-%03d.plc1.org', + 'plainname' : 'one', + 'site-format':'one%s', + 'person-format' : 'user1-%d@plc1.org', + 'key-format':'ssh-rsa 11key4plc11 user%d-key%d', + 'person-password' : 'password1', + 'gpg-keyring':'gpg_plc2.pub', + 'api-cacert':'api_plc2.crt', } -plc2={ 'plcname':'plc2 in federation', - 'hostname':'planetlab-devbox.inria.fr', - 'url-format':'https://%s:443/PLCAPI/', - 'builtin-admin-id':'root@plc2.org', - 'builtin-admin-password':'root', - 'peer-admin-name':'peer2@planet-lab.org', - 'peer-admin-password':'peer', - 'node-format':'n2%02d.plc2.org', - 'plainname' : 'two', - 'site-format':'two%s', - 'person-format' : 'user2-%d@plc2.org', - 'key-format':'ssh-rsa 22key4plc22 user%d-key%d', - 'person-password' : 'password2', +plc[2]={ 'plcname':'Thierry plc2', + 'hostname':'lurch.cs.princeton.edu', + 'url-format':'https://%s:443/PLCAPI/', + 'builtin-admin-id':'root@plc2.org', + 'builtin-admin-password':'root', + 'peer-admin-name':'peer2@planet-lab.org', + 'peer-admin-password':'peer', + 'node-format':'n2-%03d.plc2.org', + 'plainname' : 'two', + 'site-format':'two%s', + 'person-format' : 'user2-%d@plc2.org', + 'key-format':'ssh-rsa 22key4plc22 user%d-key%d', + 'person-password' : 'password2', + 'gpg-keyring':'gpg_plc1.pub', + 'api-cacert':'api_plc1.crt', } #################### @@ -191,12 +244,6 @@ def key_name (i,n,k): def node_name (i,n): return plc[i]['node-format']%n -def map_on_site (n): - result=(n%number_sites) - if (result==0): - result=number_sites - return result - def slice_name (i,n): site_index=map_on_site(n) return "%s_slice%d"%(site_login_base(i,site_index),n) @@ -205,9 +252,24 @@ def sat_name (i): return 'sat_%d'%i # to have indexes start at 1 +def map_on (n,max): + result=(n%max) + if result==0: + result=max + return result + def myrange (n): return range (1,n+1,1) +def map_on_site (n): + return map_on (n,number_sites) + +def map_on_person (n): + return map_on (n,number_persons) + +def map_on_node (n): + return map_on (n,number_nodes) + def message (*args): print "====================", print args @@ -226,81 +288,99 @@ def timer_show (): last_time=now #################### -def test00_init (args=[1,2]): +errors=0 +def myassert (message,boolean): + if not boolean: + print 'ASSERTION FAILED',message + global errors + errors +=1 + +def epilogue (): + if errors != 0: + print 'TEST FAILED with %d errors'%errors + assert errors == 0 + +#################### +# init +def test00_init (args=[1,2],builtin_person=False): timer_start() - global plc,s,a,aa - ## have you loaded this file already (support for reload) - if plc[1]: - pass - else: - plc=[None,plc1,plc2] - for i in args: - url=plc[i]['url-format']%plc[i]['hostname'] - plc[i]['url']=url - s[i]=xmlrpclib.ServerProxy(url,allow_none=True) - print 'initializing s[%d]'%i,url - aa[i]={'Username':plc[i]['builtin-admin-id'], - 'AuthMethod':'password', - 'AuthString':plc[i]['builtin-admin-password'], - 'Role':'admin'} - print 'initialized aa[%d]'%i, aa[i] - a[i]={'Username':plc[i]['peer-admin-name'], - 'AuthMethod':'password', - 'AuthString':plc[i]['peer-admin-password'], - 'Role':'admin'} - print 'initialized a[%d]'%i, a[i] + for i in args: + url=plc[i]['url-format']%plc[i]['hostname'] + plc[i]['url']=url + if local_peer is None: + # the regular remote mode + print 'initializing s[%d]=>%s'%(i,url), + if builtin_person: + user=plc[i]['builtin-admin-id'] + password=plc[i]['builtin-admin-password'] + else: + user=plc[i]['peer-admin-name'] + password=plc[i]['peer-admin-password'] + s[i]=Shell(url=url, + user=user, + password=password) + print 'user=',user + elif local_peer == i: + # local mode - use Shell's Direct mode - use /etc/planetlab/plc_config + s[i]=Shell() + else: + # remote peer in local mode : use dummy shell instead + s[i]=DummyShell(i) + +# use new person's account def test00_print (args=[1,2]): for i in args: - print 's[%d]'%i,s[i] - print 'aa[%d]'%i, aa[i] - print 'a[%d]'%i, a[i] + print '==================== s[%d]'%i +# s[i].show_config() + print 'show_config obsoleted' + print '====================' def check_nodes (el,ef,args=[1,2]): for i in args: # use a single request and sort afterwards for efficiency # could have used GetNodes's scope as well - all_nodes = s[i].GetNodes(a[i]) + all_nodes = s[i].GetNodes() n = len ([ x for x in all_nodes if x['peer_id'] is None]) f = len ([ x for x in all_nodes if x['peer_id'] is not None]) print '%02d: Checking nodes: got %d local (e=%d) & %d foreign (e=%d)'%(i,n,el,f,ef) - assert n==el - assert f==ef + myassert ('local nodes',n==el) + myassert ('foreign nodes',f==ef) def check_keys (el,ef,args=[1,2]): for i in args: # use a single request and sort afterwards for efficiency # could have used GetKeys's scope as well - all_keys = s[i].GetKeys(a[i]) + all_keys = s[i].GetKeys() n = len ([ x for x in all_keys if x['peer_id'] is None]) f = len ([ x for x in all_keys if x['peer_id'] is not None]) print '%02d: Checking keys: got %d local (e=%d) & %d foreign (e=%d)'%(i,n,el,f,ef) - assert n==el - assert f==ef + myassert ('local keys',n==el) + myassert ('foreign_keys',f==ef) def check_persons (el,ef,args=[1,2]): for i in args: # use a single request and sort afterwards for efficiency # could have used GetPersons's scope as well - all_persons = s[i].GetPersons(a[i]) + all_persons = s[i].GetPersons() n = len ([ x for x in all_persons if x['peer_id'] is None]) f = len ([ x for x in all_persons if x['peer_id'] is not None]) print '%02d: Checking persons: got %d local (e=%d) & %d foreign (e=%d)'%(i,n,el,f,ef) - assert n==el - assert f==ef + myassert ('local persons',n==el) + myassert ('foreign persons',f==ef) # expected : local slices, foreign slices def check_slices (els,efs,args=[1,2]): for i in args: - ls=len(s[i].GetSlices(a[i],{'peer_id':None})) - fs=len(s[i].GetSlices(a[i],{'~peer_id':None})) + ls=len(s[i].GetSlices({'peer_id':None})) + fs=len(s[i].GetSlices({'~peer_id':None})) print '%02d: Checking slices: got %d local (e=%d) & %d foreign (e=%d)'%(i,ls,els,fs,efs) - assert els==ls - assert efs==fs + myassert ('local slices',els==ls) + myassert ('foreign slices',efs==fs) def show_nodes (i,node_ids): # same as above - all_nodes = s[i].GetNodes(a[i],node_ids) + all_nodes = s[i].GetNodes(node_ids) loc_nodes = filter (lambda n: n['peer_id'] is None, all_nodes) for_nodes = filter (lambda n: n['peer_id'] is not None, all_nodes) @@ -320,17 +400,17 @@ def check_slice_nodes_n (ns,expected_nodes, is_local_slice, args=[1,2]): peer=peer_index(i) if is_local_slice: sname=slice_name(i,ns) - slice=s[i].GetSlices(a[i],{'name':[sname],'peer_id':None})[0] + slice=s[i].GetSlices({'name':[sname],'peer_id':None})[0] message='local' else: sname=slice_name(peer,ns) - slice=s[i].GetSlices(a[i],{'name':[sname],'~peer_id':None})[0] + slice=s[i].GetSlices({'name':[sname],'~peer_id':None})[0] message='foreign' print '%02d: %s slice %s (e=%d) '%(i,message,sname,expected_nodes), slice_node_ids=slice['node_ids'] print 'on nodes ',slice_node_ids show_nodes (i,slice_node_ids) - assert len(slice_node_ids)>=expected_nodes + myassert ('slice nodes',len(slice_node_ids)>=expected_nodes) if len(slice_node_ids) != expected_nodes: print 'TEMPORARY' @@ -349,8 +429,8 @@ def check_conf_files (args=[1,2]): def check_conf_files_n (nn,args=[1,2]): for i in args: nodename=node_name(i,nn) - ndict= s[i].GetSlivers(a[i],[nodename])[0] - assert ndict['hostname'] == nodename + ndict= s[i].GetSlivers([nodename])[0] + myassert ('conf files',ndict['hostname'] == nodename) conf_files = ndict['conf_files'] print '%02d: %d conf_files in GetSlivers for node %s'%(i,len(conf_files),nodename) for conf_file in conf_files: @@ -373,15 +453,15 @@ def check_slivers_1 (esn,args=[1,2]): def check_slivers_n (nn,esn,args=[1,2]): for i in args: nodename=node_name(i,nn) - ndict= s[i].GetSlivers(a[i],[nodename])[0] - assert ndict['hostname'] == nodename + ndict= s[i].GetSlivers(nodename) + myassert ('slivers hostname',ndict['hostname'] == nodename) slivers = ndict['slivers'] print '%02d: %d slivers (exp. %d) in GetSlivers for node %s'\ %(i,len(slivers),esn,nodename) for sliver in slivers: print '>>slivername = ',sliver['name'] pretty_printer.pprint(sliver) - assert len(slivers) == esn + myassert ('slivers count',len(slivers) == esn) #################### @@ -390,69 +470,102 @@ def test00_admin_person (args=[1,2]): for i in args: email = plc[i]['peer-admin-name'] try: - p=s[i].GetPersons(a[i],[email])[0] + p=s[i].GetPersons([email])[0] plc[i]['peer-admin-id']=p['person_id'] except: - person_id=s[i].AddPerson(aa[i],{'first_name':'Local', - 'last_name':'PeerPoint', - 'role_ids':[10], - 'email':email, - 'password':plc[i]['peer-admin-password']}) - print '%02d:== created peer admin account %d, %s - %s'%(i, - person_id,plc[i]['peer-admin-name'], - plc[i]['peer-admin-password']) + person_id=s[i].AddPerson({'first_name':'Local', + 'last_name':'PeerPoint', + 'role_ids':[10], + 'email':email, + 'password':plc[i]['peer-admin-password']}) + if person_id: + print '%02d:== created peer admin account %d, %s - %s'%( + i, person_id,plc[i]['peer-admin-name'],plc[i]['peer-admin-password']) plc[i]['peer-admin-id']=person_id def test00_admin_enable (args=[1,2]): for i in args: - s[i].AdmSetPersonEnabled(aa[i],plc[i]['peer-admin-id'],True) - s[i].AddRoleToPerson(aa[i],'admin',plc[i]['peer-admin-id']) - print '%02d:== enabled+admin on account %d:%s'%(i,plc[i]['peer-admin-id'],plc[i]['peer-admin-name']) - -def test00_peer_person (args=[1,2]): - global plc - for i in args: - peer=peer_index(i) - email=plc[peer]['peer-admin-name'] - try: - p=s[i].GetPersons(a[i],[email])[0] - plc[i]['peer_person_id']=p['person_id'] - except: - person_id = s[i].AddPerson (a[i], {'first_name':'Peering(plain passwd)', 'last_name':plc_name(peer), 'role_ids':[3000], - 'email':email,'password':plc[peer]['peer-admin-password']}) - print '%02d:Created person %d as the peer person'%(i,person_id) - plc[i]['peer_person_id']=person_id + if s[i].AdmSetPersonEnabled(plc[i]['peer-admin-id'],True): + s[i].AddRoleToPerson('admin',plc[i]['peer-admin-id']) + print '%02d:== enabled+admin on account %d:%s'%(i,plc[i]['peer-admin-id'],plc[i]['peer-admin-name']) #################### +def locate_key (filename): + " tries to locate a key file, either in . or in /etc/planetlab" + try: + return file("./"+filename).read() + except: + try: + return file("/etc/planetlab/"+filename).read() + except: + raise Exception,"Could not locate key %s"%filename + + def test00_peer (args=[1,2]): global plc for i in args: peer=peer_index(i) peername = plc_name(peer) try: - p=s[i].GetPeers (a[i], [peername])[0] - plc[i]['peer_id']=p['peer_id'] + p=s[i].GetPeers ( [peername])[0] except: - peer_id=s[i].AddPeer (a[i], {'peername':peername,'peer_url':plc[peer]['url'],'person_id':plc[i]['peer_person_id']}) - # NOTE : need to manually reset the encrypted password through SQL at this point - print '%02d:Created peer %d'%(i,peer_id) - plc[i]['peer_id']=peer_id - print "PLEASE manually set password for person_id=%d in DB%d"%(plc[i]['peer_person_id'],i) - -def test00_peer_passwd (args=[1,2]): - for i in args: - # using an ad-hoc local command for now - never could get quotes to reach sql.... - print "Attempting to set passwd for person_id=%d in DB%d"%(plc[i]['peer_person_id'],i), - retcod=os.system("ssh root@%s new_plc_api/person-password.sh %d"%(plc[i]['hostname'],plc[i]['peer_person_id'])) - print '-> system returns',retcod + try: + keyringname=plc[i]['gpg-keyring'] + cacertname=plc[i]['api-cacert'] + print 'Trying to locate keys for peer on plc[%d]'%i, + print 'in %s and %s'%(keyringname,cacertname) + + keyring=locate_key(keyringname) + cacert=locate_key(cacertname) + peer_id=s[i].AddPeer ( {'peername':peername, + 'peer_url':plc[peer]['url'], + 'key':keyring, + 'cacert': cacert, + }) + print '%02d:Created peer %d'%(i,peer_id) + except Exception,e: + print 'Could not create peer,',e +# push various stuff across hosts through external ssh/scp +# this is broken, use peers-test.mk instead +#def test00_push_public_peer_material (args=[1,2]): +# for i in args: +# peer=peer_index(i) +# +# ### the gpg keyring +# # refresh +# local_keyring="/etc/planetlab/gpg_keyring.pub" +# command="ssh root@%s gpg --homedir=/etc/planetlab --export --armor > %s"\ +# %(plc[i]['hostname'],local_keyring) +# retcod=os.system(command) +# print '#',command,'->',retcod +# +# for i in args: +# peer=peer_index(i) +# # push +# src_url='root@%s:%s'%(plc[i]['hostname'],local_keyring) +# dst_url='root@%s:%s'%(plc[peer]['hostname'], plc[i]['gpg-keyring']) +# command = 'scp %s %s'%(src_url,dst_url) +# retcod=os.system(command) +# print '#',command,'->',retcod +# +# for i in args: +# peer=peer_index(i) +# # push cacert +# local_cacert='/etc/planetlab/api_ca_ssl.crt' +# src_url='root@%s:%s'%(plc[i]['hostname'],local_cacert) +# dst_url='root@%s:%s'%(plc[peer]['hostname'], plc[i]['api-cacert']) +# command = 'scp %s %s'%(src_url,dst_url) +# retcod=os.system(command) +# print '#',command,'->',retcod + # this one gets cached def get_peer_id (i): try: return plc[i]['peer_id'] except: peername = plc_name (peer_index(i)) - peer_id = s[i].GetPeers(a[i],[peername])[0]['peer_id'] + peer_id = s[i].GetPeers([peername])[0]['peer_id'] plc[i]['peer_id'] = peer_id return peer_id @@ -462,8 +575,21 @@ def test00_refresh (message,args=[1,2]): timer_show() for i in args: print '%02d:== Refreshing peer'%(i), - retcod=s[i].RefreshPeer(a[i],get_peer_id(i)) - print 'got ',retcod + retcod=s[i].RefreshPeer(get_peer_id(i)) + keys=retcod.keys() + keys.sort() + print "Result: {", + for key in keys: + if "time" not in key: + print key,retcod[key], + print "}" + print "+++ ellapsed: {", + timers=retcod + keys=timers.keys() + keys.sort() + for key in keys: + print key,timers[key], + print "}" timer_show() #################### @@ -479,24 +605,25 @@ def test01_site_n (ns,add_if_true,args=[1,2]): for i in args: login_base = site_login_base (i,ns) try: - site_id = s[i].GetSites(a[i],[login_base])[0]['site_id'] + site_id = s[i].GetSites([login_base])[0]['site_id'] if not add_if_true: - s[i].DeleteSite(a[i],site_id) - print "%02d:== deleted site_id %d"%(i,site_id) + if s[i].DeleteSite(site_id): + print "%02d:== deleted site_id %d"%(i,site_id) except: if add_if_true: sitename=site_name(i,ns) abbrev_name="abbr"+str(i) max_slices = number_slices - site_id=s[i].AddSite (a[i], {'name':plc_name(i), - 'abbreviated_name': abbrev_name, - 'login_base': login_base, - 'is_public': True, - 'url': 'http://%s.com/'%abbrev_name, - 'max_slices':max_slices}) + site_id=s[i].AddSite ( {'name':plc_name(i), + 'abbreviated_name': abbrev_name, + 'login_base': login_base, + 'is_public': True, + 'url': 'http://%s.com/'%abbrev_name, + 'max_slices':max_slices}) ### max_slices does not seem taken into account at that stage - s[i].UpdateSite(a[i],site_id,{'max_slices':max_slices}) - print '%02d:== Created site %d with max_slices=%d'%(i,site_id,max_slices) + if site_id: + s[i].UpdateSite(site_id,{'max_slices':max_slices}) + print '%02d:== Created site %d with max_slices=%d'%(i,site_id,max_slices) #################### def test02_person (args=[1,2]): @@ -508,43 +635,45 @@ def test02_del_person (args=[1,2]): test02_person_n (np,False,args) def test02_person_n (np,add_if_true,args=[1,2]): - test02_person_n_ks (np, myrange(number_keys),add_if_true,args) + test02_person_n_ks (np, myrange(number_keys_per_person),add_if_true,args) def test02_person_n_ks (np,nks,add_if_true,args=[1,2]): for i in args: email = person_name(i,np) try: - person_id=s[i].GetPersons(a[i],[email])[0]['person_id'] + person_id=s[i].GetPersons([email])[0]['person_id'] if not add_if_true: - s[i].DeletePerson(a[i],person_id) - print "%02d:== deleted person_id %d"%(i,person_id) + if s[i].DeletePerson(person_id): + print "%02d:== deleted person_id %d"%(i,person_id) except: if add_if_true: password = plc[i]['person-password'] - person_id=s[i].AddPerson(a[i],{'first_name':'Your average', + person_id=s[i].AddPerson({'first_name':'Your average', 'last_name':'User%d'%np, 'role_ids':[30], 'email':email, 'password': password }) - print '%02d:== created user account %d, %s - %s'%(i, person_id,email,password) - for nk in nks: - key=key_name(i,np,nk) - s[i].AddPersonKey(aa[i],email,{'key_type':'ssh', 'key':key}) - print '%02d:== added key %s to person %s'%(i,key,email) + if person_id: + print '%02d:== created user account %d, %s - %s'%(i, person_id,email,password) + for nk in nks: + key=key_name(i,np,nk) + s[i].AddPersonKey(email,{'key_type':'ssh', 'key':key}) + print '%02d:== added key %s to person %s'%(i,key,email) #################### # retrieves node_id from hostname - checks for local nodes only def get_local_node_id(i,nodename): - return s[i].GetNodes(a[i],{'hostname':nodename,'peer_id':None})[0]['node_id'] + return s[i].GetNodes({'hostname':nodename,'peer_id':None})[0]['node_id'] # clean all local nodes - foreign nodes are not supposed to be cleaned up manually def clean_all_nodes (args=[1,2]): for i in args: print '%02d:== Cleaning all nodes'%i - loc_nodes = s[i].GetNodes(a[i],{'peer_id':None}) - for node in loc_nodes: - print '%02d:==== Cleaning node %d'%(i,node['node_id']) - s[i].DeleteNode(a[i],node['node_id']) + local_nodes = s[i].GetNodes({'peer_id':None}) + if local_nodes: + for node in local_nodes: + print '%02d:==== Cleaning node %d'%(i,node['node_id']) + s[i].DeleteNode(node['node_id']) def test03_node (args=[1,2]): for nn in myrange(number_nodes): @@ -557,8 +686,9 @@ def test03_node_n (nn,args=[1,2]): get_local_node_id(i,nodename) except: login_base=site_login_base(i,map_on_site(nn)) - n=s[i].AddNode(a[i],login_base,{'hostname': nodename}) - print '%02d:== Added node %d %s'%(i,n,node_name(i,i)) + n=s[i].AddNode(login_base,{'hostname': nodename}) + if n: + print '%02d:== Added node %d %s'%(i,n,node_name(i,nn)) def test02_delnode (args=[1,2]): for nn in myrange(number_nodes): @@ -568,18 +698,19 @@ def test02_delnode_n (nn,args=[1,2]): for i in args: nodename = node_name(i,nn) node_id = get_local_node_id (i,nodename) - retcod=s[i].DeleteNode(a[i],nodename) - print '%02d:== Deleted node %d, returns %s'%(i,node_id,retcod) + retcod=s[i].DeleteNode(nodename) + if retcod: + print '%02d:== Deleted node %d, returns %s'%(i,node_id,retcod) #################### def clean_all_slices (args=[1,2]): for i in args: print '%02d:== Cleaning all slices'%i - for slice in s[i].GetSlices(a[i],{'peer_id':None}): + for slice in s[i].GetSlices({'peer_id':None}): slice_id = slice['slice_id'] if slice_id not in system_slices_ids: - print '%02d:==== Cleaning slice %d'%(i,slice_id) - s[i].DeleteSlice(a[i],slice_id) + if s[i].DeleteSlice(slice_id): + print '%02d:==== Cleaned slice %d'%(i,slice_id) def test04_slice (args=[1,2]): for n in myrange(number_slices): @@ -592,19 +723,22 @@ def test04_slice_n (ns,args=[1,2]): slicename=slice_name(i,ns) max_nodes=number_nodes try: - s[i].GetSlices(a[i],[slicename])[0] + s[i].GetSlices([slicename])[0] except: - slice_id=s[i].AddSlice (a[i],{'name':slicename, - 'description':'slice %s on %s'%(slicename,plcname), - 'url':'http://planet-lab.org/%s'%slicename, - 'max_nodes':max_nodes, - 'instanciation':'plc-instantiated', - }) - print '%02d:== created slice %d - max nodes=%d'%(i,slice_id,max_nodes) - for np in myrange(number_persons): - email = person_name (i,np) - retcod = s[i].AddPersonToSlice (a[i], email, slicename) - print '%02d:== Attached person %s to slice %s'%(i,email,slicename) + slice_id=s[i].AddSlice ({'name':slicename, + 'description':'slice %s on %s'%(slicename,plcname), + 'url':'http://planet-lab.org/%s'%slicename, + 'max_nodes':max_nodes, + 'instanciation':'plc-instantiated', + }) + if slice_id: + print '%02d:== created slice %d - max nodes=%d'%(i,slice_id,max_nodes) + actual_persons_per_slice = min (number_persons,number_persons_per_slice) + person_indexes=[map_on_person (p+ns) for p in range(actual_persons_per_slice)] + for np in person_indexes: + email = person_name (i,np) + retcod = s[i].AddPersonToSlice (email, slicename) + print '%02d:== Attached person %s to slice %s'%(i,email,slicename) def test04_node_slice (is_local, add_if_true, args=[1,2]): @@ -612,7 +746,9 @@ def test04_node_slice (is_local, add_if_true, args=[1,2]): test04_node_slice_ns (ns,is_local, add_if_true, args) def test04_node_slice_ns (ns,is_local, add_if_true, args=[1,2]): - test04_node_slice_nl_n (myrange(number_nodes),ns,is_local, add_if_true, args) + actual_nodes_per_slice = min (number_nodes,number_nodes_per_slice) + node_indexes = [ map_on_node (n+ns) for n in range(actual_nodes_per_slice)] + test04_node_slice_nl_n (node_indexes,ns,is_local, add_if_true, args) def test04_node_slice_nl_n (nnl,ns,is_local, add_if_true, args=[1,2]): for i in args: @@ -626,13 +762,14 @@ def test04_node_slice_nl_n (nnl,ns,is_local, add_if_true, args=[1,2]): hostnames=[node_name(peer,nn) for nn in nnl] nodetype='foreign' if add_if_true: - s[i].AddSliceToNodes (a[i], sname,hostnames) + res=s[i].AddSliceToNodes (sname,hostnames) message="added" else: - s[i].DeleteSliceFromNodes (a[i], sname,hostnames) + res=s[i].DeleteSliceFromNodes (sname,hostnames) message="deleted" - print '%02d:== %s in slice %s %s '%(i,message,sname,nodetype), - print hostnames + if res: + print '%02d:== %s in slice %s %s '%(i,message,sname,nodetype), + print hostnames def test04_slice_add_lnode (args=[1,2]): test04_node_slice (True,True,args) @@ -651,15 +788,15 @@ def test05_sat (args=[1,2]): for i in args: name = sat_name(i) try: - sat_id=s[i].GetSliceAttributeTypes (a[i],[name])[0] + sat_id=s[i].GetSliceAttributeTypes ([name])[0] except: description="custom sat on plc%d"%i min_role_id=10 - sat_id=s[i].AddSliceAttributeType (a[i], - { 'name':name, + sat_id=s[i].AddSliceAttributeType ({ 'name':name, 'description': description, 'min_role_id' : min_role_id}) - print '%02d:== created SliceAttributeType = %d'%(i,sat_id) + if sat_id: + print '%02d:== created SliceAttributeType = %d'%(i,sat_id) # for test, we create 4 slice_attributes # on slice1 - sat=custom_made (see above) - all nodes @@ -668,30 +805,29 @@ def test05_sat (args=[1,2]): # on slice1 - sat='net_max' - node=n1 def test05_sa_atom (slice_name,sat_name,value,node,i): - sa_id=s[i].GetSliceAttributes(a[i],{'name':sat_name, - 'value':value}) + sa_id=s[i].GetSliceAttributes({'name':sat_name, + 'value':value}) if not sa_id: if node: - sa_id=s[i].AddSliceAttribute(a[i], - slice_name, + sa_id=s[i].AddSliceAttribute(slice_name, sat_name, value, node) else: print 'slice_name',slice_name,'sat_name',sat_name - sa_id=s[i].AddSliceAttribute(a[i], - slice_name, + sa_id=s[i].AddSliceAttribute(slice_name, sat_name, value) - print '%02d:== created SliceAttribute = %d'%(i,sa_id), - print 'On slice',slice_name,'and node',node + if sa_id: + print '%02d:== created SliceAttribute = %d'%(i,sa_id), + print 'On slice',slice_name,'and node',node def test05_sa (args=[1,2]): for i in args: test05_sa_atom (slice_name(i,1),sat_name(i),'custom sat/all nodes',None,i) test05_sa_atom (slice_name(i,1),sat_name(i),'custom sat/node1',node_name(i,1),i) - test05_sa_atom (slice_name(i,1),'net_max','predefined sat/all nodes',None,i) - test05_sa_atom (slice_name(i,1),'net_max','predefined sat/node1',node_name(i,1),i) + test05_sa_atom (slice_name(i,1),'vref','predefined sat/all nodes',None,i) + test05_sa_atom (slice_name(i,1),'vref','predefined sat/node1',node_name(i,1),i) ############################## # readable dumps @@ -710,13 +846,14 @@ def p_node(n): def p_slice(s): print s['slice_id'],s['peer_id'],s['name'],'nodes=',s['node_ids'],'persons=',s['person_ids'] - print '---','sas=',s['slice_attribute_ids'],s['name'],'crp=',s['creator_person_id'],'e=',s['expires'] + print '---','sas=',s['slice_attribute_ids'],s['name'],'crp=',s['creator_person_id'] + print "--- 'expires':",s['expires'] def p_sat(sat): - print sat['attribute_type_id'],sat['peer_id'], sat['name'], sat['min_role_id'], sat['description'] + print sat['attribute_type_id'], sat['name'], sat['min_role_id'], sat['description'] def p_sa (sa): - print sa['slice_attribute_id'],sa['peer_id'],sa['name'],'AT_id:',sa['attribute_type_id'] + print sa['slice_attribute_id'],sa['name'],'AT_id:',sa['attribute_type_id'] print '---','v=',sa['value'],'sl=',sa['slice_id'],'n=',sa['node_id'] import pprint @@ -739,21 +876,26 @@ def dump (args=[1,2]): for i in args: print '%02d:============================== DUMPING'%i print '%02d: SITES'%i - [p_site(x) for x in s[i].GetSites(a[i])] + [p_site(x) for x in s[i].GetSites()] print '%02d: KEYS'%i - [p_key(x) for x in s[i].GetKeys(a[i])] + [p_key(x) for x in s[i].GetKeys()] print '%02d: PERSONS'%i - [p_person(x) for x in s[i].GetPersons(a[i])] + [p_person(x) for x in s[i].GetPersons()] print '%02d: NODES'%i - [p_node(x) for x in s[i].GetNodes(a[i])] + [p_node(x) for x in s[i].GetNodes()] print '%02d: SLICES'%i - [p_slice(x) for x in s[i].GetSlices(a[i])] + [p_slice(x) for x in s[i].GetSlices()] print '%02d: Slice Attribute Types'%i - [p_sat(x) for x in s[i].GetSliceAttributeTypes(a[i])] + [p_sat(x) for x in s[i].GetSliceAttributeTypes()] print '%02d: Slice Attributes'%i - [p_sa(x) for x in s[i].GetSliceAttributes(a[i])] - print '%02d: SLIVERS'%i - [p_sliver('%02d:'%i,x) for x in s[i].GetSlivers(a[i])] + [p_sa(x) for x in s[i].GetSliceAttributes()] + timer_show() + snodes=min(3,number_nodes) + print '%02d: SLIVERS for first %d nodes'%(i,snodes) + print 'WARNING - GetSlivers needs fix' +# for id in myrange(snodes): +# p_sliver('%02d:'%i,s[i].GetSlivers(id)) + print '%02d:============================== END DUMP'%i @@ -812,13 +954,15 @@ def all(): #################### def test_all_init (): message ("INIT") - test00_init () + test00_init (builtin_person=True) test00_print () test00_admin_person () test00_admin_enable () - test00_peer_person () + test00_init (builtin_person=False) +# required before we can add peers +# use make -f peers-test.mk peers instead +# test00_push_public_peer_material() test00_peer () - test00_peer_passwd () def test_all_sites (): test01_site () @@ -835,10 +979,10 @@ def test_all_persons (): message ("1 extra del/add cycle for unique indexes") test02_del_person([2]) test02_person([2]) - check_keys(number_persons*number_keys,0) + check_keys(number_persons*number_keys_per_person,0) check_persons(system_persons+number_persons,system_persons_cross) test00_refresh ('after persons&keys creation') - check_keys(number_persons*number_keys,number_persons*number_keys) + check_keys(number_persons*number_keys_per_person,number_persons*number_keys_per_person) check_persons(system_persons+number_persons,system_persons_cross+number_persons) def test_all_nodes (): @@ -884,12 +1028,12 @@ def test_all_addslices (): # create slices on plc1 message ("CREATING SLICES on plc1") test04_slice ([1]) - # each site has 3 local slices and 0 foreign slice - check_slices (total_slices(),0,[1]) - check_slices (system_slices(),0,[2]) + + check_slices (total_slices(),system_slices(),[1]) + check_slices (system_slices(),system_slices(),[2]) test00_refresh ("after slice created on plc1") - check_slices (total_slices(),0,[1]) - check_slices (system_slices(),number_slices,[2]) + check_slices (total_slices(),system_slices(),[1]) + check_slices (system_slices(),total_slices(),[2]) # no slice has any node yet check_local_slice_nodes(0,[1]) check_foreign_slice_nodes(0,[2]) @@ -898,26 +1042,26 @@ def test_all_addslices (): message ("ADDING LOCAL NODES IN SLICES") test04_slice_add_lnode ([1]) # of course the change is only local - check_local_slice_nodes (number_nodes,[1]) + check_local_slice_nodes (number_nodes_per_slice,[1]) check_foreign_slice_nodes(0,[2]) # refreshing test00_refresh ("After local nodes were added on plc1") - check_local_slice_nodes (number_nodes,[1]) - check_foreign_slice_nodes (number_nodes,[2]) + check_local_slice_nodes (number_nodes_per_slice,[1]) + check_foreign_slice_nodes (number_nodes_per_slice,[2]) # now we add foreign nodes into local slice message ("ADDING FOREIGN NODES IN SLICES") test04_slice_add_fnode ([1]) - check_local_slice_nodes (2*number_nodes,[1]) - check_foreign_slice_nodes (number_nodes,[2]) + check_local_slice_nodes (2*number_nodes_per_slice,[1]) + check_foreign_slice_nodes (number_nodes_per_slice,[2]) # refreshing test00_refresh ("After foreign nodes were added in plc1") # remember that foreign slices only know about LOCAL nodes # so this does not do anything - check_local_slice_nodes (2*number_nodes,[1]) - check_foreign_slice_nodes (2*number_nodes,[2]) + check_local_slice_nodes (2*number_nodes_per_slice,[1]) + check_foreign_slice_nodes (2*number_nodes_per_slice,[2]) check_slivers_1(total_slivers()) @@ -925,19 +1069,19 @@ def test_all_delslices (): message ("DELETING FOREIGN NODES FROM SLICES") test04_slice_del_fnode([1]) - check_local_slice_nodes (number_nodes,[1]) - check_foreign_slice_nodes (2*number_nodes,[2]) + check_local_slice_nodes (number_nodes_per_slice,[1]) + check_foreign_slice_nodes (2*number_nodes_per_slice,[2]) # mmh? check_slivers_1(total_slivers(),[1]) test00_refresh ("After foreign nodes were removed on plc1") - check_local_slice_nodes (number_nodes,[1]) - check_foreign_slice_nodes (number_nodes,[2]) + check_local_slice_nodes (number_nodes_per_slice,[1]) + check_foreign_slice_nodes (number_nodes_per_slice,[2]) message ("DELETING LOCAL NODES FROM SLICES") test04_slice_del_lnode([1]) check_local_slice_nodes (0,[1]) - check_foreign_slice_nodes (number_nodes,[2]) + check_foreign_slice_nodes (number_nodes_per_slice,[2]) test00_refresh ("After local nodes were removed on plc1") check_local_slice_nodes (0,[1]) @@ -945,10 +1089,10 @@ def test_all_delslices (): message ("CHECKING SLICES CLEAN UP") clean_all_slices([1]) - check_slices (system_slices(),0,[1]) - check_slices (system_slices(),number_slices,[2]) + check_slices (system_slices(),system_slices(),[1]) + check_slices (system_slices(),total_slices(),[2]) test00_refresh ("After slices clenaup") - check_slices(system_slices(),0) + check_slices(system_slices(),system_slices()) def test_all_slices (): test_all_addslices () @@ -976,6 +1120,7 @@ def test_all (): message("END") ### ad hoc test sequences +# we just create objects here so we can dump the DB def populate (): timer_start() test_all_init() @@ -994,18 +1139,31 @@ def populate (): timer_show() test05_sa([1]) timer_show() - test00_refresh ("populate: refreshing peer 1",[1]) + message("END") + +def populate_end(): + test00_init(builtin_person=False) + test00_refresh ("Peer 1 for publishing foreign nodes from 2",[1]) timer_show() test04_slice_add_fnode([1]) timer_show() test00_refresh("populate: refresh all") + timer_show() + test00_refresh("empty refresh") dump() timer_show() + message("END") # temporary - scratch as needed def test_now (): test_all_init() - test_all_sites () + +# populate() +# test00_refresh('peer 1 gets plc2 nodes',[1]) +# test04_slice_add_fnode([1]) +# test00_refresh('final',[1]) +# +# test_all_sites () # clean_all_nodes() # clean_all_slices() # populate() @@ -1015,16 +1173,18 @@ def usage (): print "Usage: %s [-n] [-f]"%sys.argv[0] print " -n runs test_now instead of test_all" print " -p runs populate instead of test_all" + print " -e runs populate_end of test_all" print " -m run in mini mode (1 instance of each class)" - print " -b performs big run (%d times as large as normal)"%big_factor - print " -H performs huge run (%d times as large as normal)"%huge_factor + print " -b performs big run" + print " -H performs huge run" + print " -f n : increases normal sizes by " print " -l n : tester runs locally for peer , rather than through xmlrpc" sys.exit(1) def main (): try: - (o,a) = getopt.getopt(sys.argv[1:], "mnpbHl:") + (o,a) = getopt.getopt(sys.argv[1:], "emnpbHf:l:") except: usage() func = test_all @@ -1035,19 +1195,22 @@ def main (): elif opt=='-p': print 'Running populate' func = populate + elif opt=='-e': + print 'Running populate_end' + func = populate_end elif opt=='-m': mini() elif opt=='-b': big() elif opt=='-H': huge() + elif opt=='-f': + factor=int(val) + apply_factor(factor) elif opt=='-l': - if val in (1,2): - local_index=val - print '-l option not implemented yet' - # need to figure a way to use Shell.py-like calling paradigm - sys.exit(1) - else: + global local_peer + local_peer=int(val) + if local_peer not in (1,2): usage() else: usage() @@ -1056,6 +1219,7 @@ def main (): show_test() func() timer_show() + epilogue if __name__ == '__main__': normal()