X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=Test.py;h=3e2e5aa20a0c89a0f94db61c2b098c0bb040c448;hb=refs%2Fheads%2Fplanetlab-4_0-branch;hp=1e35cb6b0dcd0653029f27a26b28fd6a52b029c6;hpb=794eb39a9cda777c3f5401666cb1eea52e863c5c;p=plcapi.git diff --git a/Test.py b/Test.py index 1e35cb6..3e2e5aa 100755 --- a/Test.py +++ b/Test.py @@ -1,487 +1,281 @@ #!/usr/bin/python # -# Test script example +# Test script for peer caching # # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: Test.py,v 1.16 2007/01/09 16:13:36 mlhuang Exp $ +# $Id: Test.py,v 1.23 2007/09/14 15:34:01 mef Exp $ # -from pprint import pprint -from string import letters, digits, punctuation -from traceback import print_exc -from optparse import OptionParser -import base64 -import os - -from random import Random -random = Random() +""" +Test script for peer caching. Intended for testing multiple PLCs +running on the same machine in different chroots. Here is how I set +things up after installing and configuring MyPLC: -from PLC.Shell import Shell -shell = Shell(globals()) +# Shut down MyPLC +service plc stop -def randfloat(min = 0.0, max = 1.0): - return float(min) + (random.random() * (float(max) - float(min))) +# Copy to /plc2 +cp -ra /plc /plc2 +ln -sf plc /etc/init.d/plc2 +echo 'PLC_ROOT=/plc2/root' > /etc/sysconfig/plc2 +echo 'PLC_DATA=/plc2/data' >> /etc/sysconfig/plc2 -def randint(min = 0, max = 1): - return int(randfloat(min, max + 1)) - -# See "2.2 Characters" in the XML specification: +# Edit /plc2/data/etc/planetlab/plc_config.xml and change at least the +# following so that they do not conflict with the defaults: # -# #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] -# avoiding -# [#x7F-#x84], [#x86-#x9F], [#xFDD0-#xFDDF] - -ascii_xml_chars = map(unichr, [0x9, 0xA, 0xD]) -ascii_xml_chars += map(unichr, xrange(0x20, 0x7F - 1)) -low_xml_chars = list(ascii_xml_chars) -low_xml_chars += map(unichr, xrange(0x84 + 1, 0x86 - 1)) -low_xml_chars += map(unichr, xrange(0x9F + 1, 0xFF)) -valid_xml_chars = list(low_xml_chars) -valid_xml_chars += map(unichr, xrange(0xFF + 1, 0xD7FF)) -valid_xml_chars += map(unichr, xrange(0xE000, 0xFDD0 - 1)) -valid_xml_chars += map(unichr, xrange(0xFDDF + 1, 0xFFFD)) - -def randstr(length, pool = valid_xml_chars, encoding = "utf-8"): - sample = random.sample(pool, min(length, len(pool))) - while True: - s = u''.join(sample) - bytes = len(s.encode(encoding)) - if bytes > length: - sample.pop() - elif bytes < length: - sample += random.sample(pool, min(length - bytes, len(pool))) - random.shuffle(sample) - else: - break - return s - -def randhostname(): - # 1. Each part begins and ends with a letter or number. - # 2. Each part except the last can contain letters, numbers, or hyphens. - # 3. Each part is between 1 and 64 characters, including the trailing dot. - # 4. At least two parts. - # 5. Last part can only contain between 2 and 6 letters. - hostname = 'a' + randstr(61, letters + digits + '-') + '1.' + \ - 'b' + randstr(61, letters + digits + '-') + '2.' + \ - 'c' + randstr(5, letters) - return hostname - -def randpath(length): - parts = [] - for i in range(randint(1, 10)): - parts.append(randstr(randint(1, 30), ascii_xml_chars)) - return os.sep.join(parts)[0:length] - -def randemail(): - return (randstr(100, letters + digits) + "@" + randhostname()).lower() - -def randkey(bits = 2048): - key_types = ["ssh-dss", "ssh-rsa"] - key_type = random.sample(key_types, 1)[0] - return ' '.join([key_type, - base64.b64encode(''.join(randstr(bits / 8).encode("utf-8"))), - randemail()]) - -def random_site(): - return { - 'name': randstr(254), - 'abbreviated_name': randstr(50), - 'login_base': randstr(20, letters).lower(), - 'latitude': int(randfloat(-90.0, 90.0) * 1000) / 1000.0, - 'longitude': int(randfloat(-180.0, 180.0) * 1000) / 1000.0, - } - -def random_address_type(): - return { - 'name': randstr(20), - 'description': randstr(254), - } - -def random_address(): - return { - 'line1': randstr(254), - 'line2': randstr(254), - 'line3': randstr(254), - 'city': randstr(254), - 'state': randstr(254), - 'postalcode': randstr(64), - 'country': randstr(128), - } - -def random_person(): - return { - 'first_name': randstr(128), - 'last_name': randstr(128), - 'email': randemail(), - 'bio': randstr(254), - # Accounts are disabled by default - 'enabled': False, - 'password': randstr(254), - } - -def random_key(): - return { - 'key_type': random.sample(key_types, 1)[0], - 'key': randkey() - } - -def random_slice(): - return { - 'name': site['login_base'] + "_" + randstr(11, letters).lower(), - 'url': "http://" + randhostname() + "/", - 'description': randstr(2048), - } - -class Test: - def __init__(self, check = True, verbose = True): - self.check = check - self.verbose = verbose - - self.site_ids = [] - self.address_type_ids = [] - self.address_ids = [] - self.person_ids = [] - - def run(self, - sites = 100, - address_types = 3, - addresses = 2, - persons = 1000, - keys = 3): - try: - try: - self.AddSites(sites) - self.AddAddressTypes(address_types) - self.AddAddresses(addresses) - self.AddPersons(persons) - except: - print_exc() - finally: - self.cleanup() - - def cleanup(self): - self.DeletePersons() - self.DeleteAddresses() - self.DeleteAddressTypes() - self.DeleteSites() - - def AddSites(self, n = 3): - """ - Add a number of random sites. - """ - - for i in range(n): - # Add site - site_fields = random_site() - site_id = AddSite(site_fields) - - # Should return a unique site_id - assert site_id not in self.site_ids - self.site_ids.append(site_id) - - if self.check: - # Check site - site = GetSites([site_id])[0] - for field in site_fields: - assert site[field] == site_fields[field] - - # Update site - site_fields = random_site() - # XXX Currently cannot change login_base - del site_fields['login_base'] - site_fields['max_slices'] = randint(1, 10) - UpdateSite(site_id, site_fields) - - if self.check: - # Check site again - site = GetSites([site_id])[0] - for field in site_fields: - assert site[field] == site_fields[field] - - if self.check: - sites = GetSites(self.site_ids) - assert set(self.site_ids) == set([site['site_id'] for site in sites]) - - if self.verbose: - print "Added sites", self.site_ids - - def DeleteSites(self): - """ - Delete any random sites we may have added. - """ - - for site_id in self.site_ids: - DeleteSite(site_id) - if self.check: - assert not GetSites([site_id]) - - if self.check: - assert not GetSites(self.site_ids) - - if self.verbose: - print "Deleted sites", self.site_ids - - self.site_ids = [] - - def AddAddressTypes(self, n = 3): - """ - Add a number of random address types. - """ - - for i in range(n): - address_type_fields = random_address_type() - address_type_id = AddAddressType(address_type_fields) - - # Should return a unique address_type_id - assert address_type_id not in self.address_type_ids - self.address_type_ids.append(address_type_id) - - if self.check: - # Check address type - address_type = GetAddressTypes([address_type_id])[0] - for field in 'name', 'description': - assert address_type[field] == address_type_fields[field] - - # Update address type - address_type_fields = random_address_type() - UpdateAddressType(address_type_id, address_type_fields) - - # Check address type again - address_type = GetAddressTypes([address_type_id])[0] - for field in 'name', 'description': - assert address_type[field] == address_type_fields[field] - - if self.check: - address_types = GetAddressTypes(self.address_type_ids) - assert set(self.address_type_ids) == set([address_type['address_type_id'] for address_type in address_types]) - - if self.verbose: - print "Added address types", self.address_type_ids - - def DeleteAddressTypes(self): - """ - Delete any random address types we may have added. - """ - - for address_type_id in self.address_type_ids: - DeleteAddressType(address_type_id) - if self.check: - assert not GetAddressTypes([address_type_id]) - - if self.check: - assert not GetAddressTypes(self.address_type_ids) - - if self.verbose: - print "Deleted address types", self.address_type_ids - - self.address_type_ids = [] - - def AddAddresses(self, n = 3): - """ - Add a number of random addresses to each site. - """ - - for site_id in self.site_ids: - for i in range(n): - address_fields = random_address() - address_id = AddSiteAddress(site_id, address_fields) - - # Should return a unique address_id - assert address_id not in self.address_ids - self.address_ids.append(address_id) - - if self.check: - # Check address - address = GetAddresses([address_id])[0] - for field in address_fields: - assert address[field] == address_fields[field] - - # Update address - address_fields = random_address() - UpdateAddress(address_id, address_fields) - - # Check address again - address = GetAddresses([address_id])[0] - for field in address_fields: - assert address[field] == address_fields[field] - - # Add address types - for address_type_id in self.address_type_ids: - AddAddressTypeToAddress(address_type_id, address_id) - - if self.check: - addresses = GetAddresses(self.address_ids) - assert set(self.address_ids) == set([address['address_id'] for address in addresses]) - for address in addresses: - assert set(self.address_type_ids) == set(address['address_type_ids']) - - if self.verbose: - print "Added addresses", self.address_ids - - def DeleteAddresses(self): - """ - Delete any random addresses we may have added. - """ - - # Delete site addresses - for address_id in self.address_ids: - # Remove address types - for address_type_id in self.address_type_ids: - DeleteAddressTypeFromAddress(address_type_id, address_id) - - if self.check: - address = GetAddresses([address_id])[0] - assert not address['address_type_ids'] - - DeleteAddress(address_id) - if self.check: - assert not GetAddresses([address_id]) - - if self.check: - assert not GetAddresses(self.address_ids) - - if self.verbose: - print "Deleted addresses", self.address_ids - - self.address_ids = [] - - def AddPersons(self, n = 3): - """ - Add a number of random users to each site. - """ - - roles = GetRoles() - role_ids = [role['role_id'] for role in roles] - roles = [role['name'] for role in roles] - roles = dict(zip(roles, role_ids)) - - for i in range(n): - - # Add account - person_fields = random_person() - person_id = AddPerson(person_fields) - - # Should return a unique person_id - assert person_id not in self.person_ids - self.person_ids.append(person_id) - - if self.check: - # Check account - person = GetPersons([person_id])[0] - for field in person_fields: - if field != 'password': - assert person[field] == person_fields[field] - - # Update account - person_fields = random_person() - UpdatePerson(person_id, person_fields) - - # Check account again - person = GetPersons([person_id])[0] - for field in person_fields: - if field != 'password': - assert person[field] == person_fields[field] - - auth = {'AuthMethod': "password", - 'Username': person_fields['email'], - 'AuthString': person_fields['password']} - - if self.check: - # Check that account is disabled - try: - assert not AuthCheck(auth) - except: - pass - - # Add random set of roles - person_roles = random.sample(['user', 'pi', 'tech'], randint(1, 3)) - for person_role in person_roles: - role_id = roles[person_role] - AddRoleToPerson(role_id, person_id) - - if self.check: - person = GetPersons([person_id])[0] - assert set(person_roles) == set(person['roles']) - - # Enable account - UpdatePerson(person_id, {'enabled': True}) - - if self.check: - # Check that account is enabled - assert AuthCheck(auth) - - # Associate account with random set of sites - person_site_ids = [] - for site_id in random.sample(self.site_ids, randint(1, len(self.site_ids))): - AddPersonToSite(person_id, site_id) - person_site_ids.append(site_id) - - if self.check: - # Make sure it really did it - person = GetPersons([person_id])[0] - assert set(person_site_ids) == set(person['site_ids']) - - # Set a primary site - primary_site_id = random.sample(person_site_ids, randint(1, len(person_site_ids)))[0] - SetPersonPrimarySite(person_id, primary_site_id) - - if self.check: - person = GetPersons([person_id])[0] - assert person['site_ids'][0] == primary_site_id - - if self.verbose: - print "Added users", self.person_ids +# PLC_NAME (e.g., PlanetLab Two) +# PLC_SLICE_PREFIX (e.g., two) +# PLC_ROOT_USER (e.g., root@planetlab.two) +# PLC_API_MAINTENANCE_USER (e.g., maint@planetlab.two) +# PLC_DB_PORT (e.g., 5433) +# PLC_WWW_PORT (e.g., 81) +# PLC_WWW_SSL_PORT (e.g., 444) +# PLC_API_PORT (must be the same as PLC_WWW_SSL_PORT, e.g., 444) +# PLC_BOOT_SSL_PORT (must be the same as PLC_WWW_SSL_PORT, e.g., 444) +# PLC_BOOT_PORT (may be the same as PLC_WWW_PORT, e.g., 81) + +# Start up both MyPLC instances +service plc start +service plc2 start + +# Run test +./Test.py -f /etc/planetlab/plc_config -f /plc2/data/etc/planetlab/plc_config + +# If the test fails and your databases are corrupt and/or you want to +# start over, you can always just blow the databases away. +service plc stop +rm -rf /plc/data/var/lib/pgsql/data +service plc start + +service plc2 stop +rm -rf /plc2/data/var/lib/pgsql/data +service plc2 start +""" + +import re +from optparse import OptionParser - def DeletePersons(self): - # Delete users - for person_id in self.person_ids: - # Remove from each site - for site_id in self.site_ids: - DeletePersonFromSite(person_id, site_id) +from PLC.Config import Config +from PLC.GPG import gpg_export +from PLC.Shell import Shell +from PLC.Test import Test + +def todict(list_of_dicts, key): + """ + Turn a list of dicts into a dict keyed on key. + """ + + return dict([(d[key], d) for d in list_of_dicts]) + +def RefreshPeers(plcs): + """ + Refresh each peer with each other. + """ + + for plc in plcs: + for peer in plcs: + if peer == plc: + continue + + print plc.config.PLC_NAME, "refreshing", peer.config.PLC_NAME + plc.RefreshPeer(peer.config.PLC_NAME) + + peer_id = plc.GetPeers([peer.config.PLC_NAME])[0]['peer_id'] + + peer_sites = todict(plc.GetSites({'peer_id': peer_id}), 'site_id') + sites_at_peer = todict(peer.GetSites(), 'site_id') + + peer_keys = todict(plc.GetKeys({'peer_id': peer_id}), 'key_id') + keys_at_peer = todict(peer.GetKeys(), 'key_id') + + peer_persons = todict(plc.GetPersons({'peer_id': peer_id}), 'person_id') + persons_at_peer = todict(peer.GetPersons(), 'person_id') + + peer_nodes = todict(plc.GetNodes({'peer_id': peer_id}), 'node_id') + nodes_at_peer = todict(peer.GetNodes(), 'node_id') + + our_nodes = todict(plc.GetNodes({'peer_id': None}), 'node_id') + our_peer_id_at_peer = peer.GetPeers([plc.config.PLC_NAME])[0]['peer_id'] + our_nodes_at_peer = todict(peer.GetNodes({'peer_id': our_peer_id_at_peer, + 'peer_node_id': our_nodes.keys()}), 'peer_node_id') + + peer_slices = todict(plc.GetSlices({'peer_id': peer_id}), 'peer_slice_id') + slices_at_peer = todict(peer.GetSlices(), 'slice_id') + + for site_id, site in peer_sites.iteritems(): + # Verify that this site exists at the peer + peer_site_id = site['peer_site_id'] + assert peer_site_id in sites_at_peer + peer_site = sites_at_peer[peer_site_id] + + # And is the same + for field in ['name', 'abbreviated_name', 'login_base', 'is_public', + 'latitude', 'longitude', 'url', + 'max_slices', 'max_slivers',]: + assert site[field] == peer_site[field] + + for key_id, key in peer_keys.iteritems(): + # Verify that this key exists at the peer + peer_key_id = key['peer_key_id'] + assert peer_key_id in keys_at_peer + peer_key = keys_at_peer[peer_key_id] + + # And is the same + for field in ['key_type', 'key']: + assert key[field] == peer_key[field] + + for person_id, person in peer_persons.iteritems(): + # Verify that this user exists at the peer + peer_person_id = person['peer_person_id'] + assert peer_person_id in persons_at_peer + peer_person = persons_at_peer[peer_person_id] + + # And is the same + for field in ['first_name', 'last_name', 'title', 'email', 'phone', + 'url', 'bio', 'enabled']: + assert person[field] == peer_person[field] + + for key_id in person['key_ids']: + # Verify that the user is not associated with any local keys + assert key_id in peer_keys + key = peer_keys[key_id] + peer_key_id = key['peer_key_id'] + + # Verify that this key exists at the peer + assert peer_key_id in keys_at_peer + peer_key = keys_at_peer[peer_key_id] + + # And is related to the same user at the peer + assert peer_key['key_id'] in peer_person['key_ids'] + + for node_id, node in peer_nodes.iteritems(): + # Verify that this node exists at the peer + peer_node_id = node['peer_node_id'] + assert peer_node_id in nodes_at_peer + peer_node = nodes_at_peer[peer_node_id] + + # And is the same + for field in ['boot_state', 'ssh_rsa_key', 'hostname', + 'version', 'model']: + assert node[field] == peer_node[field] + + # Verify that the node is not associated with any local sites + assert node['site_id'] in peer_sites + site = peer_sites[node['site_id']] + + # Verify that this site exists at the peer + peer_site_id = site['peer_site_id'] + assert peer_site_id in sites_at_peer + peer_site = sites_at_peer[peer_site_id] + + # And is related to the same node at the peer + assert peer_site['site_id'] == peer_node['site_id'] + + for slice_id, slice in peer_slices.iteritems(): + # Verify that this slice exists at the peer + peer_slice_id = slice['peer_slice_id'] + assert peer_slice_id in slices_at_peer + peer_slice = slices_at_peer[peer_slice_id] + + # And is the same + for field in ['name', 'instantiation', 'url', 'description', + 'max_nodes', 'expires']: + assert slice[field] == peer_slice[field] + + for node_id in slice['node_ids']: + # Verify that the slice is associated only with + # the peer's own nodes, or with our nodes as + # last cached by the peer. + assert node_id in peer_nodes or node_id in our_nodes_at_peer + if node_id in peer_nodes: + node = peer_nodes[node_id] + peer_node_id = node['peer_node_id'] + elif node_id in our_nodes_at_peer: + peer_node = our_nodes_at_peer[node_id] + peer_node_id = peer_node['node_id'] + + # Verify that this node exists at the peer + assert peer_node_id in nodes_at_peer + + # And is related to the same slice at the peer + assert peer_node_id in peer_slice['node_ids'] + +def TestPeers(plcs, check = True, verbose = True, tiny = False): + # Register each peer with each other + for plc in plcs: + for peer in plcs: + if peer == plc: + continue + + key = gpg_export(peer.chroot + peer.config.PLC_ROOT_GPG_KEY_PUB) + cacert = file(peer.chroot + peer.config.PLC_API_CA_SSL_CRT).read() + + if plc.GetPeers([peer.config.PLC_NAME]): + print plc.config.PLC_NAME, "updating peer", peer.config.PLC_NAME + plc.UpdatePeer(peer.config.PLC_NAME, + {'peer_url': peer.url, 'key': key, 'cacert': cacert}) + else: + print plc.config.PLC_NAME, "adding peer", peer.config.PLC_NAME + plc.AddPeer({'peername': peer.config.PLC_NAME, + 'peer_url': peer.url, 'key': key, 'cacert': cacert}) + + # Populate the DB + plc.test = Test(api = plc, check = check, verbose = verbose) + + if tiny: + params = Test.tiny + else: + params = Test.default - if self.check: - person = GetPersons([person_id])[0] - assert not person['site_ids'] + print "Populating", plc.config.PLC_NAME + plc.test.Add(**params) - # Revoke roles - person = GetPersons([person_id])[0] - for role_id in person['role_ids']: - DeleteRoleFromPerson(role_id, person_id) - - if self.check: - person = GetPersons([person_id])[0] - assert not person['role_ids'] - - # Disable account - UpdatePerson(person_id, {'enabled': False}) + # Refresh each other + RefreshPeers(plcs) - if self.check: - person = GetPersons([person_id])[0] - assert not person['enabled'] - - # Delete account - DeletePerson(person_id) - - if self.check: - assert not GetPersons([person_id]) - - if self.check: - assert not GetPersons(self.person_ids) - - if self.verbose: - print "Deleted users", self.person_ids + # Change some things + for plc in plcs: + print "Updating", plc.config.PLC_NAME + plc.test.Update() - self.person_ids = [] + # Refresh each other again + RefreshPeers(plcs) -if __name__ == "__main__": +def main(): parser = OptionParser() + parser.add_option("-f", "--config", dest = "configs", action = "append", default = [], help = "Configuration file (default: %default)") parser.add_option("-c", "--check", action = "store_true", default = False, help = "Verify actions (default: %default)") parser.add_option("-q", "--quiet", action = "store_true", default = False, help = "Be quiet (default: %default)") - parser.add_option("-p", "--populate", action = "store_true", default = False, help = "Do not cleanup (default: %default)") + parser.add_option("-t", "--tiny", action = "store_true", default = False, help = "Run a tiny test (default: %default)") (options, args) = parser.parse_args() - test = Test(check = options.check, verbose = not options.quiet) - test.run() - if not options.populate: - test.cleanup() + + # Test single peer by default + if not options.configs: + options.configs = ["/etc/planetlab/plc_config"] + + plcs = [] + for path in options.configs: + # Load configuration file + config = Config(path) + + # Determine path to chroot + m = re.match(r'(.*)/etc/planetlab', path) + if m is not None: + chroot = m.group(1) + else: + chroot = "" + + # Fix up path to SSL certificate + cacert = chroot + config.PLC_API_CA_SSL_CRT + + # Always connect with XML-RPC + plc = Shell(config = path, cacert = cacert, xmlrpc = True) + plc.chroot = chroot + plcs.append(plc) + + TestPeers(plcs, check = options.check, verbose = not options.quiet, tiny = options.tiny) + +if __name__ == "__main__": + main()