2to3'ed plcsh-stress-test, which is required as we run on top of plcsh
[tests.git] / system / plcsh_stress_test.py
index d3c0f69..efbce06 100755 (executable)
@@ -1,11 +1,20 @@
 #!/usr/bin/env plcsh
 #
+# WARNING: as opposed to the rest of the python code in this repo
+# the current script runs on top of plcsh and so it is for now
+# pinned as python2 code
+#
+#
 # Test script utility class
 #
 # Mark Huang <mlhuang@cs.princeton.edu>
 # Copyright (C) 2006 The Trustees of Princeton University
 #
-# $Id$
+
+# NOTE on porting to python3
+#
+# this file gets fed to plcsh on the tested myplc, so
+# it needs to remain python2 for now
 #
 
 from pprint import pprint
@@ -16,7 +25,7 @@ import socket
 import base64
 import struct
 import os
-import xmlrpclib
+import xmlrpc.client
 
 from PLC.Shell import Shell
 
@@ -41,25 +50,25 @@ def randint(min = 0, max = 1):
 # [#x7F-#x84], [#x86-#x9F], [#xFDD0-#xFDDF]
 #
 
-ascii_xml_chars = map(unichr, [0x9, 0xA])
+ascii_xml_chars = list(map(chr, [0x9, 0xA]))
 # xmlrpclib uses xml.parsers.expat, which always converts either '\r'
 # (#xD) or '\n' (#xA) to '\n'. So avoid using '\r', too, if this is
 # still the case.
-if xmlrpclib.loads(xmlrpclib.dumps(('\r',)))[0][0] == '\r':
+if xmlrpc.client.loads(xmlrpc.client.dumps(('\r',)))[0][0] == '\r':
     ascii_xml_chars.append('\r')
-ascii_xml_chars += map(unichr, xrange(0x20, 0x7F - 1))
+ascii_xml_chars += list(map(chr, range(0x20, 0x7F - 1)))
 low_xml_chars = list(ascii_xml_chars)
-low_xml_chars += map(unichr, xrange(0x84 + 1, 0x86 - 1))
-low_xml_chars += map(unichr, xrange(0x9F + 1, 0xFF))
+low_xml_chars += list(map(chr, range(0x84 + 1, 0x86 - 1)))
+low_xml_chars += list(map(chr, range(0x9F + 1, 0xFF)))
 valid_xml_chars = list(low_xml_chars)
-valid_xml_chars += map(unichr, xrange(0xFF + 1, 0xD7FF))
-valid_xml_chars += map(unichr, xrange(0xE000, 0xFDD0 - 1))
-valid_xml_chars += map(unichr, xrange(0xFDDF + 1, 0xFFFD))
+valid_xml_chars += list(map(chr, range(0xFF + 1, 0xD7FF)))
+valid_xml_chars += list(map(chr, range(0xE000, 0xFDD0 - 1)))
+valid_xml_chars += list(map(chr, range(0xFDDF + 1, 0xFFFD)))
 
 def randstr(length, pool = valid_xml_chars, encoding = "utf-8"):
     sample = random.sample(pool, min(length, len(pool)))
     while True:
-        s = u''.join(sample)
+        s = ''.join(sample)
         bytes = len(s.encode(encoding))
         if bytes > length:
             sample.pop()
@@ -79,13 +88,13 @@ def randhostname(namelengths):
     hostname = 'a' + randstr(namelengths['hostname1'], letters + digits + '-') + '1.' + \
                'b' + randstr(namelengths['hostname1'], letters + digits + '-') + '2.' + \
                'c' + randstr(namelengths['hostname2'], letters)
-    return hostname
+    return hostname.lower()
 
 def randpath(length):
     parts = []
     for i in range(randint(1, 10)):
         parts.append(randstr(randint(1, 30), ascii_xml_chars))
-    return u'/'.join(parts)[0:length]
+    return '/'.join(parts)[0:length]
 
 def randemail(namelengths):
     return (randstr(namelengths['email'], letters + digits) + "@" + randhostname(namelengths)).lower()
@@ -97,16 +106,21 @@ def randkey(namelengths,bits = 2048):
                      base64.b64encode(''.join(randstr(bits / 8).encode("utf-8"))),
                      randemail(namelengths)])
 
+def random_peer():
+    return {
+        'peername': randstr(24,letters + ' ' + digits),
+        'peer_url': "https://" + randhostname ({'hostname1':8,'hostname2':3}) + ":443/PLCAPI/",
+        'key' : randstr(1024,letters+digits),
+        'cacert' : randstr(1024,letters+digits),
+        'shortname' : randstr(1,letters) + 'LAB',
+        'hrn_root' : 'planetlab.' + randstr (3,letters),
+        }
+
 def random_site(namelengths):
-    try:
-        sitename=randstr(namelengths['sitename'],namelengths['sitename_contents'])
-    except:
-        sitename=randstr(namelengths['sitename'])
-    try:
-        abbreviated_name=randstr(namelengths['abbreviated_name'],namelengths['abbreviated_name_contents'])
-    except:
-        abbreviated_name=randstr(namelengths['abbreviated_name'])
+    sitename=randstr(namelengths['sitename'],namelengths['sitename_contents'])
+    abbreviated_name=randstr(namelengths['abbreviated_name'],namelengths['abbreviated_name_contents'])
 
+    print('nl[a] in random_site',namelengths['abbreviated_name'],'actual',len(abbreviated_name))
     return {
         'name': sitename,
         'abbreviated_name': abbreviated_name,
@@ -114,7 +128,7 @@ def random_site(namelengths):
         'latitude': int(randfloat(-90.0, 90.0) * 1000) / 1000.0,
         'longitude': int(randfloat(-180.0, 180.0) * 1000) / 1000.0,
         }
-            
+
 def random_address_type():
     return {
         'name': randstr(20),
@@ -150,14 +164,17 @@ def random_key(key_types,namelengths):
         }
 
 def random_tag_type (role_ids):
-    return  {'tagname': randstr(12),
-             'category':randstr(8),
-             'min_role_id': random.sample(role_ids, 1)[0],
-             'description' : randstr(128),
+    return  {'tagname': randstr(12,letters+digits),
+             'category':randstr(4,letters+digits)+'/'+randstr(6,letters+digits),
+             'description' : randstr(128,letters+digits+whitespace+punctuation),
              }
 
 def random_nodegroup():
-    return {'groupname' : randstr(50) }
+    return {'groupname' : randstr(30, letters+digits+whitespace) }
+
+def random_roles(role_ids):
+    nb_roles=len(role_ids)
+    return random.sample(role_ids,random.choice(list(range(1,nb_roles+1))))
 
 tag_fields=['arch']
 def random_node(node_types,boot_states,namelengths):
@@ -171,7 +188,7 @@ def random_node(node_types,boot_states,namelengths):
         'arch':randstr(10),
         }
 
-def random_interface(method, type):
+def random_interface(method, type,namelengths):
     interface_fields = {
         'method': method,
         'type': type,
@@ -188,6 +205,8 @@ def random_interface(method, type):
 
         for field in 'ip', 'netmask', 'network', 'broadcast', 'gateway', 'dns1':
             interface_fields[field] = socket.inet_ntoa(struct.pack('>L', locals()[field]))
+        if randint(0,1):
+            interface_fields['hostname']=randhostname(namelengths);
 
     return interface_fields
 
@@ -262,12 +281,48 @@ class Test:
         'attributes_per_slice': 2,
         }
 
+    sizes_large = {
+        'sites': 200,
+        'address_types': 4,
+        'addresses_per_site': 2,
+        'persons_per_site': 5,
+        'keys_per_person': 2,
+        'slice_tags': 4,
+        'nodegroups': 20,
+        'nodes_per_site': 2,
+        'interfaces_per_node': 2,
+        'ilinks':100,
+        'pcus_per_site': 1,
+        'conf_files': 50,
+        'slices_per_site': 10,
+        'attributes_per_slice': 4,
+        }
+
+    sizes_xlarge = {
+        'sites': 1000,
+        'address_types': 4,
+        'addresses_per_site': 2,
+        'persons_per_site': 5,
+        'keys_per_person': 2,
+        'slice_tags': 4,
+        'nodegroups': 20,
+        'nodes_per_site': 2,
+        'interfaces_per_node': 2,
+        'ilinks':100,
+        'pcus_per_site': 1,
+        'conf_files': 50,
+        'slices_per_site': 10,
+        'attributes_per_slice': 4,
+        }
+
     namelengths_default = {
         'hostname1': 61,
         'hostname2':5,
         'login_base':20,
         'sitename':254,
+        'sitename_contents':letters+digits,
         'abbreviated_name':50,
+        'abbreviated_name_contents':letters+digits+whitespace+punctuation,
         'model':255,
         'first_name':128,
         'last_name':128,
@@ -275,11 +330,11 @@ class Test:
         }
 
     namelengths_short = {
-        'hostname1': 12,
+        'hostname1': 8,
         'hostname2':3,
         'login_base':8,
         'sitename':64,
-        'sitename_contents':letters+digits+whitespace+punctuation,
+        'sitename_contents':letters+digits,
         'abbreviated_name':24,
         'abbreviated_name_contents':letters+digits+whitespace+punctuation,
         'model':40,
@@ -288,12 +343,13 @@ class Test:
         'email':24,
         }
 
-    def __init__(self, api, check = True, verbose = True, preserve = False):
+    def __init__(self, api, check, verbose, preserve, federating):
         self.api = api
         self.check = check
         self.verbose = verbose
         self.preserve = preserve
-        
+        self.federating = federating
+
         self.site_ids = []
         self.address_type_ids = []
         self.address_ids = []
@@ -312,7 +368,7 @@ class Test:
         self.slice_tag_ids = []
 
     def Cardinals (self):
-        return [len(x) for x in ( 
+        return [len(x) for x in (
                 self.api.GetNodes({},['node_id']),
                 self.api.GetSites({},['site_id']),
                 self.api.GetPersons({},['person_id']),
@@ -332,20 +388,22 @@ class Test:
         """
 
         cardinals_before=self.Cardinals()
-        print 'Cardinals before test (n,s,p,sl)',cardinals_before
+        print('Cardinals before test (n,s,p,sl)',cardinals_before)
 
         self.Add(**kwds)
-        self.Update()
-        if self.preserve:
-            print 'Preserving - delete skipped'
+        # if federating : we're done
+
+        if self.federating or self.preserve:
+            print('Preserving - update & delete skipped')
         else:
+            self.Update()
             self.Delete()
 
             cardinals_after=self.Cardinals()
-            print 'Cardinals after test (n,s,p,sl)',cardinals_after
+            print('Cardinals after test (n,s,p,sl)',cardinals_after)
 
             if cardinals_before != cardinals_after:
-                raise Exception, 'cardinals before and after differ - check deletion mechanisms'
+                raise Exception('cardinals before and after differ - check deletion mechanisms')
 
     def Add(self, **kwds):
         """
@@ -357,20 +415,32 @@ class Test:
         sizes = self.sizes_default.copy()
         sizes.update(kwds)
 
-        self.AddSites(sizes['sites'])
-        self.AddAddressTypes(sizes['address_types'])
-        self.AddAddresses(sizes['addresses_per_site'])
-        self.AddPersons(sizes['persons_per_site'])
-        self.AddKeys(sizes['keys_per_person'])
-        self.AddTagTypes(sizes['slice_tags'],sizes['nodegroups'],sizes['ilinks'])
-        self.AddNodeGroups(sizes['nodegroups'])
-        self.AddNodes(sizes['nodes_per_site'])
-        self.AddInterfaces(sizes['interfaces_per_node'])
-        self.AddIlinks (sizes['ilinks'])
-        self.AddPCUs(sizes['pcus_per_site'])
-        self.AddConfFiles(sizes['conf_files'])
-        self.AddSlices(sizes['slices_per_site'])
-        self.AddSliceTags(sizes['attributes_per_slice'])
+        if not self.federating:
+            self.AddSites(sizes['sites'])
+            self.AddAddressTypes(sizes['address_types'])
+            self.AddAddresses(sizes['addresses_per_site'])
+            self.AddPersons(sizes['persons_per_site'])
+            self.AddKeys(sizes['keys_per_person'])
+            self.AddTagTypes(sizes['slice_tags'],sizes['nodegroups'],sizes['ilinks'])
+            self.AddNodeGroups(sizes['nodegroups'])
+            self.AddNodes(sizes['nodes_per_site'])
+            self.AddInterfaces(sizes['interfaces_per_node'])
+            self.AddIlinks (sizes['ilinks'])
+            self.AddPCUs(sizes['pcus_per_site'])
+            self.AddConfFiles(sizes['conf_files'])
+            self.AddSlices(sizes['slices_per_site'])
+            self.AddSliceTags(sizes['attributes_per_slice'])
+
+        else:
+            self.RecordStatus()
+            self.AddSites(sizes['sites'])
+            self.AddPersons(sizes['persons_per_site'])
+            self.AddKeys(sizes['keys_per_person'])
+            self.AddNodes(sizes['nodes_per_site'])
+            self.AddSlices(sizes['slices_per_site'])
+            # create peer and add newly created entities
+            self.AddPeer()
+
 
     def Update(self):
         self.UpdateSites()
@@ -404,6 +474,33 @@ class Test:
         self.DeleteAddressTypes()
         self.DeleteSites()
 
+    # record current (old) objects
+    def RecordStatus (self):
+        self.old_site_ids = [ s['site_id'] for s in self.api.GetSites({},['site_id']) ]
+        self.old_person_ids = [ s['person_id'] for s in self.api.GetPersons({},['person_id']) ]
+        self.old_key_ids = [ s['key_id'] for s in self.api.GetKeys({},['key_id']) ]
+        self.old_node_ids = [ s['node_id'] for s in self.api.GetNodes({},['node_id']) ]
+        self.old_slice_ids = [ s['slice_id'] for s in self.api.GetSlices({},['slice_id']) ]
+
+    def AddPeer (self):
+        peer_id=self.api.AddPeer (random_peer())
+        peer = GetPeers([peer_id])[0]
+        if self.verbose:
+            print("Added peer",peer_id)
+
+        # add new sites (the ones not in self.site_ids) in the peer
+        # cheating a bit
+        for site in self.api.GetSites ({'~site_id':self.old_site_ids}):
+            peer.add_site(site,site['site_id'])
+        for person in self.api.GetPersons ({'~person_id':self.old_person_ids}):
+            peer.add_person(person,person['person_id'])
+        for key in self.api.GetKeys ({'~key_id':self.old_key_ids}):
+            peer.add_key(key,key['key_id'])
+        for node in self.api.GetNodes ({'~node_id':self.old_node_ids}):
+            peer.add_node(node,node['node_id'])
+        for slice in self.api.GetSlices ({'~slice_id':self.old_slice_ids}):
+            peer.add_slice(slice,slice['slice_id'])
+
     def AddSites(self, n = 10):
         """
         Add a number of random sites.
@@ -429,7 +526,7 @@ class Test:
                     assert site[field] == site_fields[field]
 
             if self.verbose:
-                print "Added site", site_id
+                print("Added site", site_id)
 
     def UpdateSites(self):
         """
@@ -451,7 +548,7 @@ class Test:
                     assert site[field] == site_fields[field]
 
             if self.verbose:
-                print "Updated site", site_id
+                print("Updated site", site_id)
 
     def DeleteSites(self):
         """
@@ -465,7 +562,7 @@ class Test:
                 assert not self.api.GetSites([site_id])
 
             if self.verbose:
-                print "Deleted site", site_id
+                print("Deleted site", site_id)
 
         if self.check:
             assert not self.api.GetSites(self.site_ids)
@@ -492,7 +589,7 @@ class Test:
                     assert address_type[field] == address_type_fields[field]
 
             if self.verbose:
-                print "Added address type", address_type_id
+                print("Added address type", address_type_id)
 
     def UpdateAddressTypes(self):
         """
@@ -511,7 +608,7 @@ class Test:
                     assert address_type[field] == address_type_fields[field]
 
             if self.verbose:
-                print "Updated address_type", address_type_id
+                print("Updated address_type", address_type_id)
 
     def DeleteAddressTypes(self):
         """
@@ -525,7 +622,7 @@ class Test:
                 assert not self.api.GetAddressTypes([address_type_id])
 
             if self.verbose:
-                print "Deleted address type", address_type_id
+                print("Deleted address type", address_type_id)
 
         if self.check:
             assert not self.api.GetAddressTypes(self.address_type_ids)
@@ -558,7 +655,7 @@ class Test:
                         assert address[field] == address_fields[field]
 
                 if self.verbose:
-                    print "Added address", address_id, "to site", site_id
+                    print("Added address", address_id, "to site", site_id)
 
     def UpdateAddresses(self):
         """
@@ -577,7 +674,7 @@ class Test:
                     assert address[field] == address_fields[field]
 
             if self.verbose:
-                print "Updated address", address_id
+                print("Updated address", address_id)
 
     def DeleteAddresses(self):
         """
@@ -600,7 +697,7 @@ class Test:
                 assert not self.api.GetAddresses([address_id])
 
             if self.verbose:
-                print "Deleted address", address_id
+                print("Deleted address", address_id)
 
         if self.check:
             assert not self.api.GetAddresses(self.address_ids)
@@ -665,7 +762,7 @@ class Test:
                     assert person['site_ids'][0] == site_id
 
                 if self.verbose:
-                    print "Added user", person_id, "to site", site_id
+                    print("Added user", person_id, "to site", site_id)
 
     def UpdatePersons(self):
         """
@@ -687,7 +784,7 @@ class Test:
                         assert person[field] == person_fields[field]
 
             if self.verbose:
-                print "Updated person", person_id
+                print("Updated person", person_id)
 
             person = self.api.GetPersons([person_id])[0]
 
@@ -706,7 +803,7 @@ class Test:
                 assert set(site_ids) == set(person['site_ids'])
 
             if self.verbose:
-                print "Updated person", person_id, "to sites", site_ids
+                print("Updated person", person_id, "to sites", site_ids)
 
     def DeletePersons(self):
         """
@@ -742,10 +839,10 @@ class Test:
             self.api.DeletePerson(person_id)
 
             if self.check:
-                assert not self.api.GetPersons([person_id])                         
+                assert not self.api.GetPersons([person_id])
 
             if self.verbose:
-                print "Deleted user", person_id
+                print("Deleted user", person_id)
 
         if self.check:
             assert not self.api.GetPersons(self.person_ids)
@@ -759,7 +856,7 @@ class Test:
 
         key_types = self.api.GetKeyTypes()
         if not key_types:
-            raise Exception, "No key types"
+            raise Exception("No key types")
 
         for person_id in self.person_ids:
             for i in range(per_person):
@@ -790,11 +887,11 @@ class Test:
                     try:
                         key_id = self.api.AddPersonKey(person_id, key_fields)
                         assert False
-                    except Exception, e:
+                    except Exception as e:
                         pass
 
                 if self.verbose:
-                    print "Added key", key_id, "to user", person_id
+                    print("Added key", key_id, "to user", person_id)
 
     def UpdateKeys(self):
         """
@@ -803,7 +900,7 @@ class Test:
 
         key_types = self.api.GetKeyTypes()
         if not key_types:
-            raise Exception, "No key types"
+            raise Exception("No key types")
 
         for key_id in self.key_ids:
             # Update key
@@ -817,7 +914,7 @@ class Test:
                     assert key[field] == key_fields[field]
 
             if self.verbose:
-                print "Updated key", key_id
+                print("Updated key", key_id)
 
     def DeleteKeys(self):
         """
@@ -831,7 +928,7 @@ class Test:
                 assert not self.api.GetKeys([key_id])
 
             if self.verbose:
-                print "Deleted key", key_id
+                print("Deleted key", key_id)
 
         if self.check:
             assert not self.api.GetKeys(self.key_ids)
@@ -847,7 +944,7 @@ class Test:
             # locate tag type
             tag_type_id = self.nodegroup_type_ids[i]
             tagname=self.api.GetTagTypes([tag_type_id])[0]['tagname']
-            
+
             # Add node group
             groupname = random_nodegroup() ['groupname']
             value = 'yes'
@@ -865,7 +962,7 @@ class Test:
                 assert nodegroup['value'] == value
 
             if self.verbose:
-                print "Added node group", nodegroup_id
+                print("Added node group", nodegroup_id)
 
     def UpdateNodeGroups(self):
         """
@@ -886,7 +983,7 @@ class Test:
                     assert nodegroup[field] == nodegroup_fields[field]
 
             if self.verbose:
-                print "Updated node group", nodegroup_id
+                print("Updated node group", nodegroup_id)
 
     def DeleteNodeGroups(self):
         """
@@ -900,7 +997,7 @@ class Test:
                 assert not self.api.GetNodeGroups([nodegroup_id])
 
             if self.verbose:
-                print "Deleted node group", nodegroup_id
+                print("Deleted node group", nodegroup_id)
 
         if self.check:
             assert not self.api.GetNodeGroups(self.nodegroup_ids)
@@ -913,13 +1010,13 @@ class Test:
         be added to a random node group if AddNodeGroups() was
         previously run.
         """
-        
+
         node_types = self.api.GetNodeTypes()
         if not node_types:
-            raise Exception, "No node types"
+            raise Exception("No node types")
         boot_states = self.api.GetBootStates()
         if not boot_states:
-            raise Exception, "No boot states"
+            raise Exception("No boot states")
 
         for site_id in self.site_ids:
             for i in range(per_site):
@@ -945,7 +1042,7 @@ class Test:
                             assert node[field] == node_fields[field]
 
                 if self.verbose:
-                    print "Added node", node_id
+                    print("Added node", node_id)
 
     def UpdateNodes(self):
         """
@@ -954,16 +1051,16 @@ class Test:
 
         node_types = self.api.GetNodeTypes()
         if not node_types:
-            raise Exception, "No node types"
+            raise Exception("No node types")
         boot_states = self.api.GetBootStates()
         if not boot_states:
-            raise Exception, "No boot states"
+            raise Exception("No boot states")
 
         for node_id in self.node_ids:
             # Update node
             node_fields = random_node(node_types,boot_states,self.namelengths)
             self.api.UpdateNode(node_id, node_fields)
-            
+
             node = self.api.GetNodes([node_id])[0]
 
             # Add to a random set of node groups
@@ -993,18 +1090,17 @@ class Test:
                 for field in node_fields:
                     if field not in tag_fields:
                         if node[field] != node_fields[field]:
-                            raise Exception, "Unexpected field %s in node after GetNodes()"%field
+                            raise Exception("Unexpected field %s in node after GetNodes()"%field)
                 assert set(nodegroup_ids) == set(node['nodegroup_ids'])
 
-                print 'WARNING: skipping updatenode with tags as this is not implemented yet'
-                # again when fetching 'arch' explicitly
-                node2 = self.api.GetNodes([node_id],node_fields.keys())[0]
+                # again but we are now fetching 'arch' explicitly
+                node2 = self.api.GetNodes([node_id],list(node_fields.keys()))[0]
                 for field in node_fields:
                     if node2[field] != node_fields[field]:
-                        raise Exception, "Unexpected field %s in node after GetNodes(tags)"%field
+                        raise Exception("Unexpected field %s in node after GetNodes(tags)"%field)
 
             if self.verbose:
-                print "Updated node", node_id
+                print("Updated node", node_id)
 
     def DeleteNodes(self):
         """
@@ -1014,7 +1110,7 @@ class Test:
         for node_id in self.node_ids:
             # Remove from node groups
             node = self.api.GetNodes([node_id])[0]
-            for node_tag in GetNodeTags ( {'node_id': node_id} ):
+            for node_tag in self.api.GetNodeTags ( {'node_id': node_id} ):
                 self.api.UpdateNodeTag(node_tag['node_tag_id'],'')
 
             if self.check:
@@ -1027,7 +1123,7 @@ class Test:
                 assert not self.api.GetNodes([node_id])
 
             if self.verbose:
-                print "Deleted node", node_id
+                print("Deleted node", node_id)
 
         if self.check:
             assert not self.api.GetNodes(self.node_ids)
@@ -1041,11 +1137,11 @@ class Test:
 
         network_methods = self.api.GetNetworkMethods()
         if not network_methods:
-            raise Exception, "No network methods"
-        
+            raise Exception("No network methods")
+
         network_types = self.api.GetNetworkTypes()
         if not network_types:
-            raise Exception, "No network types"
+            raise Exception("No network types")
 
         for node_id in self.node_ids:
             for i in range(per_node):
@@ -1053,7 +1149,7 @@ class Test:
                 type = random.sample(network_types, 1)[0]
 
                 # Add interface
-                interface_fields = random_interface(method, type)
+                interface_fields = random_interface(method, type,self.namelengths)
                 interface_id = self.api.AddInterface(node_id, interface_fields)
 
                 # Should return a unique interface_id
@@ -1067,7 +1163,7 @@ class Test:
                         assert interface[field] == interface_fields[field]
 
                 if self.verbose:
-                    print "Added interface", interface_id, "to node", node_id
+                    print("Added interface", interface_id, "to node", node_id)
 
     def UpdateInterfaces(self):
         """
@@ -1076,18 +1172,18 @@ class Test:
 
         network_methods = self.api.GetNetworkMethods()
         if not network_methods:
-            raise Exception, "No network methods"
-        
+            raise Exception("No network methods")
+
         network_types = self.api.GetNetworkTypes()
         if not network_types:
-            raise Exception, "No network types"
+            raise Exception("No network types")
 
         for interface_id in self.interface_ids:
             method = random.sample(network_methods, 1)[0]
             type = random.sample(network_types, 1)[0]
 
             # Update interface
-            interface_fields = random_interface(method, type)
+            interface_fields = random_interface(method, type,self.namelengths)
             self.api.UpdateInterface(interface_id, interface_fields)
 
             if self.check:
@@ -1097,7 +1193,7 @@ class Test:
                     assert interface[field] == interface_fields[field]
 
             if self.verbose:
-                print "Updated interface", interface_id
+                print("Updated interface", interface_id)
 
     def DeleteInterfaces(self):
         """
@@ -1111,13 +1207,13 @@ class Test:
                 assert not self.api.GetInterfaces([interface_id])
 
             if self.verbose:
-                print "Deleted interface", interface_id
+                print("Deleted interface", interface_id)
 
         if self.check:
             assert not self.api.GetInterfaces(self.interface_ids)
 
         self.interface_ids = []
-        
+
     def AddIlinks (self, n):
         """
         Add random links between interfaces.
@@ -1134,7 +1230,7 @@ class Test:
             self.ilink_ids.append(ilink_id)
 
             if self.verbose:
-                print 'Added Ilink',ilink_id,' - attached interface',src,'to',dst
+                print('Added Ilink',ilink_id,' - attached interface',src,'to',dst)
 
             if self.check:
                 retrieve=GetIlinks({'src_interface_id':src,'dst_interface_id':dst,
@@ -1153,7 +1249,7 @@ class Test:
                 assert ilink['value'] == new_value
 
             if self.verbose:
-                print 'Updated Ilink',ilink_id
+                print('Updated Ilink',ilink_id)
 
     def DeleteIlinks (self):
         for ilink_id in self.ilink_ids:
@@ -1163,7 +1259,7 @@ class Test:
                 assert not self.api.GetIlinks({'ilink_id':ilink_id})
 
             if self.verbose:
-                print 'Deleted Ilink',ilink_id
+                print('Deleted Ilink',ilink_id)
 
         if self.check:
             assert not self.api.GetIlinks(self.ilink_ids)
@@ -1202,7 +1298,7 @@ class Test:
                         assert pcu[field] == pcu_fields[field]
 
                 if self.verbose:
-                    print "Added PCU", pcu_id, "to site", site_id
+                    print("Added PCU", pcu_id, "to site", site_id)
 
     def UpdatePCUs(self):
         """
@@ -1221,7 +1317,7 @@ class Test:
                     assert pcu[field] == pcu_fields[field]
 
             if self.verbose:
-                print "Updated PCU", pcu_id
+                print("Updated PCU", pcu_id)
 
     def DeletePCUs(self):
         """
@@ -1244,7 +1340,7 @@ class Test:
                 assert not self.api.GetPCUs([pcu_id])
 
             if self.verbose:
-                print "Deleted PCU", pcu_id
+                print("Deleted PCU", pcu_id)
 
         if self.check:
             assert not self.api.GetPCUs(self.pcu_ids)
@@ -1301,12 +1397,12 @@ class Test:
                     assert conf_file[field] == conf_file_fields[field]
 
             if self.verbose:
-                print "Added configuration file", conf_file_id,
+                print("Added configuration file", conf_file_id, end=' ')
                 if nodegroup_id is not None:
-                    print "to node group", nodegroup_id,
+                    print("to node group", nodegroup_id, end=' ')
                 elif node_id is not None:
-                    print "to node", node_id,
-                print
+                    print("to node", node_id, end=' ')
+                print()
 
     def UpdateConfFiles(self):
         """
@@ -1328,7 +1424,7 @@ class Test:
                     assert conf_file[field] == conf_file_fields[field]
 
             if self.verbose:
-                print "Updated configuration file", conf_file_id
+                print("Updated configuration file", conf_file_id)
 
     def DeleteConfFiles(self):
         """
@@ -1342,7 +1438,7 @@ class Test:
                 assert not self.api.GetConfFiles([conf_file_id])
 
             if self.verbose:
-                print "Deleted configuration file", conf_file_id
+                print("Deleted configuration file", conf_file_id)
 
         if self.check:
             assert not self.api.GetConfFiles(self.conf_file_ids)
@@ -1351,13 +1447,13 @@ class Test:
 
     def AddTagTypes(self,n_sa,n_ng,n_il):
         """
-        Add as many tag types as there are nodegroups, 
+        Add as many tag types as there are nodegroups,
         will use value=yes for each nodegroup
         """
 
         roles = self.api.GetRoles()
         if not roles:
-            raise Exception, "No roles"
+            raise Exception("No roles")
         role_ids = [role['role_id'] for role in roles]
 
         for i in range (n_sa + n_ng + n_il):
@@ -1368,7 +1464,11 @@ class Test:
                 self.slice_type_ids + \
                 self.nodegroup_type_ids + \
                 self.ilink_type_ids
-            
+
+            tt_role_ids=random_roles(role_ids)
+            for tt_role_id in tt_role_ids:
+                self.api.AddRoleToTagType(tt_role_id,tag_type_id)
+
             if i < n_sa:
                 self.slice_type_ids.append(tag_type_id)
             elif i < n_sa+n_ng :
@@ -1380,8 +1480,10 @@ class Test:
                 tag_type = self.api.GetTagTypes([tag_type_id])[0]
                 for field in tag_type_fields:
                     assert tag_type[field] == tag_type_fields[field]
+                for tt_role_id in tt_role_ids:
+                    assert tt_role_id in tag_type['role_ids']
             if self.verbose:
-                print "Updated slice attribute type", tag_type_id
+                print("Created tag type", tag_type_id)
 
     def UpdateTagTypes(self):
         """
@@ -1390,7 +1492,7 @@ class Test:
 
         roles = self.api.GetRoles()
         if not roles:
-            raise Exception, "No roles"
+            raise Exception("No roles")
         role_ids = [role['role_id'] for role in roles]
 
         for tag_type_id in self.slice_type_ids + self.nodegroup_type_ids + self.ilink_type_ids:
@@ -1404,7 +1506,7 @@ class Test:
                 for field in tag_type_fields:
                     assert tag_type[field] == tag_type_fields[field]
             if self.verbose:
-                print "Updated slice attribute type", tag_type_id
+                print("Updated tag type", tag_type_id)
 
     def DeleteTagTypes(self):
         """
@@ -1418,7 +1520,7 @@ class Test:
                 assert not self.api.GetTagTypes([tag_type_id])
 
             if self.verbose:
-                print "Deleted slice attribute type", tag_type_id
+                print("Deleted tag type", tag_type_id)
 
         if self.check:
             assert not self.api.GetTagTypes(self.slice_type_ids+self.nodegroup_type_ids+self.ilink_type_ids)
@@ -1461,12 +1563,12 @@ class Test:
                     assert set(person_ids) == set(slice['person_ids'])
 
                 if self.verbose:
-                    print "Added slice", slice_id, "to site", site['site_id'],
+                    print("Added slice", slice_id, "to site", site['site_id'], end=' ')
                     if node_ids:
-                        print "and nodes", node_ids,
-                    print
+                        print("and nodes", node_ids, end=' ')
+                    print()
                     if person_ids:
-                        print "Added users", site['person_ids'], "to slice", slice_id
+                        print("Added users", site['person_ids'], "to slice", slice_id)
 
     def UpdateSlices(self):
         """
@@ -1503,9 +1605,9 @@ class Test:
                 assert set(person_ids) == set(slice['person_ids'])
 
             if self.verbose:
-                print "Updated slice", slice_id
-                print "Added nodes", node_ids, "to slice", slice_id
-                print "Added persons", person_ids, "to slice", slice_id
+                print("Updated slice", slice_id)
+                print("Added nodes", node_ids, "to slice", slice_id)
+                print("Added persons", person_ids, "to slice", slice_id)
 
     def DeleteSlices(self):
         """
@@ -1519,7 +1621,7 @@ class Test:
                 assert not self.api.GetSlices([slice_id])
 
             if self.verbose:
-                print "Deleted slice", slice_id
+                print("Deleted slice", slice_id)
 
         if self.check:
             assert not self.api.GetSlices(self.slice_ids)
@@ -1564,11 +1666,11 @@ class Test:
                             assert slice_tag[field] == locals()[field]
 
                     if self.verbose:
-                        print "Added slice attribute", slice_tag_id, "of type", tag_type_id,
+                        print("Added slice attribute", slice_tag_id, "of type", tag_type_id, end=' ')
                         if node_id is not None:
-                            print "to node", node_id,
-                        print
-                        
+                            print("to node", node_id, end=' ')
+                        print()
+
     def UpdateSliceTags(self):
         """
         Make random changes to any slice attributes we may have added.
@@ -1584,7 +1686,7 @@ class Test:
             assert slice_tag['value'] == value
 
             if self.verbose:
-                print "Updated slice attribute", slice_tag_id
+                print("Updated slice attribute", slice_tag_id)
 
     def DeleteSliceTags(self):
         """
@@ -1598,42 +1700,69 @@ class Test:
                 assert not self.api.GetSliceTags([slice_tag_id])
 
             if self.verbose:
-                print "Deleted slice attribute", slice_tag_id
+                print("Deleted slice attribute", slice_tag_id)
 
         if self.check:
             assert not self.api.GetSliceTags(self.slice_tag_ids)
 
         self.slice_tag_ids = []
 
+    # convenience for cleaning up
+    # not exactly accurate -- use on test plcs only
+    def WipeSitesFromLength(self):
+        for site in self.api.GetSites():
+            abbrev=site['abbreviated_name']
+#            print 'matching',len(abbrev),'against',self.namelengths['abbreviated_name']
+            if len(abbrev)==self.namelengths['abbreviated_name']:
+#            if len(abbrev)==17:
+                print('wiping site %d (%s)'%(site['site_id'],site['name']))
+                self.api.DeleteSite(site['site_id'])
+
 def main():
     parser = OptionParser()
-    parser.add_option("-c", "--check", action = "store_true", default = False, 
+    parser.add_option("-c", "--check", action = "store_true", default = False,
                       help = "Check most actions (default: %default)")
-    parser.add_option("-q", "--quiet", action = "store_true", default = False, 
+    parser.add_option("-q", "--quiet", action = "store_true", default = False,
                       help = "Be quiet (default: %default)")
     parser.add_option("-p","--preserve", action="store_true", default =False,
                       help = "Do not delete created objects")
-    parser.add_option("-t", "--tiny", action = "store_true", default = False, 
+    parser.add_option("-t", "--tiny", action = "store_true", default = False,
                       help = "Run a tiny test (default: %default)")
-    parser.add_option("-s", "--short-names", action="store_true", dest="short_names", default = False, 
+    parser.add_option("-l", "--large", action = "store_true", default = False,
+                      help = "Run a large test (default: %default)")
+    parser.add_option("-x", "--xlarge", action = "store_true", default = False,
+                      help = "Run an XL test (default: %default)")
+    parser.add_option("-s", "--short-names", action="store_true", dest="short_names", default = False,
                       help = "Generate smaller names for checking UI rendering")
+    parser.add_option ("-f", "--foreign", action="store_true", dest="federating", default = False,
+                       help = "Create a fake peer and add items in it (no update, no delete)")
+    parser.add_option ("-w", "--wipe", action="store_true", dest="wipe", default = False,
+                       help = "Wipe sites whose abbrev matches what the tests created")
     (options, args) = parser.parse_args()
 
     test = Test(api = Shell(),
                 check = options.check,
                 verbose = not options.quiet,
-                preserve = options.preserve)
-
-    if options.tiny:
-        sizes = Test.sizes_tiny
-    else:
-        sizes = Test.sizes_default
+                preserve = options.preserve,
+                federating = options.federating)
 
     if options.short_names:
         test.namelengths = Test.namelengths_short
     else:
         test.namelengths = Test.namelengths_default
 
+    if options.wipe:
+        test.WipeSitesFromLength()
+        return
+
+    if options.tiny:
+        sizes = Test.sizes_tiny
+    elif options.large:
+        sizes = Test.sizes_large
+    elif options.xlarge:
+        sizes = Test.sizes_xlarge
+    else:
+        sizes = Test.sizes_default
     test.Run(**sizes)
 
 if __name__ == "__main__":