From: Sandrine Avakian Date: Tue, 10 Dec 2013 13:05:53 +0000 (+0100) Subject: Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3 X-Git-Tag: sfa-3.1-2~51 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=fae309c53ba7e65f1b51047d8c8fbb7a10477713;hp=a4c02db39311a1b6ff0a6ece3cb2ba5c6bcca112 Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3 --- diff --git a/setup.py b/setup.py index 162d0577..33f04340 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,6 @@ packages = [ 'sfa/rspecs', 'sfa/rspecs/elements', 'sfa/rspecs/elements/versions', - 'sfa/rspecs/elements/v3', 'sfa/rspecs/versions', 'sfa/client', 'sfa/planetlab', diff --git a/sfa.spec b/sfa.spec index e8f8dafe..ed3af433 100644 --- a/sfa.spec +++ b/sfa.spec @@ -1,6 +1,6 @@ %define name sfa -%define version 3.0 -%define taglevel 2 +%define version 3.1 +%define taglevel 1 %define release %{taglevel}%{?pldistro:.%{pldistro}}%{?date:.%{date}} %global python_sitearch %( python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)" ) @@ -265,6 +265,25 @@ fi #[ "$1" -ge "1" ] && service sfa-cm restart || : %changelog +* Tue Dec 10 2013 Thierry Parmentelat - sfa-3.1-1 +- -- core +- clean up rspecs. +- GENIv3 rspecs enhirits everything from pgv2. +- fix dbsession mgt by attributing each incoming API call has its own dbsession. +- update all drivers and v2_to_v3_adapter according to dbsession fix. +- fix DB migration. +- -- client +- fix sfi.py myslice. +- sfi.py new option "my_pkcs12". +- sfi.py defaulr rspec version is GENIv3 +- sfi.py trusted target either Registry interface or Slice interface. +- -- PlanetLab +- rebase all pl driver operations around the objects HRNs. +- ignore objects (Site/Slice/Person/Node) coming from MyPLC peering. +- importer ignores sites with tag sfa_created='True'. +- -- IoTLab +- In progress writing of native AM API v3 compliant driver. + * Thu Oct 10 2013 Thierry Parmentelat - sfa-3.0-2 - -- core - Extend all versions of rspecs in order to support "links" and "channels" management methods diff --git a/sfa/client/sfi.py b/sfa/client/sfi.py index 2f47839d..157828eb 100644 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@ -387,13 +387,15 @@ class Sfi: help='how myslice config variables as well') if command in ("version"): - parser.add_option("-R","--registry-version", - action="store_true", dest="version_registry", default=False, - help="probe registry version instead of sliceapi") parser.add_option("-l","--local", action="store_true", dest="version_local", default=False, help="display version of the local client") + if command in ("version", "trusted"): + parser.add_option("-R","--registry-interface", + action="store_true", dest="registry-interface", default=False, + help="target the registry interface instead of slice interface") + if command in ("add", "update"): parser.add_option('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') parser.add_option('-t', '--type', dest='type', metavar='', help='object type', default=None) @@ -878,7 +880,7 @@ use this if you mean an authority instead""") if options.version_local: version=version_core() else: - if options.version_registry: + if options.registry-interface: server=self.registry() else: server = self.sliceapi() @@ -1095,7 +1097,7 @@ use this if you mean an authority instead""") if self.options.raw: save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) if options.file is not None: - save_rspec_to_file(value['geni_rspec'], options.file) + save_rspec_to_file(value, options.file) if (self.options.raw is None) and (options.file is None): display_rspec(value, options.format) @@ -1616,7 +1618,15 @@ $ sfi m -b http://mymanifold.foo.com:7080/ """ return the trusted certs at this interface (get_trusted_certs) """ - trusted_certs = self.registry().get_trusted_certs() + if options.registry-interface: + server=self.registry() + else: + server = self.sliceapi() + cred = self.my_authority_credential_string() + trusted_certs = server.get_trusted_certs(cred) + if not options.registry-interface: + trusted_certs = ReturnValue.get_value(trusted_certs) + for trusted_cert in trusted_certs: print "\n===========================================================\n" gid = GID(string=trusted_cert) diff --git a/sfa/importer/plimporter.py b/sfa/importer/plimporter.py index d281c614..1a296e1d 100644 --- a/sfa/importer/plimporter.py +++ b/sfa/importer/plimporter.py @@ -194,7 +194,11 @@ class PlImporter: # start importing for site in sites: - if site['name'].startswith('sfa:'): + try: + site_sfa_created = shell.GetSiteSfaCreated(site['site_id']) + except: + site_sfa_created = None + if site['name'].startswith('sfa:') or site_sfa_created == 'True': continue site_hrn = _get_site_hrn(interface_hrn, site) diff --git a/sfa/managers/v2_to_v3_adapter.py b/sfa/managers/v2_to_v3_adapter.py index 3ed22765..1b934ce0 100644 --- a/sfa/managers/v2_to_v3_adapter.py +++ b/sfa/managers/v2_to_v3_adapter.py @@ -20,11 +20,8 @@ class V2ToV3Adapter: elif flavour == "fd": from sfa.federica.fddriver import FdDriver self.driver = FdDriver(api) - elif flavour == "iotlab": - from sfa.iotlab.iotlabdriver import IotlabDriver - self.driver = IotlabDriver(api) else: - logger.info("DriverV2Adapter unknown flavour !!!\n supported flavours: pl, nitos, fd, iotlab") + logger.error("V2ToV3Adapter: Unknown Flavour !!!\n Supported Flavours: nitos, fd") # Caching if config.SFA_AGGREGATE_CACHING: diff --git a/sfa/planetlab/plslices.py b/sfa/planetlab/plslices.py index 6d21a13d..5e106cdc 100644 --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@ -415,8 +415,10 @@ class PlSlices: slice_name = '_'.join([login_base, slice_part]) persons = self.driver.shell.GetPersons({'peer_id': None},['person_id','email','hrn']) - site = self.driver.shell.GetSites({'peer_id': None, 'login_base': login_base})[0] - slice = self.driver.shell.GetSlices({'peer_id': None, 'name': slice_name})[0] + sites = self.driver.shell.GetSites({'peer_id': None}, ['node_ids', 'site_id', 'name', 'person_ids', 'slice_ids', 'login_base', 'hrn']) + site = [site for site in sites if site['hrn'] == site_hrn][0] + slices = self.driver.shell.GetSlices({'peer_id': None}, ['slice_id', 'node_ids', 'person_ids', 'expires', 'site_id', 'name', 'hrn']) + slice = [slice for slice in slices if slice['hrn'] == slice_hrn][0] slice_persons = self.driver.shell.GetPersons({'peer_id': None, 'person_id': slice['person_ids']},['person_id','email','hrn']) persons_by_hrn = {} @@ -456,10 +458,10 @@ class PlSlices: self.driver.shell.AddRoleToPerson('user', int(person_id)) # enable the account self.driver.shell.UpdatePerson(int(person_id), {'enabled': True}) - self.driver.shell.SetPersonHrn(int(person_id), person_hrn) self.driver.shell.SetPersonSfaCreated(int(person_id), 'True') self.driver.shell.AddPersonToSite(int(person_id), site['site_id']) self.driver.shell.AddPersonToSlice(int(person_id), slice['slice_id']) + self.driver.shell.SetPersonHrn(int(person_id), person_hrn) # Add keys for key in users_by_hrn[person_hrn].get('keys', []): diff --git a/sfa/rspecs/elements/v3/__init__.py b/sfa/rspecs/elements/v3/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sfa/rspecs/elements/v3/node.py b/sfa/rspecs/elements/v3/node.py deleted file mode 100644 index 1f20c680..00000000 --- a/sfa/rspecs/elements/v3/node.py +++ /dev/null @@ -1,144 +0,0 @@ -from sfa.util.xrn import Xrn, get_leaf -from sfa.util.xml import XpathFilter - -from sfa.rspecs.elements.node import NodeElement -from sfa.rspecs.elements.sliver import Sliver -from sfa.rspecs.elements.location import Location -from sfa.rspecs.elements.hardware_type import HardwareType -from sfa.rspecs.elements.disk_image import DiskImage -from sfa.rspecs.elements.interface import Interface -from sfa.rspecs.elements.bwlimit import BWlimit -from sfa.rspecs.elements.pltag import PLTag -from sfa.rspecs.elements.v3.services import Services -from sfa.rspecs.elements.versions.pgv2SliverType import PGv2SliverType -from sfa.rspecs.elements.versions.pgv2Interface import PGv2Interface - - -class Node: - @staticmethod - def add_nodes(xml, nodes): - node_elems = [] - for node in nodes: - node_fields = ['component_manager_id', 'component_id', 'client_id', 'sliver_id', 'exclusive'] - node_elem = xml.add_instance('node', node, node_fields) - node_elems.append(node_elem) - # set component name - if node.get('component_id'): - component_name = Xrn.unescape(get_leaf(Xrn(node['component_id']).get_hrn())) - node_elem.set('component_name', component_name) - # set hardware types - if node.get('hardware_types'): - for hardware_type in node.get('hardware_types', []): - node_elem.add_instance('hardware_type', hardware_type, HardwareType.fields) - # set location - if node.get('location'): - node_elem.add_instance('location', node['location'], Location.fields) - # set interfaces - PGv2Interface.add_interfaces(node_elem, node.get('interfaces')) - #if node.get('interfaces'): - # for interface in node.get('interfaces', []): - # node_elem.add_instance('interface', interface, ['component_id', 'client_id']) - # set available element - if node.get('available'): - available_elem = node_elem.add_element('available', now=node['available']) - # add services - Services.add_services(node_elem, node.get('services', [])) - # add slivers - slivers = node.get('slivers', []) - if not slivers: - # we must still advertise the available sliver types - slivers = Sliver({'type': 'plab-vserver'}) - # we must also advertise the available initscripts - slivers['tags'] = [] - if node.get('pl_initscripts'): - for initscript in node.get('pl_initscripts', []): - slivers['tags'].append({'name': 'initscript', 'value': initscript['name']}) - PGv2SliverType.add_slivers(node_elem, slivers) - - return node_elems - - @staticmethod - def get_nodes(xml, filter={}): - xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) - node_elems = xml.xpath(xpath) - return Node.get_node_objs(node_elems) - - @staticmethod - def get_nodes_with_slivers(xml, filter={}): - xpath = '//node[count(sliver_type)>0] | //default:node[count(default:sliver_type) > 0]' - node_elems = xml.xpath(xpath) - return Node.get_node_objs(node_elems) - - @staticmethod - def get_node_objs(node_elems): - nodes = [] - for node_elem in node_elems: - node = NodeElement(node_elem.attrib, node_elem) - nodes.append(node) - if 'component_id' in node_elem.attrib: - node['authority_id'] = Xrn(node_elem.attrib['component_id']).get_authority_urn() - - # get hardware types - hardware_type_elems = node_elem.xpath('./default:hardware_type | ./hardware_type') - node['hardware_types'] = [hw_type.get_instance(HardwareType) for hw_type in hardware_type_elems] - - # get location - location_elems = node_elem.xpath('./default:location | ./location') - locations = [location_elem.get_instance(Location) for location_elem in location_elems] - if len(locations) > 0: - node['location'] = locations[0] - - # get interfaces - iface_elems = node_elem.xpath('./default:interface | ./interface') - node['interfaces'] = [iface_elem.get_instance(Interface) for iface_elem in iface_elems] - - # get services - node['services'] = Services.get_services(node_elem) - - # get slivers - node['slivers'] = PGv2SliverType.get_slivers(node_elem) - available_elems = node_elem.xpath('./default:available | ./available') - if len(available_elems) > 0 and 'name' in available_elems[0].attrib: - if available_elems[0].attrib.get('now', '').lower() == 'true': - node['boot_state'] = 'boot' - else: - node['boot_state'] = 'disabled' - return nodes - - - @staticmethod - def add_slivers(xml, slivers): - component_ids = [] - for sliver in slivers: - filter = {} - if isinstance(sliver, str): - filter['component_id'] = '*%s*' % sliver - sliver = {} - elif 'component_id' in sliver and sliver['component_id']: - filter['component_id'] = '*%s*' % sliver['component_id'] - if not filter: - continue - nodes = Node.get_nodes(xml, filter) - if not nodes: - continue - node = nodes[0] - PGv2SliverType.add_slivers(node, sliver) - - @staticmethod - def remove_slivers(xml, hostnames): - for hostname in hostnames: - nodes = Node.get_nodes(xml, {'component_id': '*%s*' % hostname}) - for node in nodes: - slivers = PGv2SliverType.get_slivers(node.element) - for sliver in slivers: - node.element.remove(sliver.element) -if __name__ == '__main__': - from sfa.rspecs.rspec import RSpec - import pdb - r = RSpec('/tmp/emulab.rspec') - r2 = RSpec(version = 'GENI') - nodes = Node.get_nodes(r.xml) - Node.add_nodes(r2.xml.root, nodes) - #pdb.set_trace() - - diff --git a/sfa/rspecs/elements/v3/services.py b/sfa/rspecs/elements/v3/services.py deleted file mode 100644 index 83b300bf..00000000 --- a/sfa/rspecs/elements/v3/services.py +++ /dev/null @@ -1,60 +0,0 @@ -from sfa.rspecs.elements.element import Element -from sfa.rspecs.elements.execute import Execute -from sfa.rspecs.elements.install import Install -from sfa.rspecs.elements.services import ServicesElement -from sfa.rspecs.elements.login import Login - -class Services: - @staticmethod - def add_services(xml, services): - if not services: - return - for service in services: - service_elem = xml.add_element('services') - child_elements = {'install': Install.fields, - 'execute': Execute.fields, - 'login': Login.fields} - for (name, fields) in child_elements.items(): - child = service.get(name) - if not child: - continue - if isinstance(child, dict): - service_elem.add_instance(name, child, fields) - elif isinstance(child, list): - for obj in child: - service_elem.add_instance(name, obj, fields) - - # add ssh_users - if service['services_user']: - for ssh_user in service['services_user']: - ssh_user_elem = service_elem.add_element('{%s}services_user' % xml.namespaces['ssh-user'], - login=ssh_user['login'], - user_urn=ssh_user['user_urn']) - for key in ssh_user['keys']: - pkey_elem = ssh_user_elem.add_element('{%s}public_key' % xml.namespaces['ssh-user']) - pkey_elem.element.text=key - - - @staticmethod - def get_services(xml): - services = [] - for services_elem in xml.xpath('./default:services | ./services'): - service = ServicesElement(services_elem.attrib, services_elem) - # get install - install_elems = services_elem.xpath('./default:install | ./install') - service['install'] = [install_elem.get_instance(Install) for install_elem in install_elems] - # get execute - execute_elems = services_elem.xpath('./default:execute | ./execute') - service['execute'] = [execute_elem.get_instance(Execute) for execute_elem in execute_elems] - # get login - login_elems = services_elem.xpath('./default:login | ./login') - service['login'] = [login_elem.get_instance(Login) for login_elem in login_elems] - - ssh_user_elems = services_elem.xpath('./ssh-user:service_user | ./service_user') - services_user = [] - for ssh_user_elem in ssh_user_elems: - services_user = ssh_user_elem.get_instance(None, fields=['login', 'user_urn']) - service['services_user'] = services_user - services.append(service) - return services - diff --git a/sfa/rspecs/elements/versions/pgv2Services.py b/sfa/rspecs/elements/versions/pgv2Services.py index 5cdf22cf..e0acc714 100644 --- a/sfa/rspecs/elements/versions/pgv2Services.py +++ b/sfa/rspecs/elements/versions/pgv2Services.py @@ -23,6 +23,16 @@ class PGv2Services: elif isinstance(child, list): for obj in child: service_elem.add_instance(name, obj, fields) + +# # add ssh_users +# if service['services_user']: +# for ssh_user in service['services_user']: +# ssh_user_elem = service_elem.add_element('{%s}services_user' % xml.namespaces['ssh-user'], +# login=ssh_user['login'], +# user_urn=ssh_user['user_urn']) +# for key in ssh_user['keys']: +# pkey_elem = ssh_user_elem.add_element('{%s}public_key' % xml.namespaces['ssh-user']) +# pkey_elem.element.text=key @staticmethod def get_services(xml): @@ -38,6 +48,13 @@ class PGv2Services: # get login login_elems = services_elem.xpath('./default:login | ./login') service['login'] = [login_elem.get_instance(Login) for login_elem in login_elems] + +# ssh_user_elems = services_elem.xpath('./ssh-user:service_user | ./service_user') +# services_user = [] +# for ssh_user_elem in ssh_user_elems: +# services_user = ssh_user_elem.get_instance(None, fields=['login', 'user_urn']) +# service['services_user'] = services_user + services.append(service) return services diff --git a/sfa/rspecs/versions/pgv3.py b/sfa/rspecs/versions/pgv3.py index a4413c16..6d6a4494 100644 --- a/sfa/rspecs/versions/pgv3.py +++ b/sfa/rspecs/versions/pgv3.py @@ -1,5 +1,4 @@ from sfa.rspecs.versions.pgv2 import PGv2 -from sfa.rspecs.elements.v3.node import Node class GENIv3(PGv2): type = 'GENI' @@ -53,10 +52,3 @@ class GENIv2Manifest(GENIv3): content_type = 'manifest' schema = 'http://www.geni.net/resources/rspec/3/manifest.xsd' template = '' - - - def add_nodes(self, nodes, check_for_dupes=False): - return Node.add_nodes(self.xml, nodes) - - def get_nodes(self, filter=None): - return Node.get_nodes(self.xml, filter)