From: Loic Baron Date: Fri, 18 May 2018 10:22:47 +0000 (+0200) Subject: Merge branch 'master' of git://git.onelab.eu/sfa X-Git-Tag: sfa-4.0-2~12 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=f1c66ceaece2d285f9b1fbd9381117124f7c69f6;hp=ae8c3ef9652be2d40dc2b39473aab6f7a08f961f Merge branch 'master' of git://git.onelab.eu/sfa --- diff --git a/sfa.spec b/sfa.spec index aebcbbe0..d80138e5 100644 --- a/sfa.spec +++ b/sfa.spec @@ -1,6 +1,6 @@ %define name sfa %define version 3.1 -%define taglevel 21 +%define taglevel 22 %define release %{taglevel}%{?pldistro:.%{pldistro}}%{?date:.%{date}} %global python_sitearch %( python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)" ) @@ -252,6 +252,14 @@ fi #[ "$1" -ge "1" ] && service sfa-cm restart || : %changelog +* Fri Mar 16 2018 Thierry - sfa-3.1-22 +- pl: tweaks for exposing country / city on nodes from site tags if set +- pl: tweaks for exposing hardware_types on nodes from node tag 'hardware_type' if set +- pl: fix exposing granularity +- sfaresetgids.py: a utility to reset gids when a update of the tolpevel gis is needed +- iotlab: various tweaks +- patch backported from plcapi about issues with xmlrpc and unicode under fedora >= 24 + * Fri Jan 13 2017 Thierry Parmentelat - sfa-3.1-21 - sfax509 command can run openssl x509 on all the parts of a gid - bugfix in sfi when running the discover subcommand diff --git a/sfa/managers/aggregate_manager.py b/sfa/managers/aggregate_manager.py index a8c3af4b..bb9559f0 100644 --- a/sfa/managers/aggregate_manager.py +++ b/sfa/managers/aggregate_manager.py @@ -1,4 +1,5 @@ -import socket +# pylint: disable=c0111, c0103, r0201 + from sfa.rspecs.version_manager import VersionManager from sfa.util.version import version_core from sfa.util.xrn import Xrn @@ -10,7 +11,8 @@ from sfa.server.api_versions import ApiVersions class AggregateManager: - def __init__(self, config): pass + def __init__(self, config): + pass # essentially a union of the core version, the generic version (this code) and # whatever the driver needs to expose @@ -20,6 +22,15 @@ class AggregateManager: ad_rspec_versions = [] request_rspec_versions = [] for rspec_version in version_manager.versions: + # avoid publishing non-relevant entries + # but stay safe however + try: + if not rspec_version.extensions \ + and not rspec_version.namespace \ + and not rspec_version.schema: + continue + except Exception as exc: + pass if rspec_version.content_type in ['*', 'ad']: ad_rspec_versions.append(rspec_version.to_dict()) if rspec_version.content_type in ['*', 'request']: @@ -57,8 +68,8 @@ class AggregateManager: cred_types = [{'geni_type': 'geni_sfa', 'geni_version': str(i)} for i in range(4)[-2:]] geni_api_versions = ApiVersions().get_versions() - geni_api_versions[ - '3'] = 'http://%s:%s' % (api.config.sfa_aggregate_host, api.config.sfa_aggregate_port) + geni_api_versions['3'] = \ + 'https://%s:%s' % (api.config.sfa_aggregate_host, api.config.sfa_aggregate_port) version_generic = { 'testbed': api.driver.testbed_name(), 'interface': 'aggregate', @@ -127,7 +138,7 @@ class AggregateManager: def Allocate(self, api, xrn, creds, rspec_string, expiration, options): """ - Allocate resources as described in a request RSpec argument + Allocate resources as described in a request RSpec argument to a slice with the named URN. """ call_id = options.get('call_id') @@ -137,7 +148,7 @@ class AggregateManager: def Provision(self, api, xrns, creds, options): """ - Create the sliver[s] (slice) at this aggregate. + Create the sliver[s] (slice) at this aggregate. Verify HRN and initialize the slice record in PLC if necessary. """ call_id = options.get('call_id') diff --git a/sfa/managers/slice_manager.py b/sfa/managers/slice_manager.py index 011a22ee..698ced0c 100644 --- a/sfa/managers/slice_manager.py +++ b/sfa/managers/slice_manager.py @@ -55,7 +55,7 @@ class SliceManager: 'interface': 'slicemgr', 'sfa': 2, 'geni_api': 3, - 'geni_api_versions': {'3': 'http://%s:%s' % (api.config.SFA_SM_HOST, api.config.SFA_SM_PORT)}, + 'geni_api_versions': {'3': 'https://%s:%s' % (api.config.SFA_SM_HOST, api.config.SFA_SM_PORT)}, 'hrn': xrn.get_hrn(), 'urn': xrn.get_urn(), 'peers': peers, diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py index 8095291d..01558602 100644 --- a/sfa/planetlab/plaggregate.py +++ b/sfa/planetlab/plaggregate.py @@ -220,7 +220,7 @@ class PlAggregate: # slice-global tags node['slice-tags'] = pltags_dict['slice-global'] # xxx - # this is where we chould maybe add the nodegroup slice tags, + # this is where we should maybe add the nodegroup slice tags, # but it's tedious... # xxx # sliver tags @@ -264,16 +264,42 @@ class PlAggregate: else: rspec_node['exclusive'] = 'false' - rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}), - HardwareType({'name': 'pc'})] + # expose hardware_types from the hardware_type tag if + # set on node + tags = self.driver.shell.GetNodeTags({ + 'node_id': node['node_id'], + 'tagname': 'hardware_type', + }) + if tags: + rspec_node['hardware_types'] = [ + HardwareType({'name': tags[0]['value']}), + ] + else: + rspec_node['hardware_types'] = [ + HardwareType({'name': 'plab-pc'}), + HardwareType({'name': 'pc'}) + ] # only doing this because protogeni rspec needs # to advertise available initscripts rspec_node['pl_initscripts'] = pl_initscripts.values() # add site/interface info to nodes. # assumes that sites, interfaces and tags have already been prepared. if site['longitude'] and site['latitude']: - location = Location({'longitude': site['longitude'], 'latitude': site[ - 'latitude'], 'country': 'unknown'}) + location_dict = { + 'longitude': site['longitude'], + 'latitude': site['latitude'], + } + for extra in ('country', 'city'): + try: + tags = self.driver.shell.GetSiteTags({ + 'site_id' : site['site_id'], + 'tagname' : extra, + }) + location_dict[extra] = tags[0]['value'] + except: + logger.log_exc('extra = {}'.format(extra)) + location_dict[extra] = 'unknown' + location = Location(location_dict) rspec_node['location'] = location # Granularity granularity = Granularity({'grain': grain}) @@ -479,10 +505,11 @@ class PlAggregate: node_tags = self.get_node_tags({'node_tag_id': tag_ids}) pl_initscripts = self.get_pl_initscripts() # convert nodes to rspec nodes + grain = self.driver.shell.GetLeaseGranularity() rspec_nodes = [] for node in nodes: rspec_node = self.node_to_rspec_node( - node, sites, interfaces, node_tags, pl_initscripts) + node, sites, interfaces, node_tags, pl_initscripts, grain) rspec_nodes.append(rspec_node) rspec.version.add_nodes(rspec_nodes) diff --git a/sfa/planetlab/plshell.py b/sfa/planetlab/plshell.py index 8877f410..94b66eac 100644 --- a/sfa/planetlab/plshell.py +++ b/sfa/planetlab/plshell.py @@ -23,7 +23,7 @@ class PlShell: 'UnBindObjectFromPeer', 'UpdateNode', 'UpdatePerson', 'UpdateSite', 'UpdateSlice', 'UpdateSliceTag', # also used as-is in importer - 'GetSites', 'GetNodes', + 'GetSites', 'GetNodes', 'GetSiteTags', # Lease management methods 'GetLeases', 'GetLeaseGranularity', 'DeleteLeases', 'UpdateLeases', 'AddLeases', diff --git a/sfa/rspecs/elements/location.py b/sfa/rspecs/elements/location.py index f99c5432..e42e1600 100644 --- a/sfa/rspecs/elements/location.py +++ b/sfa/rspecs/elements/location.py @@ -7,4 +7,5 @@ class Location(Element): 'country', 'longitude', 'latitude', + 'city', ] diff --git a/sfa/server/sfa-start.py b/sfa/server/sfa-start.py index bca06ee8..bcd00cd6 100755 --- a/sfa/server/sfa-start.py +++ b/sfa/server/sfa-start.py @@ -46,11 +46,12 @@ from sfa.server.registry import Registries from sfa.server.aggregate import Aggregates from sfa.client.return_value import ReturnValue -# after http://www.erlenstar.demon.co.uk/unix/faq_2.html - def daemon(): - """Daemonize the current process.""" + """ + Daemonize the current process. + after http://www.erlenstar.demon.co.uk/unix/faq_2.html + """ if os.fork() != 0: os._exit(0) os.setsid() diff --git a/sfa/trust/auth.py b/sfa/trust/auth.py index 16eb8a69..f8ac90f3 100644 --- a/sfa/trust/auth.py +++ b/sfa/trust/auth.py @@ -67,17 +67,19 @@ class Auth: xrns = [] error = (None, None) - def log_invalid_cred(cred): + def log_invalid_cred(cred, exception): if not isinstance(cred, StringType): logger.info( - "cannot validate credential %s - expecting a string" % cred) + "{}: cannot validate credential {}" + .format(exception, cred)) error = ('TypeMismatch', - "checkCredentials: expected a string, received {} -- {}" + "checkCredentials: expected a string, got {} -- {}" .format(type(cred), cred)) else: cred_obj = Credential(string=cred) - logger.info("failed to validate credential - dump=%s" % - cred_obj.dump_string(dump_parents=True)) + logger.info("{}: failed to validate credential dump={}" + .format(exception, + cred_obj.dump_string(dump_parents=True))) error = sys.exc_info()[:2] return error @@ -90,7 +92,7 @@ class Auth: if not isinstance(xrns, list): xrns = [xrns] - slice_xrns = Xrn.filter_type(xrns, 'slice') + # slice_xrns = Xrn.filter_type(xrns, 'slice') sliver_xrns = Xrn.filter_type(xrns, 'sliver') # we are not able to validate slivers in the traditional way so @@ -121,8 +123,8 @@ class Auth: try: self.check(cred, operation, hrn) valid.append(cred) - except: - error = log_invalid_cred(cred) + except Exception as exc: + error = log_invalid_cred(cred, exc) # make sure all sliver xrns are validated against the valid credentials if sliver_xrns: @@ -140,11 +142,11 @@ class Auth: def check(self, credential, operation, hrn=None): """ - Check the credential against the peer cert (callerGID) included - in the credential matches the caller that is connected to the - HTTPS connection, check if the credential was signed by a - trusted cert and check if the credential is allowed to perform - the specified operation. + Check the credential against the peer cert (callerGID) included + in the credential matches the caller that is connected to the + HTTPS connection, check if the credential was signed by a + trusted cert and check if the credential is allowed to perform + the specified operation. """ cred = Credential(cred=credential) self.client_cred = cred @@ -265,16 +267,16 @@ class Auth: Given an authority name, return the information for that authority. This is basically a stub that calls the hierarchy module. - @param auth_hrn human readable name of authority + @param auth_hrn human readable name of authority """ return self.hierarchy.get_auth_info(auth_hrn) def veriry_auth_belongs_to_me(self, name): """ - Verify that an authority belongs to our hierarchy. + Verify that an authority belongs to our hierarchy. This is basically left up to the implementation of the hierarchy - module. If the specified name does not belong, ane exception is + module. If the specified name does not belong, ane exception is thrown indicating the caller should contact someone else. @param auth_name human readable name of authority @@ -289,7 +291,7 @@ class Auth: this implies that the authority that owns the object belongs to our hierarchy. If it does not an exception is thrown. - @param name human readable name of object + @param name human readable name of object """ auth_name = self.get_authority(name) if not auth_name: @@ -306,10 +308,10 @@ class Auth: """ Verify that the object gid that was specified in the credential allows permission to the object 'name'. This is done by a simple - prefix test. For example, an object_gid for plc.arizona would + prefix test. For example, an object_gid for plc.arizona would match the objects plc.arizona.slice1 and plc.arizona. - @param name human readable name to test + @param name human readable name to test """ object_hrn = self.object_gid.get_hrn() if object_hrn == name: @@ -381,7 +383,7 @@ class Auth: def filter_creds_by_caller(self, creds, caller_hrn_list): """ - Returns a list of creds who's gid caller matches the + Returns a list of creds who's gid caller matches the specified caller hrn """ if not isinstance(creds, list): diff --git a/sfa/util/sfalogging.py b/sfa/util/sfalogging.py index 2b7d7823..434043fe 100644 --- a/sfa/util/sfalogging.py +++ b/sfa/util/sfalogging.py @@ -166,12 +166,8 @@ class _SfaLogger: self.logger.addHandler(handler) -info_logger = _SfaLogger(loggername='info', level=logging.INFO) -debug_logger = _SfaLogger(loggername='debug', level=logging.DEBUG) -warn_logger = _SfaLogger(loggername='warning', level=logging.WARNING) -error_logger = _SfaLogger(loggername='error', level=logging.ERROR) -critical_logger = _SfaLogger(loggername='critical', level=logging.CRITICAL) -logger = info_logger +logger = _SfaLogger(loggername='info', level=logging.INFO) + sfi_logger = _SfaLogger(logfile=os.path.expanduser("~/.sfi/") + 'sfi.log', loggername='sfilog', level=logging.DEBUG) ######################################## diff --git a/sfa/util/xrn.py b/sfa/util/xrn.py index 1a7b8b8d..692c80d8 100644 --- a/sfa/util/xrn.py +++ b/sfa/util/xrn.py @@ -177,9 +177,6 @@ class Xrn: self.type = type self.hrn_to_urn() self._normalize() -# happens all the time .. -# if not type: -# debug_logger.debug("type-less Xrn's are not safe") def __repr__(self): result = "