From: fsaintma Date: Tue, 17 Mar 2015 13:59:32 +0000 (+0100) Subject: Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3 X-Git-Tag: sfa-3.1-15~17 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=82d0355774b73966cbe0c838c611808dfd2f1c7a;hp=c4f459b37989f2e396c4e4e69994bc6ac07ee6f1 Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3 --- diff --git a/init.d/functions.sfa b/init.d/functions.sfa index 010ab419..038adc9b 100644 --- a/init.d/functions.sfa +++ b/init.d/functions.sfa @@ -32,6 +32,10 @@ if [ $PPID -ne 1 -a -z "$SYSTEMCTL_SKIP_REDIRECT" ] && \ esac fi +# ubuntu does not have /bin/systemctl +[ -f /bin/systemctl ] || _use_systemctl=0 + + systemctl_redirect () { local s local prog=${1##*/} diff --git a/sfa/client/manifolduploader.py b/sfa/client/manifolduploader.py index f8ca0f9d..8354e1ef 100755 --- a/sfa/client/manifolduploader.py +++ b/sfa/client/manifolduploader.py @@ -25,6 +25,10 @@ DEFAULT_URL = "http://myslice.onelab.eu:7080" DEFAULT_PLATFORM = 'ple' +# starting with 2.7.9 we need to turn off server verification +import ssl +ssl_needs_unverified_context = hasattr(ssl, '_create_unverified_context') + import xmlrpclib import getpass @@ -78,7 +82,12 @@ class ManifoldUploader: # return self._proxy url=self.url() self.logger.debug("Connecting manifold url %s"%url) - return xmlrpclib.ServerProxy(url, allow_none = True) + if not ssl_needs_unverified_context: + proxy = xmlrpclib.ServerProxy(url, allow_none = True) + else: + proxy = xmlrpclib.ServerProxy(url, allow_none = True, + context=ssl._create_unverified_context()) + return proxy # does the job for one credential # expects the credential (string) and an optional message (e.g. hrn) for reporting diff --git a/sfa/client/sfaclientlib.py b/sfa/client/sfaclientlib.py index 78f6f48d..bc4a1d14 100644 --- a/sfa/client/sfaclientlib.py +++ b/sfa/client/sfaclientlib.py @@ -121,7 +121,7 @@ class SfaClientBootstrap: ######################################## *_produce methods ### step1 # unconditionnally create a self-signed certificate - def self_signed_cert_produce (self,output): + def self_signed_cert_produce (self, output): self.assert_private_key() private_key_filename = self.private_key_filename() keypair=Keypair(filename=private_key_filename) @@ -131,7 +131,7 @@ class SfaClientBootstrap: self_signed.sign () self_signed.save_to_file (output) self.logger.debug("SfaClientBootstrap: Created self-signed certificate for %s in %s"%\ - (self.hrn,output)) + (self.hrn, output)) return output ### step2 @@ -142,7 +142,8 @@ class SfaClientBootstrap: certificate_filename = self.self_signed_cert_filename() certificate_string = self.plain_read (certificate_filename) self.assert_private_key() - registry_proxy = SfaServerProxy (self.registry_url, self.private_key_filename(), + registry_proxy = SfaServerProxy (self.registry_url, + self.private_key_filename(), certificate_filename) try: credential_string=registry_proxy.GetSelfCredential (certificate_string, self.hrn, "user") @@ -316,10 +317,14 @@ class SfaClientBootstrap: raise IOError,"Missing %s file %s"%(kind,filename) return True - def assert_private_key (self): return self.assert_filename (self.private_key_filename(),"private key") - def assert_self_signed_cert (self): return self.assert_filename (self.self_signed_cert_filename(),"self-signed certificate") - def assert_my_credential (self): return self.assert_filename (self.my_credential_filename(),"user's credential") - def assert_my_gid (self): return self.assert_filename (self.my_gid_filename(),"user's GID") + def assert_private_key (self): + return self.assert_filename (self.private_key_filename(),"private key") + def assert_self_signed_cert (self): + return self.assert_filename (self.self_signed_cert_filename(),"self-signed certificate") + def assert_my_credential (self): + return self.assert_filename (self.my_credential_filename(),"user's credential") + def assert_my_gid (self): + return self.assert_filename (self.my_gid_filename(),"user's GID") # decorator to make up the other methods diff --git a/sfa/client/sfaserverproxy.py b/sfa/client/sfaserverproxy.py index b348126e..615a8b63 100644 --- a/sfa/client/sfaserverproxy.py +++ b/sfa/client/sfaserverproxy.py @@ -1,5 +1,9 @@ # XMLRPC-specific code for SFA Client +# starting with 2.7.9 we need to turn off server verification +import ssl +ssl_needs_unverified_context = hasattr(ssl, '_create_unverified_context') + import xmlrpclib from httplib import HTTPS, HTTPSConnection @@ -7,7 +11,7 @@ try: from sfa.util.sfalogging import logger except: import logging - logger=logging.getLogger('sfaserverproxy') + logger = logging.getLogger('sfaserverproxy') ## # ServerException, ExceptionUnmarshaller @@ -30,17 +34,11 @@ class ExceptionUnmarshaller(xmlrpclib.Unmarshaller): # # A transport for XMLRPC that works on top of HTTPS -# python 2.7 xmlrpclib has changed its internal code -# it now calls 'getresponse' on the obj returned by make_connection -# while it used to call 'getreply' -# regardless of the version, httplib.HTTPS does implement getreply, -# while httplib.HTTPSConnection has getresponse -# so we create a dummy instance to check what's expected -need_HTTPSConnection=hasattr(xmlrpclib.Transport().make_connection('localhost'),'getresponse') +# targetting only python-2.7 we can get rid of some older code class XMLRPCTransport(xmlrpclib.Transport): - def __init__(self, key_file=None, cert_file=None, timeout=None): + def __init__(self, key_file = None, cert_file = None, timeout = None): xmlrpclib.Transport.__init__(self) self.timeout=timeout self.key_file = key_file @@ -50,10 +48,13 @@ class XMLRPCTransport(xmlrpclib.Transport): # create a HTTPS connection object from a host descriptor # host may be a string, or a (host, x509-dict) tuple host, extra_headers, x509 = self.get_host_info(host) - if need_HTTPSConnection: - conn = HTTPSConnection(host, None, key_file=self.key_file, cert_file=self.cert_file) + if not ssl_needs_unverified_context: + conn = HTTPSConnection(host, None, key_file = self.key_file, + cert_file = self.cert_file) else: - conn = HTTPS(host, None, key_file=self.key_file, cert_file=self.cert_file) + conn = HTTPSConnection(host, None, key_file = self.key_file, + cert_file = self.cert_file, + context = ssl._create_unverified_context()) # Some logic to deal with timeouts. It appears that some (or all) versions # of python don't set the timeout after the socket is created. We'll do it @@ -84,21 +85,27 @@ class XMLRPCServerProxy(xmlrpclib.ServerProxy): # remember url for GetVersion # xxx not sure this is still needed as SfaServerProxy has this too self.url=url - xmlrpclib.ServerProxy.__init__(self, url, transport, allow_none=allow_none, verbose=verbose) + if not ssl_needs_unverified_context: + xmlrpclib.ServerProxy.__init__(self, url, transport, allow_none=allow_none, + verbose=verbose) + else: + xmlrpclib.ServerProxy.__init__(self, url, transport, allow_none=allow_none, + verbose=verbose, + context=ssl._create_unverified_context()) def __getattr__(self, attr): - logger.debug ("xml-rpc %s method:%s"%(self.url,attr)) + logger.debug ("xml-rpc %s method:%s" % (self.url, attr)) return xmlrpclib.ServerProxy.__getattr__(self, attr) ########## the object on which we can send methods that get sent over xmlrpc class SfaServerProxy: def __init__ (self, url, keyfile, certfile, verbose=False, timeout=None): - self.url=url - self.keyfile=keyfile - self.certfile=certfile - self.verbose=verbose - self.timeout=timeout + self.url = url + self.keyfile = keyfile + self.certfile = certfile + self.verbose = verbose + self.timeout = timeout # an instance of xmlrpclib.ServerProxy transport = XMLRPCTransport(keyfile, certfile, timeout) self.serverproxy = XMLRPCServerProxy(url, transport, allow_none=True, verbose=verbose) diff --git a/sfa/client/sfi.py b/sfa/client/sfi.py index dcad9dad..6f5f64df 100644 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@ -674,6 +674,8 @@ use this if you mean an authority instead""") # init self-signed cert, user credentials and gid def bootstrap (self): + if self.options.verbose: + self.logger.info("Initializing SfaClientBootstrap with {}".format(self.reg_url)) client_bootstrap = SfaClientBootstrap (self.user, self.reg_url, self.options.sfi_dir, logger=self.logger) # if -k is provided, use this to initialize private key diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py index 28766948..9ab8c6d5 100644 --- a/sfa/planetlab/plaggregate.py +++ b/sfa/planetlab/plaggregate.py @@ -18,7 +18,7 @@ from sfa.rspecs.elements.lease import Lease from sfa.rspecs.elements.granularity import Granularity from sfa.rspecs.version_manager import VersionManager -from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn, top_auth, hash_loginbase +from sfa.planetlab.plxrn import PlXrn, hostname_to_urn from sfa.planetlab.vlink import get_tc_rate from sfa.planetlab.topology import Topology from sfa.storage.model import SliverAllocation diff --git a/sfa/planetlab/pldriver.py b/sfa/planetlab/pldriver.py index 88c41fea..d16b23a6 100644 --- a/sfa/planetlab/pldriver.py +++ b/sfa/planetlab/pldriver.py @@ -23,7 +23,7 @@ from sfa.managers.driver import Driver from sfa.planetlab.plshell import PlShell from sfa.planetlab.plaggregate import PlAggregate from sfa.planetlab.plslices import PlSlices -from sfa.planetlab.plxrn import PlXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, xrn_to_hostname, top_auth, hash_loginbase +from sfa.planetlab.plxrn import PlXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, top_auth, hash_loginbase def list_to_dict(recs, key): diff --git a/sfa/server/threadedserver.py b/sfa/server/threadedserver.py index dbdde3f2..7dfac7d7 100644 --- a/sfa/server/threadedserver.py +++ b/sfa/server/threadedserver.py @@ -145,11 +145,13 @@ class SecureXMLRpcRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): class SecureXMLRPCServer(BaseHTTPServer.HTTPServer,SimpleXMLRPCServer.SimpleXMLRPCDispatcher): def __init__(self, server_address, HandlerClass, key_file, cert_file, logRequests=True): - """Secure XML-RPC server. + """ + Secure XML-RPC server. It it very similar to SimpleXMLRPCServer but it uses HTTPS for transporting XML data. """ - logger.debug("SecureXMLRPCServer.__init__, server_address=%s, cert_file=%s, key_file=%s"%(server_address,cert_file,key_file)) + logger.debug("SecureXMLRPCServer.__init__, server_address=%s, " + "cert_file=%s, key_file=%s"%(server_address,cert_file,key_file)) self.logRequests = logRequests self.interface = None self.key_file = key_file diff --git a/sfa/trust/auth.py b/sfa/trust/auth.py index 59ca4c27..2120a80d 100644 --- a/sfa/trust/auth.py +++ b/sfa/trust/auth.py @@ -4,9 +4,10 @@ import sys from types import StringTypes -from sfa.util.faults import InsufficientRights, MissingCallerGID, MissingTrustedRoots, PermissionError, \ - BadRequestHash, ConnectionKeyGIDMismatch, SfaPermissionDenied, CredentialNotVerifiable, Forbidden, \ - BadArgs +from sfa.util.faults import InsufficientRights, MissingCallerGID, \ + MissingTrustedRoots, PermissionError, BadRequestHash, \ + ConnectionKeyGIDMismatch, SfaPermissionDenied, CredentialNotVerifiable, \ + Forbidden, BadArgs from sfa.util.sfalogging import logger from sfa.util.config import Config from sfa.util.xrn import Xrn, get_authority @@ -34,10 +35,13 @@ class Auth: self.load_trusted_certs() def load_trusted_certs(self): - self.trusted_cert_list = TrustedRoots(self.config.get_trustedroots_dir()).get_list() - self.trusted_cert_file_list = TrustedRoots(self.config.get_trustedroots_dir()).get_file_list() + self.trusted_cert_list = \ + TrustedRoots(self.config.get_trustedroots_dir()).get_list() + self.trusted_cert_file_list = \ + TrustedRoots(self.config.get_trustedroots_dir()).get_file_list() - # this convenience methods extracts speaking_for_xrn from the passed options using 'geni_speaking_for' + # this convenience methods extracts speaking_for_xrn + # from the passed options using 'geni_speaking_for' def checkCredentialsSpeaksFor (self, *args, **kwds): if 'options' not in kwds: logger.error ("checkCredentialsSpeaksFor was not passed options=options") @@ -62,7 +66,8 @@ class Auth: error="checkCredentials: expected a string, received %s"%(type(cred)) else: cred_obj=Credential(string=cred) - logger.info("failed to validate credential - dump=%s"%cred_obj.dump_string(dump_parents=True)) + logger.info("failed to validate credential - dump=%s"%\ + cred_obj.dump_string(dump_parents=True)) error = sys.exc_info()[:2] return error @@ -76,7 +81,7 @@ class Auth: if not isinstance(xrns, list): xrns = [xrns] - slice_xrns = Xrn.filter_type(xrns, 'slice') + slice_xrns = Xrn.filter_type(xrns, 'slice') sliver_xrns = Xrn.filter_type(xrns, 'sliver') # we are not able to validate slivers in the traditional way so @@ -122,7 +127,7 @@ class Auth: def check(self, credential, operation, hrn = None): """ - Check the credential against the peer cert (callerGID included + Check the credential against the peer cert (callerGID) included in the credential matches the caller that is connected to the HTTPS connection, check if the credential was signed by a trusted cert and check if the credential is allowed to perform @@ -152,7 +157,8 @@ class Auth: raise InsufficientRights(operation) if self.trusted_cert_list: - self.client_cred.verify(self.trusted_cert_file_list, self.config.SFA_CREDENTIAL_SCHEMA) + self.client_cred.verify(self.trusted_cert_file_list, + self.config.SFA_CREDENTIAL_SCHEMA) else: raise MissingTrustedRoots(self.config.get_trustedroots_dir()) @@ -168,7 +174,7 @@ class Auth: def check_ticket(self, ticket): """ - Check if the tickt was signed by a trusted cert + Check if the ticket was signed by a trusted cert """ if self.trusted_cert_list: client_ticket = SfaTicket(string=ticket) @@ -315,7 +321,8 @@ class Auth: rl = Rights() type = reg_record.type - logger.debug("entering determine_user_rights with record %s and caller_hrn %s"%(reg_record, caller_hrn)) + logger.debug("entering determine_user_rights with record %s and caller_hrn %s"%\ + (reg_record, caller_hrn)) if type == 'slice': # researchers in the slice are in the DB as-is diff --git a/sfa/trust/credential.py b/sfa/trust/credential.py index 9d0fd283..109a5290 100644 --- a/sfa/trust/credential.py +++ b/sfa/trust/credential.py @@ -26,7 +26,8 @@ # Credentials are signed XML files that assign a subject gid privileges to an object gid ## -import os +import os, os.path +import subprocess from types import StringTypes import datetime from StringIO import StringIO @@ -512,7 +513,8 @@ class Credential(object): # Below throws InUse exception if we forgot to clone the attribute first oldAttr = signed_cred.setAttributeNode(attr.cloneNode(True)) if oldAttr and oldAttr.value != attr.value: - msg = "Delegating cred from owner %s to %s over %s:\n - Replaced attribute %s value '%s' with '%s'" % (self.parent.gidCaller.get_urn(), self.gidCaller.get_urn(), self.gidObject.get_urn(), oldAttr.name, oldAttr.value, attr.value) + msg = "Delegating cred from owner %s to %s over %s:\n - Replaced attribute %s value '%s' with '%s'" % \ + (self.parent.gidCaller.get_urn(), self.gidCaller.get_urn(), self.gidObject.get_urn(), oldAttr.name, oldAttr.value, attr.value) logger.warn(msg) #raise CredentialNotVerifiable("Can't encode new valid delegated credential: %s" % msg) @@ -798,12 +800,12 @@ class Credential(object): # make sure it is not expired if self.get_expiration() < datetime.datetime.utcnow(): - raise CredentialNotVerifiable("Credential %s expired at %s" % (self.get_summary_tostring(), self.expiration.strftime(SFATIME_FORMAT))) + raise CredentialNotVerifiable("Credential %s expired at %s" % \ + (self.get_summary_tostring(), + self.expiration.strftime(SFATIME_FORMAT))) # Verify the signatures filename = self.save_to_random_tmp_file() - if trusted_certs is not None: - cert_args = " ".join(['--trusted-pem %s' % x for x in trusted_certs]) # If caller explicitly passed in None that means skip cert chain validation. # - Strange and not typical @@ -826,11 +828,25 @@ class Credential(object): if trusted_certs is None: break -# print "Doing %s --verify --node-id '%s' %s %s 2>&1" % \ -# (self.xmlsec_path, ref, cert_args, filename) - verified = os.popen('%s --verify --node-id "%s" %s %s 2>&1' \ - % (self.xmlsec_path, ref, cert_args, filename)).read() - if not verified.strip().startswith("OK"): + # Thierry - jan 2015 + # up to fedora20 we used os.popen and checked that the output begins with OK + # turns out, with fedora21, there is extra input before this 'OK' thing + # looks like we're better off just using the exit code - that's what it is made for + #cert_args = " ".join(['--trusted-pem %s' % x for x in trusted_certs]) + #command = '{} --verify --node-id "{}" {} {} 2>&1'.\ + # format(self.xmlsec_path, ref, cert_args, filename) + command = [ self.xmlsec_path, '--verify', '--node-id', ref ] + for trusted in trusted_certs: + command += ["--trusted-pem", trusted ] + command += [ filename ] + logger.debug("Running " + " ".join(command)) + try: + verified = subprocess.check_output(command, stderr=subprocess.STDOUT) + logger.debug("xmlsec command returned {}".format(verified)) + if "OK\n" not in verified: + logger.warning("WARNING: xmlsec1 seemed to return fine but without a OK in its output") + except subprocess.CalledProcessError as e: + verified = e.output # xmlsec errors have a msg= which is the interesting bit. mstart = verified.find("msg=") msg = "" @@ -838,7 +854,9 @@ class Credential(object): mstart = mstart + 4 mend = verified.find('\\', mstart) msg = verified[mstart:mend] - raise CredentialNotVerifiable("xmlsec1 error verifying cred %s using Signature ID %s: %s %s" % (self.get_summary_tostring(), ref, msg, verified.strip())) + logger.warning("Credential.verify - failed - xmlsec1 returned {}".format(verified.strip())) + raise CredentialNotVerifiable("xmlsec1 error verifying cred %s using Signature ID %s: %s" % \ + (self.get_summary_tostring(), ref, msg)) os.remove(filename) # Verify the parents (delegation) @@ -936,7 +954,8 @@ class Credential(object): # Give up, credential does not pass issuer verification - raise CredentialNotVerifiable("Could not verify credential owned by %s for object %s. Cred signer %s not the trusted authority for Cred target %s" % (self.gidCaller.get_urn(), self.gidObject.get_urn(), root_cred_signer.get_hrn(), root_target_gid.get_hrn())) + raise CredentialNotVerifiable("Could not verify credential owned by %s for object %s. Cred signer %s not the trusted authority for Cred target %s" % \ + (self.gidCaller.get_urn(), self.gidObject.get_urn(), root_cred_signer.get_hrn(), root_target_gid.get_hrn())) ## @@ -951,22 +970,26 @@ class Credential(object): # parents rights (and check delegate bits) if not parent_cred.get_privileges().is_superset(self.get_privileges()): raise ChildRightsNotSubsetOfParent(("Parent cred ref %s rights " % parent_cred.get_refid()) + - self.parent.get_privileges().save_to_string() + (" not superset of delegated cred %s ref %s rights " % (self.get_summary_tostring(), self.get_refid())) + + self.parent.get_privileges().save_to_string() + (" not superset of delegated cred %s ref %s rights " % \ + (self.get_summary_tostring(), self.get_refid())) + self.get_privileges().save_to_string()) # make sure my target gid is the same as the parent's if not parent_cred.get_gid_object().save_to_string() == \ self.get_gid_object().save_to_string(): - raise CredentialNotVerifiable("Delegated cred %s: Target gid not equal between parent and child. Parent %s" % (self.get_summary_tostring(), parent_cred.get_summary_tostring())) + raise CredentialNotVerifiable("Delegated cred %s: Target gid not equal between parent and child. Parent %s" % \ + (self.get_summary_tostring(), parent_cred.get_summary_tostring())) # make sure my expiry time is <= my parent's if not parent_cred.get_expiration() >= self.get_expiration(): - raise CredentialNotVerifiable("Delegated credential %s expires after parent %s" % (self.get_summary_tostring(), parent_cred.get_summary_tostring())) + raise CredentialNotVerifiable("Delegated credential %s expires after parent %s" % \ + (self.get_summary_tostring(), parent_cred.get_summary_tostring())) # make sure my signer is the parent's caller if not parent_cred.get_gid_caller().save_to_string(False) == \ self.get_signature().get_issuer_gid().save_to_string(False): - raise CredentialNotVerifiable("Delegated credential %s not signed by parent %s's caller" % (self.get_summary_tostring(), parent_cred.get_summary_tostring())) + raise CredentialNotVerifiable("Delegated credential %s not signed by parent %s's caller" % \ + (self.get_summary_tostring(), parent_cred.get_summary_tostring())) # Recurse if parent_cred.parent: diff --git a/tools/reset_gids.py b/tools/reset_gids.py index 453af297..21e25cec 100755 --- a/tools/reset_gids.py +++ b/tools/reset_gids.py @@ -20,7 +20,7 @@ def fix_users(): uuid = create_uuid() pkey = Keypair(create=True) pub_key=getattr(record,'reg_keys',None) - if pub_key is not None: + if len(pub_key) > 0: # use only first key in record if pub_key and isinstance(pub_key, types.ListType): pub_key = pub_key[0] pub_key = pub_key.key