From: parmentelat Date: Wed, 23 May 2018 21:40:12 +0000 (+0200) Subject: more pep8-friendly X-Git-Tag: sfa-4.0-0~28 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=7eb34251548a271ae964b2f7d7e0fad7a1e41f5a more pep8-friendly --- diff --git a/sfa/client/manifolduploader.py b/sfa/client/manifolduploader.py index e377c8d7..69561e0d 100755 --- a/sfa/client/manifolduploader.py +++ b/sfa/client/manifolduploader.py @@ -169,13 +169,17 @@ def main(): help="the filenames to upload") parser.add_argument('-u', '--url', dest='url', action='store', default=None, help='the URL of the manifold API') - parser.add_argument('-p', '--platform', dest='platform', action='store', default=None, + parser.add_argument('-p', '--platform', dest='platform', + action='store', default=None, help='the manifold platform name') - parser.add_argument('-U', '--user', dest='username', action='store', default=None, + parser.add_argument('-U', '--user', dest='username', + action='store', default=None, help='the manifold username') - parser.add_argument('-P', '--password', dest='password', action='store', default=None, + parser.add_argument('-P', '--password', dest='password', + action='store', default=None, help='the manifold password') - parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0, + parser.add_argument('-v', '--verbose', dest='verbose', + action='count', default=0, help='more and more verbose') args = parser.parse_args() diff --git a/sfa/client/sfaadmin.py b/sfa/client/sfaadmin.py index a6d5a07c..d67734ed 100755 --- a/sfa/client/sfaadmin.py +++ b/sfa/client/sfaadmin.py @@ -1,10 +1,13 @@ #!/usr/bin/python + +# pylint: disable=c0111, c0103, w0402, w0622 + from __future__ import print_function import os import sys import copy -from pprint import pformat, PrettyPrinter +from pprint import PrettyPrinter from optparse import OptionParser from sfa.generic import Generic @@ -15,7 +18,9 @@ from sfa.trust.hierarchy import Hierarchy from sfa.trust.gid import GID from sfa.trust.certificate import convert_public_key -from sfa.client.common import optparse_listvalue_callback, optparse_dictvalue_callback, terminal_render, filter_records +from sfa.client.common import (optparse_listvalue_callback, + optparse_dictvalue_callback, + terminal_render, filter_records) from sfa.client.candidates import Candidates from sfa.client.sfi import save_records_to_file @@ -23,7 +28,7 @@ pprinter = PrettyPrinter(indent=4) try: help_basedir = Hierarchy().basedir -except: +except Exception: help_basedir = '*unable to locate Hierarchy().basedir' @@ -54,14 +59,19 @@ class RegistryCommands(Commands): version = self.api.manager.GetVersion(self.api, {}) pprinter.pprint(version) - - @add_options('-x', '--xrn', dest='xrn', metavar='', help='authority to list (hrn/urn - mandatory)') - @add_options('-t', '--type', dest='type', metavar='', help='object type', default='all') - @add_options('-r', '--recursive', dest='recursive', metavar='', help='list all child records', + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='authority to list (hrn/urn - mandatory)') + @add_options('-t', '--type', dest='type', metavar='', + help='object type', default='all') + @add_options('-r', '--recursive', dest='recursive', metavar='', + help='list all child records', + action='store_true', default=False) + @add_options('-v', '--verbose', dest='verbose', action='store_true', default=False) - @add_options('-v', '--verbose', dest='verbose', action='store_true', default=False) def list(self, xrn, type=None, recursive=False, verbose=False): - """List names registered at a given authority - possibly filtered by type""" + """ + List names registered at a given authority, possibly filtered by type + """ xrn = Xrn(xrn, type) options_dict = {'recursive': recursive} records = self.api.manager.List( @@ -69,18 +79,22 @@ class RegistryCommands(Commands): list = filter_records(type, records) # terminal_render expects an options object - class Options: - pass - options = Options() - options.verbose = verbose - terminal_render(list, options) + class Options: # pylint: disable=r0903 + def __init__(self, verbose): + self.verbose = verbose + options = Options(verbose) + terminal_render(list, options) - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') - @add_options('-t', '--type', dest='type', metavar='', help='object type', default=None) - @add_options('-o', '--outfile', dest='outfile', metavar='', help='save record to file') - @add_options('-f', '--format', dest='format', metavar='', type='choice', - choices=('text', 'xml', 'simple'), help='display record in different formats') + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn (mandatory)') + @add_options('-t', '--type', dest='type', metavar='', + help='object type', default=None) + @add_options('-o', '--outfile', dest='outfile', metavar='', + help='save record to file') + @add_options('-f', '--format', dest='format', metavar='', + type='choice', choices=('text', 'xml', 'simple'), + help='display record in different formats') def show(self, xrn, type=None, format=None, outfile=None): """Display details for a registered object""" records = self.api.manager.Resolve(self.api, xrn, type, details=True) @@ -90,7 +104,8 @@ class RegistryCommands(Commands): if outfile: save_records_to_file(outfile, records) - def _record_dict(self, xrn, type, email, key, + @staticmethod + def _record_dict(xrn, type, email, key, slices, researchers, pis, url, description, extras): record_dict = {} @@ -124,11 +139,16 @@ class RegistryCommands(Commands): record_dict.update(extras) return record_dict - - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn', default=None) - @add_options('-t', '--type', dest='type', metavar='', help='object type (mandatory)') - @add_options('-a', '--all', dest='all', metavar='', action='store_true', default=False, help='check all users GID') - @add_options('-v', '--verbose', dest='verbose', metavar='', action='store_true', default=False, help='verbose mode: display user\'s hrn ') + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn', default=None) + @add_options('-t', '--type', dest='type', metavar='', + help='object type (mandatory)') + @add_options('-a', '--all', dest='all', metavar='', + action='store_true', default=False, + help='check all users GID') + @add_options('-v', '--verbose', dest='verbose', metavar='', + action='store_true', default=False, + help='verbose mode: display user\'s hrn ') def check_gid(self, xrn=None, type=None, all=None, verbose=None): """Check the correspondance between the GID and the PubKey""" @@ -160,7 +180,7 @@ class RegistryCommands(Commands): db_pubkey_str = record.reg_keys[0].key try: db_pubkey_obj = convert_public_key(db_pubkey_str) - except: + except Exception: ERROR.append(record.hrn) continue else: @@ -183,81 +203,111 @@ class RegistryCommands(Commands): print("Users NOT having a PubKey: %s\n\ Users having a non RSA PubKey: %s\n\ Users having a GID/PubKey correpondence OK: %s\n\ -Users having a GID/PubKey correpondence Not OK: %s\n" % (len(NOKEY), len(ERROR), len(OK), len(NOK))) +Users having a GID/PubKey correpondence Not OK: %s\n" + % (len(NOKEY), len(ERROR), len(OK), len(NOK))) else: print("Users NOT having a PubKey: %s and are: \n%s\n\n\ Users having a non RSA PubKey: %s and are: \n%s\n\n\ Users having a GID/PubKey correpondence OK: %s and are: \n%s\n\n\ -Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n" % (len(NOKEY), NOKEY, len(ERROR), ERROR, len(OK), OK, len(NOK), NOK)) +Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n" + % (len(NOKEY), NOKEY, len(ERROR), ERROR, + len(OK), OK, len(NOK), NOK)) - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') - @add_options('-t', '--type', dest='type', metavar='', help='object type', default=None) + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn (mandatory)') + @add_options('-t', '--type', dest='type', metavar='', + help='object type', default=None) @add_options('-e', '--email', dest='email', default="", help="email (mandatory for users)") @add_options('-u', '--url', dest='url', metavar='', default=None, help="URL, useful for slices") - @add_options('-d', '--description', dest='description', metavar='', + @add_options('-d', '--description', dest='description', + metavar='', help='Description, useful for slices', default=None) - @add_options('-k', '--key', dest='key', metavar='', help='public key string or file', + @add_options('-k', '--key', dest='key', metavar='', + help='public key string or file', default=None) - @add_options('-s', '--slices', dest='slices', metavar='', help='Set/replace slice xrns', - default='', type="str", action='callback', callback=optparse_listvalue_callback) - @add_options('-r', '--researchers', dest='researchers', metavar='', help='Set/replace slice researchers', - default='', type="str", action='callback', callback=optparse_listvalue_callback) + @add_options('-s', '--slices', dest='slices', metavar='', + help='Set/replace slice xrns', + default='', type="str", action='callback', + callback=optparse_listvalue_callback) + @add_options('-r', '--researchers', dest='researchers', + metavar='', help='Set/replace slice researchers', + default='', type="str", action='callback', + callback=optparse_listvalue_callback) @add_options('-p', '--pis', dest='pis', metavar='', help='Set/replace Principal Investigators/Project Managers', - default='', type="str", action='callback', callback=optparse_listvalue_callback) - @add_options('-X', '--extra', dest='extras', default={}, type='str', metavar="", - action="callback", callback=optparse_dictvalue_callback, nargs=1, - help="set extra/testbed-dependent flags, e.g. --extra enabled=true") + default='', type="str", action='callback', + callback=optparse_listvalue_callback) + @add_options('-X', '--extra', dest='extras', + default={}, type='str', metavar="", + action="callback", callback=optparse_dictvalue_callback, + nargs=1, + help="set extra/testbed-dependent flags," + " e.g. --extra enabled=true") def register(self, xrn, type=None, email='', key=None, slices='', pis='', researchers='', url=None, description=None, extras={}): """Create a new Registry record""" - record_dict = self._record_dict(xrn=xrn, type=type, email=email, key=key, - slices=slices, researchers=researchers, pis=pis, - url=url, description=description, extras=extras) + record_dict = self._record_dict( + xrn=xrn, type=type, email=email, key=key, + slices=slices, researchers=researchers, pis=pis, + url=url, description=description, extras=extras) self.api.manager.Register(self.api, record_dict) - - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') - @add_options('-t', '--type', dest='type', metavar='', help='object type', default=None) - @add_options('-u', '--url', dest='url', metavar='', help='URL', default=None) - @add_options('-d', '--description', dest='description', metavar='', + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn (mandatory)') + @add_options('-t', '--type', dest='type', metavar='', + help='object type', default=None) + @add_options('-u', '--url', dest='url', metavar='', + help='URL', default=None) + @add_options('-d', '--description', dest='description', + metavar='', help='Description', default=None) - @add_options('-k', '--key', dest='key', metavar='', help='public key string or file', + @add_options('-k', '--key', dest='key', metavar='', + help='public key string or file', default=None) - @add_options('-s', '--slices', dest='slices', metavar='', help='Set/replace slice xrns', - default='', type="str", action='callback', callback=optparse_listvalue_callback) - @add_options('-r', '--researchers', dest='researchers', metavar='', help='Set/replace slice researchers', - default='', type="str", action='callback', callback=optparse_listvalue_callback) + @add_options('-s', '--slices', dest='slices', metavar='', + help='Set/replace slice xrns', + default='', type="str", action='callback', + callback=optparse_listvalue_callback) + @add_options('-r', '--researchers', dest='researchers', + metavar='', help='Set/replace slice researchers', + default='', type="str", action='callback', + callback=optparse_listvalue_callback) @add_options('-p', '--pis', dest='pis', metavar='', help='Set/replace Principal Investigators/Project Managers', - default='', type="str", action='callback', callback=optparse_listvalue_callback) - @add_options('-X', '--extra', dest='extras', default={}, type='str', metavar="", - action="callback", callback=optparse_dictvalue_callback, nargs=1, - help="set extra/testbed-dependent flags, e.g. --extra enabled=true") + default='', type="str", action='callback', + callback=optparse_listvalue_callback) + @add_options('-X', '--extra', dest='extras', default={}, type='str', + metavar="", nargs=1, + action="callback", callback=optparse_dictvalue_callback, + help="set extra/testbed-dependent flags," + " e.g. --extra enabled=true") def update(self, xrn, type=None, email='', key=None, slices='', pis='', researchers='', url=None, description=None, extras={}): """Update an existing Registry record""" - record_dict = self._record_dict(xrn=xrn, type=type, email=email, key=key, - slices=slices, researchers=researchers, pis=pis, - url=url, description=description, extras=extras) + record_dict = self._record_dict( + xrn=xrn, type=type, email=email, key=key, + slices=slices, researchers=researchers, pis=pis, + url=url, description=description, extras=extras) self.api.manager.Update(self.api, record_dict) - - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') - @add_options('-t', '--type', dest='type', metavar='', help='object type', default=None) + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn (mandatory)') + @add_options('-t', '--type', dest='type', metavar='', + help='object type', default=None) def remove(self, xrn, type=None): """Remove given object from the registry""" xrn = Xrn(xrn, type) self.api.manager.Remove(self.api, xrn) - - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') - @add_options('-t', '--type', dest='type', metavar='', help='object type', default=None) + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn (mandatory)') + @add_options('-t', '--type', dest='type', metavar='', + help='object type', default=None) def credential(self, xrn, type=None): """Invoke GetCredential""" cred = self.api.manager.GetCredential( @@ -279,14 +329,23 @@ Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n" % (len(NOK dbschema.init_or_upgrade() - @add_options('-a', '--all', dest='all', metavar='', action='store_true', default=False, - help='Remove all registry records and all files in %s area' % help_basedir) - @add_options('-c', '--certs', dest='certs', metavar='', action='store_true', default=False, - help='Remove all cached certs/gids found in %s' % help_basedir) - @add_options('-0', '--no-reinit', dest='reinit', metavar='', action='store_false', default=True, - help='Prevents new DB schema from being installed after cleanup') + @add_options('-a', '--all', dest='all', metavar='', + action='store_true', default=False, + help='Remove all registry records and all files in %s area' + % help_basedir) + @add_options('-c', '--certs', dest='certs', + metavar='', action='store_true', default=False, + help='Remove all cached certs/gids found in %s' + % help_basedir) + @add_options('-0', '--no-reinit', dest='reinit', metavar='', + action='store_false', default=True, + help="Prevents new DB schema" + " from being installed after cleanup") def nuke(self, all=False, certs=False, reinit=True): - """Cleanup local registry DB, plus various additional filesystem cleanups optionally""" + """ + Cleanup local registry DB, plus various additional + filesystem cleanups optionally + """ from sfa.storage.dbschema import DBSchema from sfa.util.sfalogging import _SfaLogger logger = _SfaLogger( @@ -296,7 +355,8 @@ Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n" % (len(NOK dbschema = DBSchema() dbschema.nuke() - # for convenience we re-create the schema here, so there's no need for an explicit + # for convenience we re-create the schema here, + # so there's no need for an explicit # service sfa restart # however in some (upgrade) scenarios this might be wrong if reinit: @@ -333,9 +393,12 @@ class CertCommands(Commands): def import_gid(self, xrn): pass - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') - @add_options('-t', '--type', dest='type', metavar='', help='object type', default=None) - @add_options('-o', '--outfile', dest='outfile', metavar='', help='output file', default=None) + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn (mandatory)') + @add_options('-t', '--type', dest='type', metavar='', + help='object type', default=None) + @add_options('-o', '--outfile', dest='outfile', metavar='', + help='output file', default=None) def export(self, xrn, type=None, outfile=None): """Fetch an object's GID from the Registry""" from sfa.storage.model import RegRecord @@ -352,7 +415,7 @@ class CertCommands(Commands): try: auth_info = hierarchy.get_auth_info(hrn) gid = auth_info.gid_object - except: + except Exception: print("Record: %s not found" % hrn) sys.exit(1) # save to file @@ -360,7 +423,8 @@ class CertCommands(Commands): outfile = os.path.abspath('./%s.gid' % gid.get_hrn()) gid.save_to_file(outfile, save_parents=True) - @add_options('-g', '--gidfile', dest='gid', metavar='', help='path of gid file to display (mandatory)') + @add_options('-g', '--gidfile', dest='gid', metavar='', + help='path of gid file to display (mandatory)') def display(self, gidfile): """Print contents of a GID file""" gid_path = os.path.abspath(gidfile) @@ -381,15 +445,20 @@ class AggregateCommands(Commands): version = self.api.manager.GetVersion(self.api, {}) pprinter.pprint(version) - @add_options('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='object hrn/urn (mandatory)') def status(self, xrn): - """Retrieve the status of the slivers belonging to the named slice (Status)""" + """ + Retrieve the status of the slivers + belonging to the named slice (Status) + """ urns = [Xrn(xrn, 'slice').get_urn()] status = self.api.manager.Status(self.api, urns, [], {}) pprinter.pprint(status) - @add_options('-r', '--rspec-version', dest='rspec_version', metavar='', - default='GENI', help='version/format of the resulting rspec response') + @add_options('-r', '--rspec-version', dest='rspec_version', + metavar='', default='GENI', + help='version/format of the resulting rspec response') def resources(self, rspec_version='GENI'): """Display the available resources at an aggregate""" options = {'geni_rspec_version': rspec_version} @@ -397,8 +466,10 @@ class AggregateCommands(Commands): resources = self.api.manager.ListResources(self.api, [], options) print(resources) - @add_options('-x', '--xrn', dest='xrn', metavar='', help='slice hrn/urn (mandatory)') - @add_options('-r', '--rspec', dest='rspec', metavar='', help='rspec file (mandatory)') + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='slice hrn/urn (mandatory)') + @add_options('-r', '--rspec', dest='rspec', metavar='', + help='rspec file (mandatory)') def allocate(self, xrn, rspec): """Allocate slivers""" xrn = Xrn(xrn, 'slice') @@ -409,7 +480,8 @@ class AggregateCommands(Commands): self.api, slice_urn, [], rspec_string, options) print(manifest) - @add_options('-x', '--xrn', dest='xrn', metavar='', help='slice hrn/urn (mandatory)') + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='slice hrn/urn (mandatory)') def provision(self, xrn): """Provision slivers""" xrn = Xrn(xrn, 'slice') @@ -419,7 +491,8 @@ class AggregateCommands(Commands): self.api, [slice_urn], [], options) print(manifest) - @add_options('-x', '--xrn', dest='xrn', metavar='', help='slice hrn/urn (mandatory)') + @add_options('-x', '--xrn', dest='xrn', metavar='', + help='slice hrn/urn (mandatory)') def delete(self, xrn): """Delete slivers""" self.api.manager.Delete(self.api, [xrn], [], {}) @@ -461,7 +534,6 @@ class SfaAdmin: if name.startswith('_'): continue margin = 15 - format = "%%-%ds" % margin print("%-15s" % name, end=' ') doc = getattr(method, '__doc__', None) if not doc: @@ -526,8 +598,6 @@ class SfaAdmin: sys.exit(0) except TypeError: print("Possible wrong number of arguments supplied") - #import traceback - # traceback.print_exc() print(command.__doc__) parser.print_help() sys.exit(1) diff --git a/sfa/client/sfi.py b/sfa/client/sfi.py index b94dcf62..f03634ed 100644 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@ -572,7 +572,7 @@ class Sfi: parser.add_option("-p", "--pi", dest='delegate_pi', default=None, action='store_true', help="delegate your PI credentials, so s.t. like -A your_hrn^") parser.add_option("-A", "--to-authority", dest='delegate_to_authority', action='store_true', default=False, - help="""by default the mandatory argument is expected to be a user, + help="""by default the mandatory argument is expected to be a user, use this if you mean an authority instead""") if canonical in ("myslice"): @@ -903,7 +903,7 @@ use this if you mean an authority instead""") # resurrect this temporarily so we can support V1 aggregates for a while def server_supports_options_arg(self, server): """ - Returns true if server support the optional call_id arg, false otherwise. + Returns true if server support the optional call_id arg, false otherwise. """ server_version = self.get_cached_server_version(server) result = False @@ -1127,8 +1127,8 @@ use this if you mean an authority instead""") @declare_command("[xml-filename]", "", ['add']) def register(self, options, args): """ - create new record in registry (Register) - from command line options (recommended) + create new record in registry (Register) + from command line options (recommended) old-school method involving an xml file still supported """ if len(args) > 1: @@ -1170,8 +1170,8 @@ use this if you mean an authority instead""") @declare_command("[xml-filename]", "") def update(self, options, args): """ - update record into registry (Update) - from command line options (recommended) + update record into registry (Update) + from command line options (recommended) old-school method involving an xml file still supported """ if len(args) > 1: @@ -1302,8 +1302,8 @@ use this if you mean an authority instead""") @declare_command("slice_hrn", "") def describe(self, options, args): """ - shows currently allocated/provisioned resources - of the named slice or set of slivers (Describe) + shows currently allocated/provisioned resources + of the named slice or set of slivers (Describe) """ if len(args) != 1: self.print_help() @@ -1791,7 +1791,7 @@ $ sfi m -b http://mymanifold.foo.com:7080/ * compute all the slices that you currently have credentials on * refresh all your credentials (you as a user and pi, your slices) * upload them to the manifold backend server - for last phase, sfi_config is read to look for the [myslice] section, + for last phase, sfi_config is read to look for the [myslice] section, and namely the 'backend', 'delegate' and 'user' settings """ diff --git a/sfa/generic/__init__.py b/sfa/generic/__init__.py index a7a0741a..bd37cf49 100644 --- a/sfa/generic/__init__.py +++ b/sfa/generic/__init__.py @@ -35,7 +35,7 @@ class Generic: if flavour is None: flavour = config.SFA_GENERIC_FLAVOUR flavour = flavour.lower() - #mixed = flavour.capitalize() + # mixed = flavour.capitalize() module_path = "sfa.generic.%s" % flavour classname = "%s" % flavour logger.debug("Generic.the_flavour with flavour=%s" % flavour) @@ -97,8 +97,9 @@ class Generic: try: module_or_class = getattr(self, classname)() logger.debug("%s : %s" % (message, module_or_class)) - # this gets passed to ManagerWrapper that will call the class constructor - # if it's a class, or use the module as is if it's a module + # this gets passed to ManagerWrapper that will + # call the class constructor if it's a class, + # or use the module as is if it's a module # so bottom line is, don't try the constructor here return module_or_class except: diff --git a/sfa/importer/__init__.py b/sfa/importer/__init__.py index 65e57ec0..84513e28 100644 --- a/sfa/importer/__init__.py +++ b/sfa/importer/__init__.py @@ -1,6 +1,7 @@ #!/usr/bin/python -import sys +# pylint: disable=c0111, w1201, w0622 + from datetime import datetime from sfa.util.xrn import get_authority, hrn_to_urn @@ -8,13 +9,12 @@ from sfa.generic import Generic from sfa.util.config import Config from sfa.util.sfalogging import _SfaLogger from sfa.trust.hierarchy import Hierarchy -#from sfa.trust.trustedroots import TrustedRoots +# from sfa.trust.trustedroots import TrustedRoots from sfa.trust.gid import create_uuid # using global alchemy.session() here is fine # as importer is on standalone one-shot process from sfa.storage.alchemy import global_dbsession from sfa.storage.model import RegRecord, RegAuthority, RegUser -from sfa.trust.certificate import convert_public_key, Keypair class Importer: @@ -37,12 +37,15 @@ class Importer: # self.TrustedRoots = TrustedRoots(self.config.get_trustedroots_dir()) # check before creating a RegRecord entry as we run this over and over - def record_exists(self, type, hrn): - return global_dbsession.query(RegRecord).filter_by(hrn=hrn, type=type).count() != 0 + @staticmethod + def record_exists(type, hrn): + return (global_dbsession.query(RegRecord) + .filter_by(hrn=hrn, type=type).count() != 0) def create_top_level_auth_records(self, hrn): """ - Create top level db records (includes root and sub authorities (local/remote) + Create top level db records + includes root and sub authorities (local/remote) """ # make sure parent exists parent_hrn = get_authority(hrn) @@ -68,7 +71,7 @@ class Importer: """ Create a user record for the Slicemanager service. """ - hrn = self.interface_hrn + '.slicemanager' + hrn = self.interface_hrn + '.slicemanager' # pylint: disable=e1101 urn = hrn_to_urn(hrn, 'user') if not self.auth_hierarchy.auth_exists(urn): self.logger.info("SfaImporter: creating Slice Manager user") @@ -126,8 +129,9 @@ class Importer: if importer_class: begin_time = datetime.utcnow() self.logger.info(30 * '=') - self.logger.info("Starting import on %s, using class %s from flavour %s" % - (begin_time, importer_class.__name__, generic.flavour)) + self.logger.info( + "Starting import on %s, using class %s from flavour %s" % + (begin_time, importer_class.__name__, generic.flavour)) testbed_importer = importer_class(auth_hierarchy, self.logger) if testbed_importer: testbed_importer.add_options(options) diff --git a/sfa/importer/dummyimporter.py b/sfa/importer/dummyimporter.py index 4594ad6e..6a5bb67c 100644 --- a/sfa/importer/dummyimporter.py +++ b/sfa/importer/dummyimporter.py @@ -245,13 +245,13 @@ class DummyImporter: except: continue if not pkey: - self.logger.warn( + self.logger.warning( 'DummyImporter: unable to convert public key for %s' % user_hrn) pkey = Keypair(create=True) else: # the user has no keys. Creating a random keypair for # the user's gid - self.logger.warn( + self.logger.warning( "DummyImporter: user %s does not have a NITOS public key" % user_hrn) pkey = Keypair(create=True) return (pubkey, pkey) diff --git a/sfa/importer/openstackimporter.py b/sfa/importer/openstackimporter.py index eb531738..7ced6203 100644 --- a/sfa/importer/openstackimporter.py +++ b/sfa/importer/openstackimporter.py @@ -79,7 +79,7 @@ class OpenstackImporter: 'unable to convert public key for %s' % hrn) pkey = Keypair(create=True) else: - self.logger.warn( + self.logger.warning( "OpenstackImporter: person %s does not have a PL public key" % hrn) pkey = Keypair(create=True) user_gid = self.auth_hierarchy.create_gid( diff --git a/sfa/importer/plimporter.py b/sfa/importer/plimporter.py index 3bbd8c30..f612f47e 100644 --- a/sfa/importer/plimporter.py +++ b/sfa/importer/plimporter.py @@ -307,8 +307,9 @@ class PlImporter: elif person_id in disabled_person_ids: pass else: - self.logger.warning("PlImporter: cannot locate person_id {} in site {} - ignored" - .format(person_id, site_hrn)) + self.logger.warning( + "PlImporter: cannot locate person_id {} in site {} - ignored" + .format(person_id, site_hrn)) # make sure to NOT run this if anything is wrong if not proceed: continue @@ -316,7 +317,7 @@ class PlImporter: #person_hrn = email_to_hrn(site_hrn, person['email']) person_hrn = person['hrn'] if person_hrn is None: - self.logger.warn( + self.logger.warning( "Person {} has no hrn - skipped".format(person['email'])) continue # xxx suspicious again @@ -336,14 +337,16 @@ class PlImporter: try: pkey = convert_public_key(pubkey['key']) except: - self.logger.warn('PlImporter: unable to convert public key for {}' - .format(person_hrn)) + self.logger.warning( + 'PlImporter: unable to convert public key for {}' + .format(person_hrn)) pkey = Keypair(create=True) else: # the user has no keys. Creating a random keypair for # the user's gid - self.logger.warn("PlImporter: person {} does not have a PL public key" - .format(person_hrn)) + self.logger.warning( + "PlImporter: person {} does not have a PL public key" + .format(person_hrn)) pkey = Keypair(create=True) return (pubkey, pkey) diff --git a/sfa/planetlab/plslices.py b/sfa/planetlab/plslices.py index 635001ce..e868acd1 100644 --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@ -7,7 +7,8 @@ from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn from sfa.rspecs.rspec import RSpec from sfa.planetlab.vlink import VLink from sfa.planetlab.topology import Topology -from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename, xrn_to_hostname, top_auth, hash_loginbase +from sfa.planetlab.plxrn import (PlXrn, hrn_to_pl_slicename, xrn_to_hostname, + top_auth, hash_loginbase) from sfa.storage.model import SliverAllocation MAXINT = 2L**31 - 1 @@ -41,8 +42,9 @@ class PlSlices: person_ids = list(person_ids) all_slice_tag_ids = list(all_slice_tag_ids) # Get user information - all_persons_list = self.driver.shell.GetPersons({'person_id': person_ids, 'enabled': True}, - ['person_id', 'enabled', 'key_ids']) + all_persons_list = self.driver.shell.GetPersons( + {'person_id': person_ids, 'enabled': True}, + ['person_id', 'enabled', 'key_ids']) all_persons = {} for person in all_persons_list: all_persons[person['person_id']] = person @@ -90,13 +92,17 @@ class PlSlices: sliver_attributes = [] if node is not None: - for sliver_attribute in filter(lambda a: a['node_id'] == node['node_id'], slice_tags): + for sliver_attribute in filter( + lambda a: a['node_id'] == node['node_id'], + slice_tags): sliver_attributes.append(sliver_attribute['tagname']) attributes.append({'tagname': sliver_attribute['tagname'], 'value': sliver_attribute['value']}) # set nodegroup slice attributes - for slice_tag in filter(lambda a: a['nodegroup_id'] in node['nodegroup_ids'], slice_tags): + for slice_tag in filter( + lambda a: a['nodegroup_id'] in node['nodegroup_ids'], + slice_tags): # Do not set any nodegroup slice attributes for # which there is at least one sliver attribute # already set. @@ -104,7 +110,9 @@ class PlSlices: attributes.append({'tagname': slice_tag['tagname'], 'value': slice_tag['value']}) - for slice_tag in filter(lambda a: a['node_id'] is None, slice_tags): + for slice_tag in filter( + lambda a: a['node_id'] is None, + slice_tags): # Do not set any global slice attributes for # which there is at least one sliver attribute # already set. @@ -144,8 +152,9 @@ class PlSlices: def verify_slice_leases(self, slice, rspec_requested_leases): - leases = self.driver.shell.GetLeases({'name': slice['name'], 'clip': int(time.time())}, - ['lease_id', 'name', 'hostname', 't_from', 't_until']) + leases = self.driver.shell.GetLeases( + {'name': slice['name'], 'clip': int(time.time())}, + ['lease_id', 'name', 'hostname', 't_from', 't_until']) grain = self.driver.shell.GetLeaseGranularity() requested_leases = [] @@ -165,7 +174,8 @@ class PlSlices: if slice_name != slice['name']: continue - elif Xrn(lease['component_id']).get_authority_urn().split(':')[0] != self.driver.hrn: + elif (Xrn(lease['component_id']).get_authority_urn().split(':')[0] + != self.driver.hrn): continue hostname = xrn_to_hostname(lease['component_id']) @@ -180,8 +190,9 @@ class PlSlices: # prepare actual slice leases by lease_id leases_by_id = {} for lease in leases: - leases_by_id[lease['lease_id']] = {'name': lease['name'], 'hostname': lease['hostname'], - 't_from': lease['t_from'], 't_until': lease['t_until']} + leases_by_id[lease['lease_id']] = { + 'name': lease['name'], 'hostname': lease['hostname'], + 't_from': lease['t_from'], 't_until': lease['t_until']} added_leases = [] kept_leases_id = [] @@ -197,10 +208,11 @@ class PlSlices: try: self.driver.shell.DeleteLeases(deleted_leases_id) for lease in added_leases: - self.driver.shell.AddLeases(lease['hostname'], slice['name'], lease[ - 't_from'], lease['t_until']) + self.driver.shell.AddLeases( + lease['hostname'], slice['name'], + lease['t_from'], lease['t_until']) - except: + except Exception: logger.log_exc('Failed to add/remove slice leases') return leases @@ -235,7 +247,7 @@ class PlSlices: self.driver.shell.DeleteSliceFromNodes( slice['name'], deleted_nodes) - except: + except Exception: logger.log_exc('Failed to add/remove slice from nodes') slices = self.driver.shell.GetSlices(slice['name'], ['node_ids']) @@ -329,7 +341,8 @@ class PlSlices: self.verify_slice_tags(slice, slice_tags, { 'pltags': 'append'}, admin=True) - def verify_site(self, slice_xrn, slice_record=None, sfa_peer=None, options=None): + def verify_site(self, slice_xrn, + slice_record=None, sfa_peer=None, options=None): if slice_record is None: slice_record = {} if options is None: @@ -343,8 +356,9 @@ class PlSlices: login_base = hash_loginbase(site_hrn) # filter sites by hrn - sites = self.driver.shell.GetSites({'peer_id': None, 'hrn': site_hrn}, - ['site_id', 'name', 'abbreviated_name', 'login_base', 'hrn']) + sites = self.driver.shell.GetSites( + {'peer_id': None, 'hrn': site_hrn}, + ['site_id', 'name', 'abbreviated_name', 'login_base', 'hrn']) # alredy exists if sites: @@ -371,7 +385,8 @@ class PlSlices: return site - def verify_slice(self, slice_hrn, slice_record, sfa_peer, expiration, options=None): + def verify_slice(self, slice_hrn, slice_record, + sfa_peer, expiration, options=None): if options is None: options = {} top_auth_hrn = top_auth(slice_hrn) @@ -385,8 +400,9 @@ class PlSlices: expires = int(datetime_to_epoch(utcparse(expiration))) # Filter slices by HRN - slices = self.driver.shell.GetSlices({'peer_id': None, 'hrn': slice_hrn}, - ['slice_id', 'name', 'hrn', 'expires']) + slices = self.driver.shell.GetSlices( + {'peer_id': None, 'hrn': slice_hrn}, + ['slice_id', 'name', 'hrn', 'expires']) if slices: slice = slices[0] @@ -469,7 +485,8 @@ class PlSlices: return person_id - def verify_persons(self, slice_hrn, slice_record, users, sfa_peer, options=None): + def verify_persons(self, slice_hrn, slice_record, + users, sfa_peer, options=None): if options is None: options = {} @@ -501,8 +518,9 @@ class PlSlices: site_id = site['site_id'] # locate the slice object - slice = self.driver.shell.GetSlices({'peer_id': None, 'hrn': slice_hrn}, [ - 'slice_id', 'hrn', 'person_ids'])[0] + slice = self.driver.shell.GetSlices( + {'peer_id': None, 'hrn': slice_hrn}, + ['slice_id', 'hrn', 'person_ids'])[0] slice_id = slice['slice_id'] slice_person_ids = slice['person_ids'] @@ -538,10 +556,12 @@ class PlSlices: # and for this we need all the Person objects; we already have the target_existing ones # also we avoid issuing a call if possible target_created_persons = [] if not target_created_person_ids \ - else self.driver.shell.GetPersons \ - ({'peer_id': None, 'person_id': target_created_person_ids}, person_fields) - persons_by_person_id = {person['person_id']: person - for person in target_existing_persons + target_created_persons} + else self.driver.shell.GetPersons( + {'peer_id': None, 'person_id': target_created_person_ids}, + person_fields) + persons_by_person_id = { + person['person_id']: person + for person in target_existing_persons + target_created_persons} def user_by_person_id(person_id): person = persons_by_person_id[person_id] @@ -560,7 +580,8 @@ class PlSlices: # return hrns of the newly added persons - return [persons_by_person_id[person_id]['hrn'] for person_id in add_person_ids] + return [persons_by_person_id[person_id]['hrn'] + for person_id in add_person_ids] def verify_keys(self, persons_to_verify_keys, options=None): if options is None: @@ -578,13 +599,14 @@ class PlSlices: key = {'key': key_string, 'key_type': 'ssh'} self.driver.shell.AddPersonKey(int(person_id), key) - def verify_slice_tags(self, slice, requested_slice_attributes, options=None, admin=False): + def verify_slice_tags(self, slice, requested_slice_attributes, + options=None, admin=False): """ This function deals with slice tags, and supports 3 modes described in the 'pltags' option that can be either (*) 'ignore' (default) - do nothing (*) 'append' - only add incoming tags, that do not match an existing tag - (*) 'sync' - tries to do the plain wholesale thing, + (*) 'sync' - tries to do the plain wholesale thing, i.e. to leave the db in sync with incoming tags """ if options is None: @@ -634,8 +656,8 @@ class PlSlices: # be removed tag_found = False for requested_attribute in requested_slice_attributes: - if requested_attribute['name'] == slice_tag['tagname'] and \ - requested_attribute['value'] == slice_tag['value']: + if (requested_attribute['name'] == slice_tag['tagname'] and + requested_attribute['value'] == slice_tag['value']): tag_found = True break # remove tags only if not in append mode @@ -648,8 +670,8 @@ class PlSlices: if requested_attribute['name'] in valid_tag_names: tag_found = False for existing_attribute in existing_slice_tags: - if requested_attribute['name'] == existing_attribute['tagname'] and \ - requested_attribute['value'] == existing_attribute['value']: + if (requested_attribute['name'] == existing_attribute['tagname'] and \ + requested_attribute['value'] == existing_attribute['value']): tag_found = True break if not tag_found: @@ -659,7 +681,8 @@ class PlSlices: name = tag_or_att[ 'tagname'] if 'tagname' in tag_or_att else tag_or_att['name'] return "SliceTag slice={}, tagname={} value={}, node_id={}"\ - .format(slice['name'], tag_or_att['name'], tag_or_att['value'], tag_or_att.get('node_id')) + .format(slice['name'], tag_or_att['name'], + tag_or_att['value'], tag_or_att.get('node_id')) # remove stale tags for tag in slice_tags_to_remove: @@ -668,16 +691,17 @@ class PlSlices: friendly_message(tag))) self.driver.shell.DeleteSliceTag(tag['slice_tag_id']) except Exception as e: - logger.warn("Failed to remove slice tag {}\nCause:{}" - .format(friendly_message(tag), e)) + logger.warning("Failed to remove slice tag {}\nCause:{}" + .format(friendly_message(tag), e)) # add requested_tags for attribute in slice_attributes_to_add: try: logger.info("Adding Slice Tag {}".format( friendly_message(attribute))) - self.driver.shell.AddSliceTag(slice['name'], attribute['name'], - attribute['value'], attribute.get('node_id', None)) + self.driver.shell.AddSliceTag( + slice['name'], attribute['name'], + attribute['value'], attribute.get('node_id', None)) except Exception as e: - logger.warn("Failed to add slice tag {}\nCause:{}" - .format(friendly_message(attribute), e)) + logger.warning("Failed to add slice tag {}\nCause:{}" + .format(friendly_message(attribute), e)) diff --git a/sfa/server/sfa-start.py b/sfa/server/sfa-start.py index bcd00cd6..7ca1af56 100755 --- a/sfa/server/sfa-start.py +++ b/sfa/server/sfa-start.py @@ -66,14 +66,15 @@ def daemon(): # when installed in standalone we might not have httpd installed if not os.path.isdir(logdir): os.mkdir('/var/log/httpd') - crashlog = os.open('%s/sfa_access_log' % logdir, os.O_RDWR | os.O_APPEND | os.O_CREAT, 0644) + crashlog = os.open('%s/sfa_access_log' % logdir, + os.O_RDWR | os.O_APPEND | os.O_CREAT, 0644) os.dup2(crashlog, 1) os.dup2(crashlog, 2) def install_peer_certs(server_key_file, server_cert_file): """ - Attempt to install missing trusted gids and db records for + Attempt to install missing trusted gids and db records for our federated interfaces """ # Attempt to get any missing peer gids @@ -130,7 +131,7 @@ def install_peer_certs(server_key_file, server_cert_file): message = "installed trusted cert for %s" % new_hrn # log the message api.logger.info(message) - except: + except Exception: message = "interface: %s\tunable to install trusted gid for %s" % \ (api.interface, new_hrn) api.logger.log_exc(message) @@ -140,7 +141,7 @@ def install_peer_certs(server_key_file, server_cert_file): def update_cert_records(gids): """ - Make sure there is a record in the registry for the specified gids. + Make sure there is a record in the registry for the specified gids. Removes old records from the db. """ # import db stuff here here so this module can be loaded by PlcComponentApi @@ -167,11 +168,12 @@ def update_cert_records(gids): record = dbsession.query(RegRecord).filter_by( hrn=hrn, type=type, pointer=-1).first() if not record: - record = RegRecord(dict={'type': type, - 'hrn': hrn, - 'authority': get_authority(hrn), - 'gid': gid.save_to_string(save_parents=True), - }) + record = RegRecord( + dict={'type': type, + 'hrn': hrn, + 'authority': get_authority(hrn), + 'gid': gid.save_to_string(save_parents=True), + }) dbsession.add(record) dbsession.commit() @@ -187,7 +189,8 @@ def main(): help="run aggregate manager", default=False) parser.add_option("-c", "--component", dest="cm", action="store_true", help="run component server", default=False) - parser.add_option("-t", "--trusted-certs", dest="trusted_certs", action="store_true", + parser.add_option("-t", "--trusted-certs", + dest="trusted_certs", action="store_true", help="refresh trusted certs", default=False) parser.add_option("-d", "--daemon", dest="daemon", action="store_true", help="Run as daemon.", default=False) diff --git a/sfa/server/sfaapi.py b/sfa/server/sfaapi.py index a2950762..231fdeb2 100644 --- a/sfa/server/sfaapi.py +++ b/sfa/server/sfaapi.py @@ -1,8 +1,10 @@ +# pylint: disable=c0111 + import os import os.path import datetime -from sfa.util.faults import SfaFault, SfaAPIError, RecordNotFound +from sfa.util.faults import SfaFault, RecordNotFound from sfa.util.genicode import GENICODE from sfa.util.config import Config from sfa.util.cache import Cache @@ -20,19 +22,19 @@ from sfa.storage.alchemy import alchemy #################### -class SfaApi (XmlrpcApi): +class SfaApi(XmlrpcApi): """ An SfaApi instance is a basic xmlrpc service augmented with the local cryptographic material and hrn It also has the notion of its own interface (a string describing - whether we run a registry, aggregate or slicemgr) and has - the notion of neighbour sfa services as defined + whether we run a registry, aggregate or slicemgr) and has + the notion of neighbour sfa services as defined in /etc/sfa/{aggregates,registries}.xml Finally it contains a cache instance - It gets augmented by the generic layer with + It gets augmented by the generic layer with (*) an instance of manager (actually a manager module for now) beware that this is shared among all instances of api (*) an instance of a testbed driver @@ -77,8 +79,8 @@ class SfaApi (XmlrpcApi): def server_proxy(self, interface, cred, timeout=30): """ Returns a connection to the specified interface. Use the specified - credential to determine the caller and look for the caller's key/cert - in the registry hierarchy cache. + credential to determine the caller and look for the caller's key/cert + in the registry hierarchy cache. """ from sfa.trust.hierarchy import Hierarchy if not isinstance(cred, Credential): @@ -147,8 +149,8 @@ class SfaApi (XmlrpcApi): return delegated_cred def _getCredential(self): - """ - Get our credential from a remote registry + """ + Get our credential from a remote registry """ from sfa.server.registry import Registries registries = Registries() @@ -260,8 +262,8 @@ class SfaApi (XmlrpcApi): def prepare_response(self, result, method=""): """ - Converts the specified result into a standard GENI compliant - response + Converts the specified result into a standard GENI compliant + response """ # as of dec 13 2011 we only support API v2 if self.interface.lower() in ['aggregate', 'slicemgr']: diff --git a/sfa/trust/certificate.py b/sfa/trust/certificate.py index 936352f7..b5296a6a 100644 --- a/sfa/trust/certificate.py +++ b/sfa/trust/certificate.py @@ -410,7 +410,7 @@ class Certificate: # certs) if string is None or string.strip() == "": - logger.warn("Empty string in load_from_string") + logger.warning("Empty string in load_from_string") return string = string.strip() @@ -441,7 +441,7 @@ class Certificate: OpenSSL.crypto.FILETYPE_PEM, parts[0]) if self.x509 is None: - logger.warn( + logger.warning( "Loaded from string but cert is None: {}".format(string)) # if there are more certs, then create a parent and let the parent load @@ -467,7 +467,7 @@ class Certificate: def save_to_string(self, save_parents=True): if self.x509 is None: - logger.warn("None cert in certificate.save_to_string") + logger.warning("None cert in certificate.save_to_string") return "" string = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, self.x509) @@ -694,7 +694,7 @@ class Certificate: # pyOpenSSL does not have a way to get extensions m2x509 = M2Crypto.X509.load_cert_string(certstr) if m2x509 is None: - logger.warn("No cert loaded in get_extension") + logger.warning("No cert loaded in get_extension") return None if m2x509.get_ext(name) is None: return None @@ -903,8 +903,8 @@ class Certificate: # extension and hope there are no other basicConstraints if not self.parent.isCA and not ( self.parent.get_extension('basicConstraints') == 'CA:TRUE'): - logger.warn("verify_chain: cert {}'s parent {} is not a CA" - .format(self.pretty_name(), self.parent.pretty_name())) + logger.warning("verify_chain: cert {}'s parent {} is not a CA" + .format(self.pretty_name(), self.parent.pretty_name())) raise CertNotSignedByParent("{}: Parent {} not a CA" .format(self.pretty_name(), self.parent.pretty_name())) diff --git a/sfa/trust/credential.py b/sfa/trust/credential.py index b957fadd..301fefe4 100644 --- a/sfa/trust/credential.py +++ b/sfa/trust/credential.py @@ -599,7 +599,7 @@ class Credential(object): self.gidCaller.get_urn(), self.gidObject.get_urn(), oldAttr.name, oldAttr.value, attr.value) - logger.warn(msg) + logger.warning(msg) # raise CredentialNotVerifiable( # "Can't encode new valid delegated credential: {}" # .format(msg)) @@ -707,10 +707,10 @@ class Credential(object): def sign(self): if not self.issuer_privkey: - logger.warn("Cannot sign credential (no private key)") + logger.warning("Cannot sign credential (no private key)") return if not self.issuer_gid: - logger.warn("Cannot sign credential (no issuer gid)") + logger.warning("Cannot sign credential (no issuer gid)") return doc = parseString(self.get_xml()) sigs = doc.getElementsByTagName("signatures")[0]