From: Loic Baron Date: Wed, 23 Jul 2014 13:01:29 +0000 (+0200) Subject: Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3 X-Git-Tag: sfa-3.1-10~5^2 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=4a2337e7f70cef81a8de37829aa63fc941c4b96e;hp=3519c849835dcc5ca47a2e03db24284017254ce6 Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3 Conflicts: sfa/rspecs/rspec.py --- diff --git a/config/default_config.xml b/config/default_config.xml index f1bf7841..de707b79 100644 --- a/config/default_config.xml +++ b/config/default_config.xml @@ -118,9 +118,10 @@ Thierry Parmentelat Enable Slice Manager - true + false Allows this local SFA instance to run as a - slice manager. + slice manager. Warning, this feature is not really supported + any longer. diff --git a/sfa.spec b/sfa.spec index bb4b24aa..fcce983c 100644 --- a/sfa.spec +++ b/sfa.spec @@ -1,6 +1,6 @@ %define name sfa %define version 3.1 -%define taglevel 4 +%define taglevel 9 %define release %{taglevel}%{?pldistro:.%{pldistro}}%{?date:.%{date}} %global python_sitearch %( python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)" ) @@ -264,6 +264,39 @@ fi #[ "$1" -ge "1" ] && service sfa-cm restart || : %changelog +* Mon Jul 21 2014 Thierry Parmentelat - sfa-3.1-9 +- Register can change the user keys using 'reg-keys' as well as 'keys' +- also accept a single string rather than a list of keys +- remove 'geni_api' from the registry GetVersion (which is not based on geni anymore) +- bump the 'sfa' tag in the same registry GetVersion to 3 +- remove all mutable used as default arguments + +* Thu Jun 05 2014 Thierry Parmentelat - sfa-3.1-8 +- bugfix, sfi remove was broken + +* Wed Jun 04 2014 Thierry Parmentelat - sfa-3.1-7 +- sfi return code should be more meaningful - not yet for all commands though +- DEFAULT_CREDENTIAL_LIFETIME now 28 days (was 31) +- dropped support for legacy credentials +- bugfix: short-lived credentials triggered a bug with UTC translated into localtime +- further minor cleanup of timestamp formats + +* Mon Jun 02 2014 Thierry Parmentelat - sfa-3.1-6 +- iotlab driver: Allocate uses OAR +- iotlab driver: using actual_caller_hrn + +* Thu May 29 2014 Thierry Parmentelat - sfa-3.1-5 +- Slice Manager is down by default +- sfi renew -l/--as-long-as-possible and e.g. sfi renew <> +2[d|w|m] +- also renew tries to find a max date for renewal instead of bailing out +- sfaclientlib file names scheme keeps track of user as well as object for credentials +- none fields get removed before sending over xmlrpc - partially for now +- cleanup on time formats and - hopefully timezones +- cleanup on speaking_for +- Allocate passes actual_caller_hrn as part of options to driver +- iotlab driver and leases +- new modules abac_credential, credential_factory and speaksfor_util + * Tue May 06 2014 Thierry Parmentelat - sfa-3.1-4 - for register and update, client is expected to set - reg-researchers rather than researcher diff --git a/sfa/client/sfaadmin.py b/sfa/client/sfaadmin.py index 15004ce4..a21cb5f6 100755 --- a/sfa/client/sfaadmin.py +++ b/sfa/client/sfaadmin.py @@ -102,7 +102,7 @@ class RegistryCommands(Commands): pubkey = open(key, 'r').read() except IOError: pubkey = key - record_dict['keys'] = [pubkey] + record_dict['reg-keys'] = [pubkey] if slices: record_dict['slices'] = slices if researchers: diff --git a/sfa/client/sfi.py b/sfa/client/sfi.py index 6ab2f5eb..dcad9dad 100644 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@ -201,7 +201,7 @@ def load_record_from_opts(options): pubkey = options.key if not check_ssh_key (pubkey): raise SfaInvalidArgument(name='key',msg="Could not find file, or wrong key format") - record_dict['keys'] = [pubkey] + record_dict['reg-keys'] = [pubkey] if hasattr(options, 'slices') and options.slices: record_dict['slices'] = options.slices if hasattr(options, 'reg_researchers') and options.reg_researchers is not None: @@ -448,6 +448,9 @@ class Sfi: if canonical in ("list","resources", "describe", "provision", "allocate", "register","update","remove","delete","status","renew"): parser.add_option("-C","--credential",dest='show_credential',action='store_true',default=False, help="show credential(s) used in human-readable form") + if canonical in ("renew"): + parser.add_option("-l","--as-long-as-possible",dest='alap',action='store_true',default=False, + help="renew as long as possible") # registy filter option if canonical in ("list", "show", "remove"): parser.add_option("-t", "--type", dest="type", type="choice", @@ -572,14 +575,13 @@ use this if you mean an authority instead""") self.logger.debug("Command=%s" % self.command) try: - self.dispatch(command, command_options, command_args) + retcod = self.dispatch(command, command_options, command_args) except SystemExit: return 1 except: self.logger.log_exc ("sfi command %s failed"%command) return 1 - - return 0 + return retcod #################### def read_config(self): @@ -867,6 +869,18 @@ use this if you mean an authority instead""") sys.exit(1) + # helper function to analyze raw output + # for main : return 0 if everything is fine, something else otherwise (mostly 1 for now) + def success (self, raw): + return_value=ReturnValue (raw) + output=ReturnValue.get_output(return_value) + # means everything is fine + if not output: + return 0 + # something went wrong + print 'ERROR:',output + return 1 + #========================================================================== # Following functions implement the commands # @@ -896,6 +910,8 @@ use this if you mean an authority instead""") varname="%s_%s"%(section.upper(),name.upper()) value=getattr(self.config_instance,varname) print "%-20s = %s"%(name,value) + # xxx should analyze result + return 0 @declare_command("","") def version(self, options, args): @@ -917,6 +933,8 @@ use this if you mean an authority instead""") else: pprinter = PrettyPrinter(indent=4) pprinter.pprint(version) + # xxx should analyze result + return 0 @declare_command("authority","") def list(self, options, args): @@ -944,7 +962,8 @@ use this if you mean an authority instead""") terminal_render (list, options) if options.file: save_records_to_file(options.file, list, options.fileformat) - return + # xxx should analyze result + return 0 @declare_command("name","") def show(self, options, args): @@ -978,7 +997,8 @@ use this if you mean an authority instead""") else: print record.save_as_xml() if options.file: save_records_to_file(options.file, record_dicts, options.fileformat) - return + # xxx should analyze result + return 0 # this historically was named 'add', it is now 'register' with an alias for legacy @declare_command("[xml-filename]","",['add']) @@ -1015,7 +1035,11 @@ use this if you mean an authority instead""") record_dict['first_name'] = record_dict['hrn'] if 'last_name' not in record_dict: record_dict['last_name'] = record_dict['hrn'] - return self.registry().Register(record_dict, auth_cred) + register = self.registry().Register(record_dict, auth_cred) + # xxx looks like the result here is not ReturnValue-compatible + #return self.success (register) + # xxx should analyze result + return 0 @declare_command("[xml-filename]","") def update(self, options, args): @@ -1059,7 +1083,11 @@ use this if you mean an authority instead""") raise "unknown record type" + record_dict['type'] if options.show_credential: show_credentials(cred) - return self.registry().Update(record_dict, cred) + update = self.registry().Update(record_dict, cred) + # xxx looks like the result here is not ReturnValue-compatible + #return self.success(update) + # xxx should analyze result + return 0 @declare_command("hrn","") def remove(self, options, args): @@ -1074,7 +1102,11 @@ use this if you mean an authority instead""") type = '*' if options.show_credential: show_credentials(auth_cred) - return self.registry().Remove(hrn, auth_cred, type) + remove = self.registry().Remove(hrn, auth_cred, type) + # xxx looks like the result here is not ReturnValue-compatible + #return self.success (remove) + # xxx should analyze result + return 0 # ================================================================== # Slice-related commands @@ -1121,16 +1153,15 @@ use this if you mean an authority instead""") api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3'} else: api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3'} - result = server.ListResources (creds, api_options) - value = ReturnValue.get_value(result) + list_resources = server.ListResources (creds, api_options) + value = ReturnValue.get_value(list_resources) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(list_resources, self.options.raw, self.options.rawformat, self.options.rawbanner) if options.file is not None: save_rspec_to_file(value, options.file) if (self.options.raw is None) and (options.file is None): display_rspec(value, options.format) - - return + return self.success(list_resources) @declare_command("slice_hrn","") def describe(self, options, args): @@ -1166,16 +1197,15 @@ use this if you mean an authority instead""") api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3'} urn = Xrn(args[0], type='slice').get_urn() remove_none_fields(api_options) - result = server.Describe([urn], creds, api_options) - value = ReturnValue.get_value(result) + describe = server.Describe([urn], creds, api_options) + value = ReturnValue.get_value(describe) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(describe, self.options.raw, self.options.rawformat, self.options.rawbanner) if options.file is not None: save_rspec_to_file(value['geni_rspec'], options.file) if (self.options.raw is None) and (options.file is None): - display_rspec(value, options.format) - - return + display_rspec(value['geni_rspec'], options.format) + return self.success (describe) @declare_command("slice_hrn [...]","") def delete(self, options, args): @@ -1204,13 +1234,13 @@ use this if you mean an authority instead""") api_options ['call_id'] = unique_call_id() if options.show_credential: show_credentials(creds) - result = server.Delete(sliver_urns, creds, *self.ois(server, api_options ) ) - value = ReturnValue.get_value(result) + delete = server.Delete(sliver_urns, creds, *self.ois(server, api_options ) ) + value = ReturnValue.get_value(delete) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(delete, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value - return value + return self.success (delete) @declare_command("slice_hrn rspec","") def allocate(self, options, args): @@ -1259,16 +1289,15 @@ use this if you mean an authority instead""") api_options['sfa_users'] = sfa_users api_options['geni_users'] = geni_users - result = server.Allocate(slice_urn, creds, rspec, api_options) - value = ReturnValue.get_value(result) + allocate = server.Allocate(slice_urn, creds, rspec, api_options) + value = ReturnValue.get_value(allocate) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(allocate, self.options.raw, self.options.rawformat, self.options.rawbanner) if options.file is not None: save_rspec_to_file (value['geni_rspec'], options.file) if (self.options.raw is None) and (options.file is None): print value - return value - + return self.success(allocate) @declare_command("slice_hrn [...]","") def provision(self, options, args): @@ -1325,15 +1354,15 @@ use this if you mean an authority instead""") users = pg_users_arg(user_records) api_options['geni_users'] = users - result = server.Provision(sliver_urns, creds, api_options) - value = ReturnValue.get_value(result) + provision = server.Provision(sliver_urns, creds, api_options) + value = ReturnValue.get_value(provision) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(provision, self.options.raw, self.options.rawformat, self.options.rawbanner) if options.file is not None: save_rspec_to_file (value['geni_rspec'], options.file) if (self.options.raw is None) and (options.file is None): print value - return value + return self.success(provision) @declare_command("slice_hrn","") def status(self, options, args): @@ -1355,14 +1384,13 @@ use this if you mean an authority instead""") api_options['call_id']=unique_call_id() if options.show_credential: show_credentials(creds) - result = server.Status([slice_urn], creds, *self.ois(server,api_options)) - value = ReturnValue.get_value(result) + status = server.Status([slice_urn], creds, *self.ois(server,api_options)) + value = ReturnValue.get_value(status) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(status, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value - # Thierry: seemed to be missing - return value + return self.success (status) @declare_command("slice_hrn [...] action","") def action(self, options, args): @@ -1388,15 +1416,20 @@ use this if you mean an authority instead""") delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) - result = server.PerformOperationalAction(sliver_urns, creds, action , api_options) - value = ReturnValue.get_value(result) + perform_action = server.PerformOperationalAction(sliver_urns, creds, action , api_options) + value = ReturnValue.get_value(perform_action) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(perform_action, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value - return value - - @declare_command("slice_hrn [...] time","") + return self.success (perform_action) + + @declare_command("slice_hrn [...] time", + "\n".join(["sfi renew onelab.ple.heartbeat 2015-04-31", + "sfi renew onelab.ple.heartbeat 2015-04-31T14:00:00Z", + "sfi renew onelab.ple.heartbeat +5d", + "sfi renew onelab.ple.heartbeat +3w", + "sfi renew onelab.ple.heartbeat +2m",])) def renew(self, options, args): """ renew slice (Renew) @@ -1423,16 +1456,17 @@ use this if you mean an authority instead""") # options and call_id when supported api_options = {} api_options['call_id']=unique_call_id() + if options.alap: + api_options['geni_extend_alap']=True if options.show_credential: show_credentials(creds) - result = server.Renew(sliver_urns, creds, input_time, *self.ois(server,api_options)) - value = ReturnValue.get_value(result) + renew = server.Renew(sliver_urns, creds, input_time, *self.ois(server,api_options)) + value = ReturnValue.get_value(renew) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(renew, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value - return value - + return self.success(renew) @declare_command("slice_hrn","") def shutdown(self, options, args): @@ -1446,14 +1480,13 @@ use this if you mean an authority instead""") # creds slice_cred = self.slice_credential(slice_hrn) creds = [slice_cred] - result = server.Shutdown(slice_urn, creds) - value = ReturnValue.get_value(result) + shutdown = server.Shutdown(slice_urn, creds) + value = ReturnValue.get_value(shutdown) if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + save_raw_to_file(shutdown, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value - return value - + return self.success (shutdown) @declare_command("[name]","") def gid(self, options, args): @@ -1472,6 +1505,8 @@ use this if you mean an authority instead""") filename = os.sep.join([self.options.sfi_dir, '%s.gid' % target_hrn]) self.logger.info("writing %s gid to %s" % (target_hrn, filename)) GID(string=gid).save_to_file(filename) + # xxx should analyze result + return 0 #################### @declare_command("to_hrn","""$ sfi delegate -u -p -s ple.inria.heartbeat -s ple.inria.omftest ple.upmc.slicebrowser @@ -1672,7 +1707,8 @@ $ sfi m -b http://mymanifold.foo.com:7080/ # it is probably not helpful as people would not # need to run 'sfi delegate' at all anymore if count_success != count_all: sys.exit(1) - return + # xxx should analyze result + return 0 @declare_command("cred","") def trusted(self, options, args): @@ -1695,5 +1731,5 @@ $ sfi m -b http://mymanifold.foo.com:7080/ cert = Certificate(string=trusted_cert) self.logger.debug('Sfi.trusted -> %r'%cert.get_subject()) print "Certificate:\n%s\n\n"%trusted_cert - return - + # xxx should analyze result + return 0 diff --git a/sfa/cortexlab/cortexlabaggregate.py b/sfa/cortexlab/cortexlabaggregate.py index d9cddf3b..24a53106 100644 --- a/sfa/cortexlab/cortexlabaggregate.py +++ b/sfa/cortexlab/cortexlabaggregate.py @@ -292,7 +292,7 @@ class CortexlabAggregate: return rspec_node - def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}): + def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations=None): """Makes a geni sliver structure from all the nodes allocated to slivers in the sliver_allocations dictionary. Returns the states of the sliver. @@ -312,6 +312,8 @@ class CortexlabAggregate: .. seealso:: node_to_rspec_node """ + if sliver_allocations is None: sliver_allocations={} + if rspec_node['sliver_id'] in sliver_allocations: # set sliver allocation and operational status sliver_allocation = sliver_allocations[rspec_node['sliver_id']] @@ -555,7 +557,7 @@ class CortexlabAggregate: - def get_slivers(self, urns, options={}): + def get_slivers(self, urns, options=None): """Get slivers of the given slice urns. Slivers contains slice, node and user information. @@ -569,7 +571,7 @@ class CortexlabAggregate: .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns """ - + if options is None: options={} slice_ids = set() node_ids = [] @@ -667,7 +669,7 @@ class CortexlabAggregate: return slivers - def list_resources(self, version = None, options={}): + def list_resources(self, version = None, options=None): """ Returns an advertisement Rspec of available resources at this aggregate. This Rspec contains a resource listing along with their @@ -688,6 +690,8 @@ class CortexlabAggregate: .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#ListResources """ + if options is None: options={} + version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version(version.type, @@ -722,7 +726,7 @@ class CortexlabAggregate: return rspec.toxml() - def describe(self, urns, version=None, options={}): + def describe(self, urns, version=None, options=None): """ Retrieve a manifest RSpec describing the resources contained by the named entities, e.g. a single slice or a set of the slivers in a slice. @@ -752,6 +756,7 @@ class CortexlabAggregate: .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#Describe .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns """ + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version( diff --git a/sfa/cortexlab/cortexlabdriver.py b/sfa/cortexlab/cortexlabdriver.py index bc674f76..c5ad4e6a 100644 --- a/sfa/cortexlab/cortexlabdriver.py +++ b/sfa/cortexlab/cortexlabdriver.py @@ -1081,7 +1081,7 @@ class CortexlabDriver(Driver): - def delete(self, slice_urns, options={}): + def delete(self, slice_urns, options=None): """ Deletes the lease associated with the slice hrn and the credentials if the slice belongs to iotlab. Answer to DeleteSliver. @@ -1099,6 +1099,7 @@ class CortexlabDriver(Driver): .. note:: creds are unused, and are not used either in the dummy driver delete_sliver . """ + if options is None: options={} # collect sliver ids so we can update sliver allocation states after # we remove the slivers. aggregate = CortexlabAggregate(self) @@ -1390,17 +1391,20 @@ class CortexlabDriver(Driver): # first 2 args are None in case of resource discovery - def list_resources (self, version=None, options={}): + def list_resources (self, version=None, options=None): + if options is None: options={} aggregate = CortexlabAggregate(self) rspec = aggregate.list_resources(version=version, options=options) return rspec - def describe(self, urns, version, options={}): + def describe(self, urns, version, options=None): + if options is None: options={} aggregate = CortexlabAggregate(self) return aggregate.describe(urns, version=version, options=options) - def status (self, urns, options={}): + def status (self, urns, options=None): + if options is None: options={} aggregate = CortexlabAggregate(self) desc = aggregate.describe(urns, version='GENI 3') status = {'geni_urn': desc['geni_urn'], @@ -1408,7 +1412,8 @@ class CortexlabDriver(Driver): return status - def allocate (self, urn, rspec_string, expiration, options={}): + def allocate (self, urn, rspec_string, expiration, options=None): + if options is None: options={} xrn = Xrn(urn) aggregate = CortexlabAggregate(self) @@ -1488,7 +1493,8 @@ class CortexlabDriver(Driver): return aggregate.describe([xrn.get_urn()], version=rspec.version) - def provision(self, urns, options={}): + def provision(self, urns, options=None): + if options is None: options={} # update users slices = CortexlabSlices(self) aggregate = CortexlabAggregate(self) diff --git a/sfa/cortexlab/cortexlabshell.py b/sfa/cortexlab/cortexlabshell.py index 1321484a..b0739b14 100644 --- a/sfa/cortexlab/cortexlabshell.py +++ b/sfa/cortexlab/cortexlabshell.py @@ -7,7 +7,7 @@ holding information about which slice is running which job. from datetime import datetime from sfa.util.sfalogging import logger - +from sfa.util.sfatime import SFATIME_FORMAT from sfa.iotlab.iotlabpostgres import LeaseTableXP from sfa.cortexlab.LDAPapi import LDAPapi @@ -34,7 +34,7 @@ class CortexlabShell(): self.query_sites = CortexlabQueryNodes() self.ldap = LDAPapi() - self.time_format = "%Y-%m-%d %H:%M:%S" + self.time_format = SFATIME_FORMAT self.root_auth = config.SFA_REGISTRY_ROOT_AUTH self.grain = 60 # 10 mins lease minimum, 60 sec granularity #import logging, logging.handlers diff --git a/sfa/cortexlab/cortexlabslices.py b/sfa/cortexlab/cortexlabslices.py index ee160d43..888d7dbe 100644 --- a/sfa/cortexlab/cortexlabslices.py +++ b/sfa/cortexlab/cortexlabslices.py @@ -351,7 +351,7 @@ class CortexlabSlices: return sfa_slice - def verify_persons(self, slice_hrn, slice_record, users, options={}): + def verify_persons(self, slice_hrn, slice_record, users, options=None): """Ensures the users in users list exist and are enabled in LDAP. Adds person if needed(AddPerson). @@ -378,6 +378,7 @@ class CortexlabSlices: """ + if options is None: options={} logger.debug("CortexlabSlices \tverify_persons \tslice_hrn %s \ \t slice_record %s\r\n users %s \t " @@ -525,10 +526,11 @@ class CortexlabSlices: return added_persons - def verify_keys(self, persons, users, peer, options={}): + def verify_keys(self, persons, users, peer, options=None): """ .. warning:: unused """ + if options is None: options={} # existing keys key_ids = [] for person in persons: diff --git a/sfa/dummy/dummy_testbed_api.py b/sfa/dummy/dummy_testbed_api.py index 2673166d..f553e40e 100644 --- a/sfa/dummy/dummy_testbed_api.py +++ b/sfa/dummy/dummy_testbed_api.py @@ -12,7 +12,12 @@ for i in range(1,11): slices_list = [] for i in range(1,3): - slice = {'slice_name': 'slice'+str(i), 'user_ids': range(i,4,2), 'slice_id': i, 'node_ids': range(i,10,2), 'enabled': True, 'expires': int(time.time())+60*60*24*30} + slice = {'slice_name': 'slice'+str(i), + 'user_ids': range(i,4,2), + 'slice_id': i, + 'node_ids': range(i,10,2), + 'enabled': True, + 'expires': int(time.time())+60*60*24*30} slices_list.append(slice) users_list = [] @@ -43,7 +48,8 @@ def FilterList(myfilter, mylist): def GetTestbedInfo(): return {'name': 'dummy', 'longitude': 123456, 'latitude': 654321, 'domain':'dummy-testbed.org'} -def GetNodes(filter={}): +def GetNodes(filter=None): + if filter is None: filter={} global DB result = [] result.extend(DB['nodes_list']) @@ -55,7 +61,8 @@ def GetNodes(filter={}): result = FilterList(filter, result) return result -def GetSlices(filter={}): +def GetSlices(filter=None): + if filter is None: filter={} global DB result = [] result.extend(DB['slices_list']) @@ -69,7 +76,8 @@ def GetSlices(filter={}): return result -def GetUsers(filter={}): +def GetUsers(filter=None): + if filter is None: filter={} global DB result = [] result.extend(DB['users_list']) diff --git a/sfa/dummy/dummyaggregate.py b/sfa/dummy/dummyaggregate.py index 576ccd58..c5b4d10c 100644 --- a/sfa/dummy/dummyaggregate.py +++ b/sfa/dummy/dummyaggregate.py @@ -52,12 +52,14 @@ class DummyAggregate: return (slice, slivers) - def get_nodes(self, options={}): + def get_nodes(self, options=None): + if options is None: options={} filter = {} nodes = self.driver.shell.GetNodes(filter) return nodes - def get_slivers(self, urns, options={}): + def get_slivers(self, urns, options=None): + if options is None: options={} slice_names = set() slice_ids = set() node_ids = [] @@ -122,7 +124,8 @@ class DummyAggregate: slivers.append(node) return slivers - def node_to_rspec_node(self, node, options={}): + def node_to_rspec_node(self, node, options=None): + if options is None: options={} rspec_node = NodeElement() site=self.driver.testbedInfo rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['name'], node['hostname']) @@ -163,7 +166,8 @@ class DummyAggregate: }) return rspec_node - def get_slice_nodes(self, slice, options={}): + def get_slice_nodes(self, slice, options=None): + if options is None: options={} nodes_dict = {} filter = {} if slice and slice.get('node_ids'): @@ -176,7 +180,8 @@ class DummyAggregate: nodes_dict[node['node_id']] = node return nodes_dict - def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}): + def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = None): + if sliver_allocations is None: sliver_allocations={} if rspec_node['sliver_id'] in sliver_allocations: # set sliver allocation and operational status sliver_allocation = sliver_allocations[rspec_node['sliver_id']] @@ -202,7 +207,8 @@ class DummyAggregate: } return geni_sliver - def list_resources(self, version = None, options={}): + def list_resources(self, version = None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) @@ -224,7 +230,8 @@ class DummyAggregate: return rspec.toxml() - def describe(self, urns, version=None, options={}): + def describe(self, urns, version=None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version(version.type, version.version, 'manifest') diff --git a/sfa/dummy/dummydriver.py b/sfa/dummy/dummydriver.py index 4d7f7a38..a69662e1 100644 --- a/sfa/dummy/dummydriver.py +++ b/sfa/dummy/dummydriver.py @@ -412,16 +412,19 @@ class DummyDriver (Driver): def aggregate_version (self): return {} - def list_resources (self, version=None, options={}): + def list_resources (self, version=None, options=None): + if options is None: options={} aggregate = DummyAggregate(self) rspec = aggregate.list_resources(version=version, options=options) return rspec - def describe(self, urns, version, options={}): + def describe(self, urns, version, options=None): + if options is None: options={} aggregate = DummyAggregate(self) return aggregate.describe(urns, version=version, options=options) - def status (self, urns, options={}): + def status (self, urns, options=None): + if options is None: options={} aggregate = DummyAggregate(self) desc = aggregate.describe(urns, version='GENI 3') status = {'geni_urn': desc['geni_urn'], @@ -429,7 +432,8 @@ class DummyDriver (Driver): return status - def allocate (self, urn, rspec_string, expiration, options={}): + def allocate (self, urn, rspec_string, expiration, options=None): + if options is None: options={} xrn = Xrn(urn) aggregate = DummyAggregate(self) slices = DummySlices(self) @@ -453,7 +457,8 @@ class DummyDriver (Driver): return aggregate.describe([xrn.get_urn()], version=rspec.version) - def provision(self, urns, options={}): + def provision(self, urns, options=None): + if options is None: options={} # update users slices = DummySlices(self) aggregate = DummyAggregate(self) @@ -469,7 +474,8 @@ class DummyDriver (Driver): rspec_version = version_manager.get_version(options['geni_rspec_version']) return self.describe(urns, rspec_version, options=options) - def delete(self, urns, options={}): + def delete(self, urns, options=None): + if options is None: options={} # collect sliver ids so we can update sliver allocation states after # we remove the slivers. aggregate = DummyAggregate(self) @@ -504,7 +510,8 @@ class DummyDriver (Driver): 'geni_expires': datetime_to_string(utcparse(sliver['expires']))}) return geni_slivers - def renew (self, urns, expiration_time, options={}): + def renew (self, urns, expiration_time, options=None): + if options is None: options={} aggregate = DummyAggregate(self) slivers = aggregate.get_slivers(urns) if not slivers: @@ -516,7 +523,8 @@ class DummyDriver (Driver): description = self.describe(urns, 'GENI 3', options) return description['geni_slivers'] - def perform_operational_action (self, urns, action, options={}): + def perform_operational_action (self, urns, action, options=None): + if options is None: options={} # Dummy doesn't support operational actions. Lets pretend like it # supports start, but reject everything else. action = action.lower() @@ -535,7 +543,8 @@ class DummyDriver (Driver): geni_slivers = self.describe(urns, 'GENI 3', options)['geni_slivers'] return geni_slivers - def shutdown (self, xrn, options={}): + def shutdown (self, xrn, options=None): + if options is None: options={} xrn = DummyXrn(xrn=xrn, type='slice') slicename = xrn.pl_slicename() slices = self.shell.GetSlices({'name': slicename}, ['slice_id']) diff --git a/sfa/dummy/dummyslices.py b/sfa/dummy/dummyslices.py index cf5a6da9..7ab94ba2 100644 --- a/sfa/dummy/dummyslices.py +++ b/sfa/dummy/dummyslices.py @@ -110,7 +110,8 @@ class DummySlices: return resulting_nodes - def verify_slice(self, slice_hrn, slice_record, expiration, options={}): + def verify_slice(self, slice_hrn, slice_record, expiration, options=None): + if options is None: options={} slicename = hrn_to_dummy_slicename(slice_hrn) parts = slicename.split("_") login_base = parts[0] @@ -130,7 +131,8 @@ class DummySlices: return slice - def verify_users(self, slice_hrn, slice_record, users, options={}): + def verify_users(self, slice_hrn, slice_record, users, options=None): + if options is None: options={} slice_name = hrn_to_dummy_slicename(slice_hrn) users_by_email = {} for user in users: @@ -162,7 +164,8 @@ class DummySlices: pass - def verify_keys(self, old_users, new_users, options={}): + def verify_keys(self, old_users, new_users, options=None): + if options is None: options={} # existing keys existing_keys = [] for user in old_users: diff --git a/sfa/importer/__init__.py b/sfa/importer/__init__.py index 94d17537..e1c3a289 100644 --- a/sfa/importer/__init__.py +++ b/sfa/importer/__init__.py @@ -118,7 +118,7 @@ class Importer: generic=Generic.the_flavour() importer_class = generic.importer_class() if importer_class: - begin_time=datetime.now() + begin_time=datetime.utcnow() self.logger.info (30*'=') self.logger.info ("Starting import on %s, using class %s from flavour %s"%\ (begin_time,importer_class.__name__,generic.flavour)) @@ -126,7 +126,7 @@ class Importer: if testbed_importer: testbed_importer.add_options(options) testbed_importer.run (options) - end_time=datetime.now() + end_time=datetime.utcnow() duration=end_time-begin_time self.logger.info("Import took %s"%duration) self.logger.info (30*'=') diff --git a/sfa/iotlab/iotlabaggregate.py b/sfa/iotlab/iotlabaggregate.py index a08f822d..56e40e4f 100644 --- a/sfa/iotlab/iotlabaggregate.py +++ b/sfa/iotlab/iotlabaggregate.py @@ -306,7 +306,7 @@ class IotlabAggregate: return rspec_node - def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}): + def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = None): """Makes a geni sliver structure from all the nodes allocated to slivers in the sliver_allocations dictionary. Returns the states of the sliver. @@ -326,6 +326,7 @@ class IotlabAggregate: .. seealso:: node_to_rspec_node """ + if sliver_allocations is None: sliver_allocations={} if rspec_node['sliver_id'] in sliver_allocations: # set sliver allocation and operational status sliver_allocation = sliver_allocations[rspec_node['sliver_id']] @@ -400,6 +401,39 @@ class IotlabAggregate: return rspec_node + def get_leases(self, slice=None, options=None): + if options is None: options={} + filter={} + if slice: + filter.update({'name':slice['slice_name']}) + #return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until'] + leases = self.driver.GetLeases(lease_filter_dict=filter) + grain = self.driver.testbed_shell.GetLeaseGranularity() + + rspec_leases = [] + for lease in leases: + #as many leases as there are nodes in the job + for node in lease['reserved_nodes']: + rspec_lease = Lease() + rspec_lease['lease_id'] = lease['lease_id'] + #site = node['site_id'] + iotlab_xrn = xrn_object(self.driver.testbed_shell.root_auth, + node) + rspec_lease['component_id'] = iotlab_xrn.urn + #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ + #site, node['hostname']) + try: + rspec_lease['slice_id'] = lease['slice_id'] + except KeyError: + #No info on the slice used in testbed_xp table + pass + rspec_lease['start_time'] = lease['t_from'] + rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ + / grain + rspec_leases.append(rspec_lease) + return rspec_leases + + def get_all_leases(self, ldap_username): """ Get list of lease dictionaries which all have the mandatory keys @@ -566,7 +600,7 @@ class IotlabAggregate: FINAL RSPEC %s \r\n" % (rspec.toxml())) return rspec.toxml() - def get_slivers(self, urns, options={}): + def get_slivers(self, urns, options=None): """Get slivers of the given slice urns. Slivers contains slice, node and user information. @@ -581,7 +615,7 @@ class IotlabAggregate: .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns """ - + if options is None: options={} slice_ids = set() node_ids = [] for urn in urns: @@ -677,7 +711,7 @@ class IotlabAggregate: slivers.append(node) return slivers - def list_resources(self, version = None, options={}): + def list_resources(self, version = None, options=None): """ Returns an advertisement Rspec of available resources at this aggregate. This Rspec contains a resource listing along with their @@ -698,6 +732,7 @@ class IotlabAggregate: .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#ListResources """ + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version(version.type, @@ -732,7 +767,7 @@ class IotlabAggregate: return rspec.toxml() - def describe(self, urns, version=None, options={}): + def describe(self, urns, version=None, options=None): """ Retrieve a manifest RSpec describing the resources contained by the named entities, e.g. a single slice or a set of the slivers in a slice. @@ -762,6 +797,7 @@ class IotlabAggregate: .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#Describe .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns """ + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version( @@ -780,39 +816,34 @@ class IotlabAggregate: # lookup the sliver allocations geni_urn = urns[0] sliver_ids = [sliver['sliver_id'] for sliver in slivers] - logger.debug(" IOTLAB_API.PY \tDescribe sliver_ids %s " - % (sliver_ids)) constraint = SliverAllocation.sliver_id.in_(sliver_ids) query = self.driver.api.dbsession().query(SliverAllocation) sliver_allocations = query.filter((constraint)).all() - logger.debug(" IOTLAB_API.PY \tDescribe sliver_allocations %s " - % (sliver_allocations)) sliver_allocation_dict = {} for sliver_allocation in sliver_allocations: geni_urn = sliver_allocation.slice_urn sliver_allocation_dict[sliver_allocation.sliver_id] = \ sliver_allocation + if not options.get('list_leases') or options['list_leases'] != 'leases': + # add slivers + nodes_dict = {} + for sliver in slivers: + nodes_dict[sliver['node_id']] = sliver + rspec_nodes = [] + for sliver in slivers: + rspec_node = self.sliver_to_rspec_node(sliver, + sliver_allocation_dict) + rspec_nodes.append(rspec_node) + geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, + sliver_allocation_dict) + geni_slivers.append(geni_sliver) + rspec.version.add_nodes(rspec_nodes) - # add slivers - nodes_dict = {} - for sliver in slivers: - nodes_dict[sliver['node_id']] = sliver - rspec_nodes = [] - for sliver in slivers: - rspec_node = self.sliver_to_rspec_node(sliver, - sliver_allocation_dict) - rspec_nodes.append(rspec_node) - logger.debug(" IOTLAB_API.PY \tDescribe sliver_allocation_dict %s " - % (sliver_allocation_dict)) - geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, - sliver_allocation_dict) - geni_slivers.append(geni_sliver) - - logger.debug(" IOTLAB_API.PY \tDescribe rspec_nodes %s\ - rspec %s " - % (rspec_nodes, rspec)) - rspec.version.add_nodes(rspec_nodes) + if not options.get('list_leases') or options['list_leases'] == 'resources': + if slivers: + leases = self.get_leases(slivers[0]) + rspec.version.add_leases(leases) return {'geni_urn': geni_urn, 'geni_rspec': rspec.toxml(), - 'geni_slivers': geni_slivers} \ No newline at end of file + 'geni_slivers': geni_slivers} diff --git a/sfa/iotlab/iotlabdriver.py b/sfa/iotlab/iotlabdriver.py index e5ef90a5..797f156a 100644 --- a/sfa/iotlab/iotlabdriver.py +++ b/sfa/iotlab/iotlabdriver.py @@ -179,12 +179,12 @@ class IotlabDriver(Driver): " %(user_dict)) hrn = user_dict['hrn'] person_urn = hrn_to_urn(hrn, 'user') - pubkey = user_dict['pkey'] try: + pubkey = user_dict['pkey'] pkey = convert_public_key(pubkey) except TypeError: #key not good. create another pkey - logger.warn('__add_person_to_db: unable to convert public \ + logger.warn('__add_person_to_db: no public key or unable to convert public \ key for %s' %(hrn )) pkey = Keypair(create=True) @@ -202,7 +202,7 @@ class IotlabDriver(Driver): user_record = RegUser(hrn=hrn , pointer= '-1', \ authority=get_authority(hrn), \ email=user_dict['email'], gid = person_gid) - user_record.reg_keys = [RegKey(user_dict['pkey'])] + #user_record.reg_keys = [RegKey(user_dict['pkey'])] user_record.just_created() self.api.dbsession().add (user_record) self.api.dbsession().commit() @@ -1088,7 +1088,7 @@ class IotlabDriver(Driver): - def delete(self, slice_urns, options={}): + def delete(self, slice_urns, options=None): """ Deletes the lease associated with the slice hrn and the credentials if the slice belongs to iotlab. Answer to DeleteSliver. @@ -1106,6 +1106,7 @@ class IotlabDriver(Driver): .. note:: creds are unused, and are not used either in the dummy driver delete_sliver . """ + if options is None: options={} # collect sliver ids so we can update sliver allocation states after # we remove the slivers. aggregate = IotlabAggregate(self) @@ -1398,16 +1399,19 @@ class IotlabDriver(Driver): 'geni_ad_rspec_versions': ad_rspec_versions} # first 2 args are None in case of resource discovery - def list_resources (self, version=None, options={}): + def list_resources (self, version=None, options=None): + if options is None: options={} aggregate = IotlabAggregate(self) rspec = aggregate.list_resources(version=version, options=options) return rspec - def describe(self, urns, version, options={}): + def describe(self, urns, version, options=None): + if options is None: options={} aggregate = IotlabAggregate(self) return aggregate.describe(urns, version=version, options=options) - def status (self, urns, options={}): + def status (self, urns, options=None): + if options is None: options={} aggregate = IotlabAggregate(self) desc = aggregate.describe(urns, version='GENI 3') status = {'geni_urn': desc['geni_urn'], @@ -1415,7 +1419,8 @@ class IotlabDriver(Driver): return status - def allocate (self, urn, rspec_string, expiration, options={}): + def allocate (self, urn, rspec_string, expiration, options=None): + if options is None: options={} xrn = Xrn(urn) aggregate = IotlabAggregate(self) @@ -1423,16 +1428,38 @@ class IotlabDriver(Driver): peer = slices.get_peer(xrn.get_hrn()) sfa_peer = slices.get_sfa_peer(xrn.get_hrn()) + caller_hrn = options.get('actual_caller_hrn', []) + caller_xrn = Xrn(caller_hrn) + caller_urn = caller_xrn.get_urn() - slice_record = None - users = options.get('geni_users', []) + logger.debug("IOTLABDRIVER.PY :: Allocate caller = %s" % (caller_urn)) + slice_record = {} + users = options.get('geni_users', []) sfa_users = options.get('sfa_users', []) + if sfa_users: - slice_record = sfa_users[0].get('slice_record', []) - slice_record['user'] = {'keys': users[0]['keys'], - 'email': users[0]['email'], - 'hrn': slice_record['reg-researchers'][0]} + user = None + # Looking for the user who actually called the Allocate function in the list of users of the slice + for u in sfa_users: + if 'urn' in u and u['urn'] == caller_urn: + user = u + logger.debug("user = %s" % u) + # If we find the user in the list we use it, else we take the 1st in the list as before + if user: + user_hrn = caller_hrn + else: + user = sfa_users[0] + # XXX Always empty ??? no slice_record in the Allocate call + #slice_record = sfa_users[0].get('slice_record', []) + user_xrn = Xrn(sfa_users[0]['urn']) + user_hrn = user_xrn.get_hrn() + + slice_record = user.get('slice_record', {}) + slice_record['user'] = {'keys': user['keys'], + 'email': user['email'], + 'hrn': user_hrn} + slice_record['authority'] = xrn.get_authority_hrn() logger.debug("IOTLABDRIVER.PY \t urn %s allocate options %s " % (urn, options)) @@ -1453,8 +1480,10 @@ class IotlabDriver(Driver): # oui c'est degueulasse, le slice_record se retrouve modifie # dans la methode avec les infos du user, els infos sont propagees # dans verify_slice_leases + logger.debug("IOTLABDRIVER.PY BEFORE slices.verify_persons") persons = slices.verify_persons(xrn.hrn, slice_record, users, options=options) + logger.debug("IOTLABDRIVER.PY AFTER slices.verify_persons") # ensure slice attributes exists # slices.verify_slice_attributes(slice, requested_attributes, # options=options) @@ -1487,7 +1516,10 @@ class IotlabDriver(Driver): client_id = hostname node_urn = xrn_object(self.testbed_shell.root_auth, hostname).urn component_id = node_urn - slice_urn = current_slice['reg-urn'] + if 'reg-urn' in current_slice: + slice_urn = current_slice['reg-urn'] + else: + slice_urn = current_slice['urn'] for lease in leases: if hostname in lease['reserved_nodes']: index = lease['reserved_nodes'].index(hostname) @@ -1502,7 +1534,8 @@ class IotlabDriver(Driver): return aggregate.describe([xrn.get_urn()], version=rspec.version) - def provision(self, urns, options={}): + def provision(self, urns, options=None): + if options is None: options={} # update users slices = IotlabSlices(self) aggregate = IotlabAggregate(self) diff --git a/sfa/iotlab/iotlabshell.py b/sfa/iotlab/iotlabshell.py index ba62ca91..25aba100 100644 --- a/sfa/iotlab/iotlabshell.py +++ b/sfa/iotlab/iotlabshell.py @@ -7,6 +7,7 @@ holding information about which slice is running which job. from datetime import datetime from sfa.util.sfalogging import logger +from sfa.util.sfatime import SFATIME_FORMAT from sfa.iotlab.OARrestapi import OARrestapi from sfa.iotlab.LDAPapi import LDAPapi @@ -30,7 +31,7 @@ class IotlabShell(): # self.leases_db = TestbedAdditionalSfaDB(config) self.oar = OARrestapi() self.ldap = LDAPapi() - self.time_format = "%Y-%m-%d %H:%M:%S" + self.time_format = SFATIME_FORMAT self.root_auth = config.SFA_REGISTRY_ROOT_AUTH self.grain = 60 # 10 mins lease minimum, 60 sec granularity #import logging, logging.handlers @@ -559,10 +560,12 @@ class IotlabShell(): #They will be set to None. if lease_dict['lease_start_time'] is not '0': #Readable time accepted by OAR + # converting timestamp to date in the local timezone tz = None start_time = datetime.fromtimestamp( \ - int(lease_dict['lease_start_time'])).\ + int(lease_dict['lease_start_time']), tz=None).\ strftime(lease_dict['time_format']) - reqdict['reservation'] = start_time + + reqdict['reservation'] = str(start_time) #If there is not start time, Immediate XP. No need to add special # OAR parameters @@ -593,7 +596,11 @@ class IotlabShell(): lease_dict['slice_name'] = slice_name lease_dict['slice_user'] = slice_user lease_dict['grain'] = self.GetLeaseGranularity() - lease_dict['time_format'] = self.time_format + # I don't know why the SFATIME_FORMAT has changed... + # from sfa.util.sfatime import SFATIME_FORMAT + # Let's use a fixed format %Y-%m-%d %H:%M:%S + #lease_dict['time_format'] = self.time_format + lease_dict['time_format'] = '%Y-%m-%d %H:%M:%S' logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\ diff --git a/sfa/iotlab/iotlabslices.py b/sfa/iotlab/iotlabslices.py index 91d89ed2..966a26b6 100644 --- a/sfa/iotlab/iotlabslices.py +++ b/sfa/iotlab/iotlabslices.py @@ -221,7 +221,9 @@ class IotlabSlices: #Deleted leases are the ones with lease id not declared in the Rspec if deleted_leases: self.driver.testbed_shell.DeleteLeases(deleted_leases, - sfa_slice['user']['uid']) + sfa_slice['login']) + #self.driver.testbed_shell.DeleteLeases(deleted_leases, + # sfa_slice['user']['uid']) logger.debug("IOTLABSLICES \ verify_slice_leases slice %s deleted_leases %s" % (sfa_slice, deleted_leases)) @@ -334,8 +336,8 @@ class IotlabSlices: 'authority': slice_record['authority'], 'gid': slice_record['gid'], 'slice_id': slice_record['record_id'], - 'reg-researchers': slice_record['reg-researchers'], - 'peer_authority': str(sfa_peer) + #'reg-researchers': slice_record['reg-researchers'], + #'peer_authority': str(sfa_peer) } if ldap_user: @@ -354,7 +356,7 @@ class IotlabSlices: return sfa_slice - def verify_persons(self, slice_hrn, slice_record, users, options={}): + def verify_persons(self, slice_hrn, slice_record, users, options=None): """Ensures the users in users list exist and are enabled in LDAP. Adds person if needed (AddPerson). @@ -380,7 +382,7 @@ class IotlabSlices: """ - + if options is None: options={} logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \ \t slice_record %s\r\n users %s \t " % (slice_hrn, slice_record, users)) @@ -501,8 +503,9 @@ class IotlabSlices: for k in k_list: if k in added_user: person[k] = added_user[k] - - person['pkey'] = added_user['keys'][0] + # bug user without key + if added_user['keys']: + person['pkey'] = added_user['keys'][0] person['mail'] = added_user['email'] person['email'] = added_user['email'] person['key_ids'] = added_user.get('key_ids', []) @@ -525,10 +528,11 @@ class IotlabSlices: return added_persons - def verify_keys(self, persons, users, peer, options={}): + def verify_keys(self, persons, users, peer, options=None): """ .. warning:: unused """ + if options is None: options={} # existing keys key_ids = [] for person in persons: diff --git a/sfa/managers/aggregate_manager.py b/sfa/managers/aggregate_manager.py index 28645675..8b736444 100644 --- a/sfa/managers/aggregate_manager.py +++ b/sfa/managers/aggregate_manager.py @@ -29,7 +29,8 @@ class AggregateManager: 'geni_ad_rspec_versions': ad_rspec_versions, } - def get_rspec_version_string(self, rspec_version, options={}): + def get_rspec_version_string(self, rspec_version, options=None): + if options is None: options={} version_string = "rspec_%s" % (rspec_version) #panos adding the info option to the caching key (can be improved) @@ -147,22 +148,16 @@ class AggregateManager: call_id = options.get('call_id') if Callids().already_handled(call_id): return True - # extend as long as possible - if options.get('geni_extend_alap'): - now = datetime.datetime.now() - requested = utcparse(expiration_time) - max = adjust_datetime(now, days=int(api.config.SFA_MAX_SLICE_RENEW)) - if requested > max: - expiration_time = max - return api.driver.renew(xrns, expiration_time, options) - def PerformOperationalAction(self, api, xrns, creds, action, options={}): + def PerformOperationalAction(self, api, xrns, creds, action, options=None): + if options is None: options={} call_id = options.get('call_id') if Callids().already_handled(call_id): return True return api.driver.perform_operational_action(xrns, action, options) - def Shutdown(self, api, xrn, creds, options={}): + def Shutdown(self, api, xrn, creds, options=None): + if options is None: options={} call_id = options.get('call_id') if Callids().already_handled(call_id): return True return api.driver.shutdown(xrn, options) diff --git a/sfa/managers/aggregate_manager_eucalyptus.py b/sfa/managers/aggregate_manager_eucalyptus.py index 228361d0..b8c83d8c 100644 --- a/sfa/managers/aggregate_manager_eucalyptus.py +++ b/sfa/managers/aggregate_manager_eucalyptus.py @@ -644,7 +644,7 @@ class AggregateManagerEucalyptus: ramdisk_id = instRamDisk, key_pair = instKey, inst_type = instType, - meta = Meta(start_time=datetime.datetime.now())) + meta = Meta(start_time=datetime.datetime.utcnow())) eucaInst.reserveInstance(conn, pubKeys) # xxx - should return altered rspec diff --git a/sfa/managers/aggregate_manager_max.py b/sfa/managers/aggregate_manager_max.py index 64fd0ecf..88df708f 100644 --- a/sfa/managers/aggregate_manager_max.py +++ b/sfa/managers/aggregate_manager_max.py @@ -4,6 +4,7 @@ import re #from sfa.util.faults import * from sfa.util.sfalogging import logger +from sfa.util.sfatime import SFATIME_FORMAT from sfa.util.config import Config from sfa.util.callids import Callids from sfa.util.version import version_core @@ -56,7 +57,7 @@ class AggregateManagerMax (AggregateManager): # save request RSpec xml content to a tmp file def save_rspec_to_file(self, rspec): path = AggregateManagerMax.RSPEC_TMP_FILE_PREFIX + "_" + \ - time.strftime('%Y%m%dT%H:%M:%S', time.gmtime(time.time())) +".xml" + time.strftime(SFATIME_FORMAT, time.gmtime(time.time())) +".xml" file = open(path, "w") file.write(rspec) file.close() diff --git a/sfa/managers/driver.py b/sfa/managers/driver.py index 0e8b71dc..1985e0ee 100644 --- a/sfa/managers/driver.py +++ b/sfa/managers/driver.py @@ -78,7 +78,8 @@ class Driver: # answer to ListResources # returns : advertisment rspec (xml string) - def list_resources (self, version=None, options={}): + def list_resources (self, version=None, options=None): + if options is None: options={} return "dummy Driver.list_resources needs to be redefined" # the answer to Describe on a slice or a set of the slivers in a slice @@ -97,40 +98,48 @@ class Driver: # ... # ] #} - def describe (self, urns, version, options={}): + def describe (self, urns, version, options=None): + if options is None: options={} return "dummy Driver.describe needs to be redefined" # the answer to Allocate on a given slicei or a set of the slivers in a slice # returns: same struct as for describe. - def allocate (self, urn, rspec_string, expiration, options={}): + def allocate (self, urn, rspec_string, expiration, options=None): + if options is None: options={} return "dummy Driver.allocate needs to be redefined" # the answer to Provision on a given slice or a set of the slivers in a slice # returns: same struct as for describe. - def provision(self, urns, options={}): + def provision(self, urns, options=None): + if options is None: options={} return "dummy Driver.provision needs to be redefined" # the answer to PerformOperationalAction on a given slice or a set of the slivers in a slice # returns: struct containing "geni_slivers" list of the struct returned by describe. - def perform_operational_action (self, urns, action, options={}): + def perform_operational_action (self, urns, action, options=None): + if options is None: options={} return "dummy Driver.perform_operational_action needs to be redefined" # the answer to Status on a given slice or a set of the slivers in a slice # returns: struct containing "geni_urn" and "geni_slivers" list of the struct returned by describe. - def status (self, urns, options={}): + def status (self, urns, options=None): + if options is None: options={} return "dummy Driver.status needs to be redefined" # the answer to Renew on a given slice or a set of the slivers in a slice # returns: struct containing "geni_slivers" list of the struct returned by describe. - def renew (self, urns, expiration_time, options={}): + def renew (self, urns, expiration_time, options=None): + if options is None: options={} return "dummy Driver.renew needs to be redefined" # the answer to Delete on a given slice # returns: struct containing "geni_slivers" list of the struct returned by describe. - def delete(self, urns, options={}): + def delete(self, urns, options=None): + if options is None: options={} return "dummy Driver.delete needs to be redefined" # the answer to Shutdown on a given slice # returns: boolean - def shutdown (self, xrn, options={}): + def shutdown (self, xrn, options=None): + if options is None: options={} return False diff --git a/sfa/managers/registry_manager.py b/sfa/managers/registry_manager.py index 7306380d..6d7bb6dd 100644 --- a/sfa/managers/registry_manager.py +++ b/sfa/managers/registry_manager.py @@ -33,7 +33,7 @@ from sqlalchemy.orm.collections import InstrumentedList # * write operations (register, update) need e.g. # 'researcher' or 'pi' to be set - reg-* are just ignored # -# the 'normalize' helper functions below aim at ironing this out +# the '_normalize_input' helper functions below aim at ironing this out # however in order to break as few code as possible we essentially make sure that *both* fields are set # upon entering the write methods (so again register and update) for legacy, as some driver code # might depend on the presence of, say, 'researcher' @@ -48,16 +48,21 @@ def _normalize_input (record, reg_key, driver_key): # and issue a warning if they were both set and different # as we're overwriting some user data here if driver_key in record: - logger.warning ("normalize_input_researcher: incoming record has both values, using reg-researchers") + logger.warning ("normalize_input: incoming record has both values, using %s"%reg_key) record[driver_key]=record[reg_key] # we only have one key set, duplicate for the other one elif driver_key in record: - logger.warning ("normalize_input_researcher: you should use '%s' instead ot '%s'"%(reg_key,driver_key)) + logger.warning ("normalize_input: you should use '%s' instead of '%s'"%(reg_key,driver_key)) record[reg_key]=record[driver_key] def normalize_input_record (record): _normalize_input (record, 'reg-researchers','researcher') _normalize_input (record, 'reg-pis','pi') + _normalize_input (record, 'reg-keys','keys') + # xxx the keys thing could use a little bit more attention: + # some parts of the code are using 'keys' while they should use 'reg-keys' + # but I run out of time for now + if 'reg-keys' in record: record['keys']=record['reg-keys'] return record class RegistryManager: @@ -71,8 +76,7 @@ class RegistryManager: if hrn != api.hrn]) xrn=Xrn(api.hrn,type='authority') return version_core({'interface':'registry', - 'sfa': 2, - 'geni_api': 2, + 'sfa': 3, 'hrn':xrn.get_hrn(), 'urn':xrn.get_urn(), 'peers':peers}) @@ -237,7 +241,8 @@ class RegistryManager: return records - def List (self, api, xrn, origin_hrn=None, options={}): + def List (self, api, xrn, origin_hrn=None, options=None): + if options is None: options={} dbsession=api.dbsession() # load all know registry names into a prefix tree and attempt to find # the longest matching prefix @@ -357,11 +362,10 @@ class RegistryManager: if not record.gid: uuid = create_uuid() pkey = Keypair(create=True) - if getattr(record,'keys',None): - pub_key=record.keys + pub_key=getattr(record,'reg-keys',None) + if pub_key is not None: # use only first key in record - if isinstance(record.keys, types.ListType): - pub_key = record.keys[0] + if pub_key and isinstance(pub_key, types.ListType): pub_key = pub_key[0] pkey = convert_public_key(pub_key) gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey) @@ -388,9 +392,12 @@ class RegistryManager: elif isinstance (record, RegUser): # create RegKey objects for incoming keys - if hasattr(record,'keys'): - logger.debug ("creating %d keys for user %s"%(len(record.keys),record.hrn)) - record.reg_keys = [ RegKey (key) for key in record.keys ] + if hasattr(record,'reg-keys'): + keys=getattr(record,'reg-keys') + # some people send the key as a string instead of a list of strings + if isinstance(keys,types.StringTypes): keys=[keys] + logger.debug ("creating %d keys for user %s"%(len(keys),record.hrn)) + record.reg_keys = [ RegKey (key) for key in keys ] # update testbed-specific data if needed pointer = api.driver.register (record.__dict__, hrn, pub_key) diff --git a/sfa/managers/slice_manager.py b/sfa/managers/slice_manager.py index 05b0f1ee..2a99b6f4 100644 --- a/sfa/managers/slice_manager.py +++ b/sfa/managers/slice_manager.py @@ -508,7 +508,8 @@ class SliceManager: multiclient.get_results() return 1 - def Shutdown(self, api, xrn, creds, options={}): + def Shutdown(self, api, xrn, creds, options=None): + if options is None: options={} xrn = Xrn(xrn) # get the callers hrn valid_cred = api.auth.checkCredentials(creds, 'stopslice', xrn.hrn)[0] diff --git a/sfa/methods/Allocate.py b/sfa/methods/Allocate.py index ff543b73..a7dc8f98 100644 --- a/sfa/methods/Allocate.py +++ b/sfa/methods/Allocate.py @@ -36,15 +36,17 @@ class Allocate(Method): def call(self, xrn, creds, rspec, options): xrn = Xrn(xrn, type='slice') - self.api.logger.info("interface: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, xrn.get_hrn(), self.name)) - (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) # Find the valid credentials - valid_creds = self.api.auth.checkCredentials(creds, 'createsliver', xrn.get_hrn(), speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'createsliver', xrn.get_hrn(), options=options) + the_credential = Credential(cred=valid_creds[0]) + # use the expiration from the first valid credential to determine when # the slivers should expire. - expiration = datetime_to_string(Credential(cred=valid_creds[0]).expiration) + expiration = datetime_to_string(the_credential.expiration) + self.api.logger.debug("Allocate, received expiration from credential: %s"%expiration) + # make sure request is not empty slivers = RSpec(rspec).version.get_nodes_with_slivers() if not slivers: @@ -56,12 +58,14 @@ class Allocate(Method): elif self.api.interface in ['slicemgr']: chain_name = 'FORWARD-INCOMING' self.api.logger.debug("Allocate: sfatables on chain %s"%chain_name) - origin_hrn = Credential(cred=valid_creds[0]).get_gid_caller().get_hrn() - self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, xrn, self.name)) - rspec = run_sfatables(chain_name, xrn.get_hrn(), origin_hrn, rspec) + actual_caller_hrn = the_credential.actual_caller_hrn() + self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, actual_caller_hrn, xrn.get_hrn(), self.name)) + rspec = run_sfatables(chain_name, xrn.get_hrn(), actual_caller_hrn, rspec) slivers = RSpec(rspec).version.get_nodes_with_slivers() if not slivers: raise SfatablesRejected(slice_xrn) + # pass this to the driver code in case they need it + options['actual_caller_hrn'] = actual_caller_hrn result = self.api.manager.Allocate(self.api, xrn.get_urn(), creds, rspec, expiration, options) return result diff --git a/sfa/methods/Delete.py b/sfa/methods/Delete.py index eed8a398..593de28f 100644 --- a/sfa/methods/Delete.py +++ b/sfa/methods/Delete.py @@ -24,10 +24,9 @@ class Delete(Method): returns = Parameter(int, "1 if successful") def call(self, xrns, creds, options): - (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) - valid_creds = self.api.auth.checkCredentials(creds, 'deletesliver', xrns, - check_sliver_callback = self.api.driver.check_sliver_credentials, - speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'deletesliver', xrns, + check_sliver_callback = self.api.driver.check_sliver_credentials, + options=options) #log the call origin_hrn = Credential(cred=valid_creds[0]).get_gid_caller().get_hrn() diff --git a/sfa/methods/Describe.py b/sfa/methods/Describe.py index ec604895..018f8034 100644 --- a/sfa/methods/Describe.py +++ b/sfa/methods/Describe.py @@ -36,10 +36,9 @@ class Describe(Method): options['geni_rspec_version'] = options['rspec_version'] else: raise SfaInvalidArgument('Must specify an rspec version option. geni_rspec_version cannot be null') - (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) - valid_creds = self.api.auth.checkCredentials(creds, 'listnodes', urns, \ - check_sliver_callback = self.api.driver.check_sliver_credentials, - speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'listnodes', urns, + check_sliver_callback = self.api.driver.check_sliver_credentials, + options=options) # get hrn of the original caller origin_hrn = options.get('origin_hrn', None) diff --git a/sfa/methods/GetVersion.py b/sfa/methods/GetVersion.py index cb682e44..f043992e 100644 --- a/sfa/methods/GetVersion.py +++ b/sfa/methods/GetVersion.py @@ -15,6 +15,7 @@ class GetVersion(Method): returns = Parameter(dict, "Version information") # API v2 specifies options is optional, so.. - def call(self, options={}): + def call(self, options=None): + if options is None: options={} self.api.logger.info("interface: %s\tmethod-name: %s" % (self.api.interface, self.name)) return self.api.manager.GetVersion(self.api, options) diff --git a/sfa/methods/List.py b/sfa/methods/List.py index d53a0a5e..83d7a6e0 100644 --- a/sfa/methods/List.py +++ b/sfa/methods/List.py @@ -25,7 +25,8 @@ class List(Method): # xxx used to be [SfaRecord] returns = [Parameter(dict, "registry record")] - def call(self, xrn, creds, options={}): + def call(self, xrn, creds, options=None): + if options is None: options={} hrn, type = urn_to_hrn(xrn) valid_creds = self.api.auth.checkCredentials(creds, 'list') diff --git a/sfa/methods/ListResources.py b/sfa/methods/ListResources.py index a9ffe760..33777fd7 100644 --- a/sfa/methods/ListResources.py +++ b/sfa/methods/ListResources.py @@ -33,10 +33,8 @@ class ListResources(Method): else: raise SfaInvalidArgument('Must specify an rspec version option. geni_rspec_version cannot be null') - (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) - # Find the valid credentials - valid_creds = self.api.auth.checkCredentials(creds, 'listnodes', speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'listnodes', options=options) # get hrn of the original caller origin_hrn = options.get('origin_hrn', None) diff --git a/sfa/methods/PerformOperationalAction.py b/sfa/methods/PerformOperationalAction.py index a2635142..41bf58fd 100644 --- a/sfa/methods/PerformOperationalAction.py +++ b/sfa/methods/PerformOperationalAction.py @@ -34,9 +34,9 @@ class PerformOperationalAction(Method): (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) # Find the valid credentials - valid_creds = self.api.auth.checkCredentials(creds, 'createsliver', xrns, - check_sliver_callback = self.api.driver.check_sliver_credentials, - speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'createsliver', xrns, + check_sliver_callback = self.api.driver.check_sliver_credentials, + options=options) origin_hrn = Credential(cred=valid_creds[0]).get_gid_caller().get_hrn() self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, xrns, self.name)) result = self.api.manager.PerformOperationalAction(self.api, xrns, creds, action, options) diff --git a/sfa/methods/Provision.py b/sfa/methods/Provision.py index a3fd0feb..7177854d 100644 --- a/sfa/methods/Provision.py +++ b/sfa/methods/Provision.py @@ -31,12 +31,10 @@ class Provision(Method): def call(self, xrns, creds, options): self.api.logger.info("interface: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, xrns, self.name)) - (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) - # Find the valid credentials - valid_creds = self.api.auth.checkCredentials(creds, 'createsliver', xrns, - check_sliver_callback = self.api.driver.check_sliver_credentials, - speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'createsliver', xrns, + check_sliver_callback = self.api.driver.check_sliver_credentials, + options=options) origin_hrn = Credential(cred=valid_creds[0]).get_gid_caller().get_hrn() self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, xrns, self.name)) result = self.api.manager.Provision(self.api, xrns, creds, options) diff --git a/sfa/methods/Renew.py b/sfa/methods/Renew.py index 7b470f10..0a7ac1af 100644 --- a/sfa/methods/Renew.py +++ b/sfa/methods/Renew.py @@ -3,7 +3,7 @@ import datetime from sfa.util.faults import InsufficientRights from sfa.util.xrn import urn_to_hrn from sfa.util.method import Method -from sfa.util.sfatime import utcparse +from sfa.util.sfatime import utcparse, add_datetime from sfa.trust.credential import Credential @@ -14,7 +14,7 @@ class Renew(Method): Renews the resources in the specified slice or slivers by extending the lifetime. - @param surn ([string]) List of URNs of to renew + @param urns ([string]) List of URNs of to renew @param credentials ([string]) of credentials @param expiration_time (string) requested time of expiration @param options (dict) options @@ -30,21 +30,39 @@ class Renew(Method): def call(self, urns, creds, expiration_time, options): - self.api.logger.info("interface: %s\ttarget-hrn: %s\tcaller-creds: %s\tmethod-name: %s"%(self.api.interface, urns, creds, self.name)) - (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) - # Find the valid credentials - valid_creds = self.api.auth.checkCredentials(creds, 'renewsliver', urns, - check_sliver_callback = self.api.driver.check_sliver_credentials, - speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'renewsliver', urns, + check_sliver_callback = self.api.driver.check_sliver_credentials, + options=options) + the_credential = Credential(cred=valid_creds[0]) + actual_caller_hrn = the_credential.actual_caller_hrn() + self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-urns: %s\texpiration:%s\tmethod-name: %s"%\ + (self.api.interface, actual_caller_hrn, urns, expiration_time,self.name)) + + + # extend as long as possible : take the min of requested and now+SFA_MAX_SLICE_RENEW + if options.get('geni_extend_alap'): + # ignore requested time and set to max + expiration_time = add_datetime(datetime.datetime.utcnow(), days=int(self.api.config.SFA_MAX_SLICE_RENEW)) # Validate that the time does not go beyond the credential's expiration time - requested_time = utcparse(expiration_time) + requested_expire = utcparse(expiration_time) + self.api.logger.info("requested_expire = %s"%requested_expire) + credential_expire = the_credential.get_expiration() + self.api.logger.info("credential_expire = %s"%credential_expire) max_renew_days = int(self.api.config.SFA_MAX_SLICE_RENEW) - if requested_time > Credential(cred=valid_creds[0]).get_expiration(): - raise InsufficientRights('Renewsliver: Credential expires before requested expiration time') - if requested_time > datetime.datetime.utcnow() + datetime.timedelta(days=max_renew_days): - raise Exception('Cannot renew > %s days from now' % max_renew_days) - return self.api.manager.Renew(self.api, urns, creds, expiration_time, options) + max_expire = datetime.datetime.utcnow() + datetime.timedelta (days=max_renew_days) + if requested_expire > credential_expire: + # used to throw an InsufficientRights exception here, this was not right + self.api.logger.warning("Requested expiration %s, after credential expiration (%s) -> trimming to the latter/sooner"%\ + (requested_expire, credential_expire)) + requested_expire = credential_expire + if requested_expire > max_expire: + # likewise + self.api.logger.warning("Requested expiration %s, after maximal expiration %s days (%s) -> trimming to the latter/sooner"%\ + (requested_expire, self.api.config.SFA_MAX_SLICE_RENEW,max_expire)) + requested_expire = max_expire + + return self.api.manager.Renew(self.api, urns, creds, requested_expire, options) diff --git a/sfa/methods/Resolve.py b/sfa/methods/Resolve.py index f3a6e671..dc34f759 100644 --- a/sfa/methods/Resolve.py +++ b/sfa/methods/Resolve.py @@ -30,7 +30,8 @@ class Resolve(Method): # xxx used to be [SfaRecord] returns = [Parameter(dict, "registry record")] - def call(self, xrns, creds, options={}): + def call(self, xrns, creds, options=None): + if options is None: options={} # use details=False by default, only when explicitly specified do we want # to mess with the testbed details if 'details' in options: details=options['details'] diff --git a/sfa/methods/Shutdown.py b/sfa/methods/Shutdown.py index 3eee8785..f6f1841f 100644 --- a/sfa/methods/Shutdown.py +++ b/sfa/methods/Shutdown.py @@ -20,7 +20,7 @@ class Shutdown(Method): def call(self, xrn, creds): valid_creds = self.api.auth.checkCredentials(creds, 'stopslice', xrn, - check_sliver_callback = self.api.driver.check_sliver_credentials) + check_sliver_callback = self.api.driver.check_sliver_credentials) #log the call origin_hrn = Credential(cred=valid_creds[0]).get_gid_caller().get_hrn() self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, xrn, self.name)) diff --git a/sfa/methods/Status.py b/sfa/methods/Status.py index 36da58e1..68d928e6 100644 --- a/sfa/methods/Status.py +++ b/sfa/methods/Status.py @@ -19,10 +19,9 @@ class Status(Method): returns = Parameter(dict, "Status details") def call(self, xrns, creds, options): - (speaking_for, _) = urn_to_hrn(options.get('geni_speaking_for')) - valid_creds = self.api.auth.checkCredentials(creds, 'sliverstatus', xrns, - check_sliver_callback = self.api.driver.check_sliver_credentials, - speaking_for_hrn=speaking_for) + valid_creds = self.api.auth.checkCredentialsSpeaksFor(creds, 'sliverstatus', xrns, + check_sliver_callback = self.api.driver.check_sliver_credentials, + options=options) self.api.logger.info("interface: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, xrns, self.name)) return self.api.manager.Status(self.api, xrns, creds, options) diff --git a/sfa/nitos/nitosaggregate.py b/sfa/nitos/nitosaggregate.py index bb7c56d6..832a2c7c 100644 --- a/sfa/nitos/nitosaggregate.py +++ b/sfa/nitos/nitosaggregate.py @@ -68,7 +68,9 @@ class NitosAggregate: - def get_nodes(self, slice_xrn, slice=None,slivers={}, options={}): + def get_nodes(self, slice_xrn, slice=None,slivers=None, options=None): + if slivers is None: slivers={} + if options is None: options={} # if we are dealing with a slice that has no node just return # and empty list if slice_xrn: @@ -126,8 +128,9 @@ class NitosAggregate: rspec_nodes.append(rspec_node) return rspec_nodes - def get_leases_and_channels(self, slice=None, slice_xrn=None, options={}): - + def get_leases_and_channels(self, slice=None, slice_xrn=None, options=None): + + if options is None: options={} slices = self.driver.shell.getSlices({}, []) nodes = self.driver.shell.getNodes({}, []) leases = self.driver.shell.getReservedNodes({}, []) @@ -216,8 +219,9 @@ class NitosAggregate: return (rspec_leases, rspec_channels) - def get_channels(self, slice=None, options={}): - + def get_channels(self, slice=None, options=None): + if options is None: options={} + all_channels = self.driver.shell.getChannels({}, []) channels = [] if slice: @@ -245,7 +249,8 @@ class NitosAggregate: - def get_rspec(self, slice_xrn=None, version = None, options={}): + def get_rspec(self, slice_xrn=None, version = None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) diff --git a/sfa/nitos/nitosslices.py b/sfa/nitos/nitosslices.py index 3eac8aa3..875a5a90 100644 --- a/sfa/nitos/nitosslices.py +++ b/sfa/nitos/nitosslices.py @@ -153,7 +153,8 @@ class NitosSlices: - def verify_slice(self, slice_hrn, slice_record, sfa_peer, options={}): + def verify_slice(self, slice_hrn, slice_record, sfa_peer, options=None): + if options is None: options={} slicename = hrn_to_nitos_slicename(slice_hrn) slices = self.driver.shell.getSlices({}, []) slices = self.driver.filter_nitos_results(slices, {'slice_name': slicename}) @@ -168,7 +169,8 @@ class NitosSlices: return slice - def verify_users(self, slice_hrn, slice_record, users, sfa_peer, options={}): + def verify_users(self, slice_hrn, slice_record, users, sfa_peer, options=None): + if options is None: options={} # get slice info slicename = hrn_to_nitos_slicename(slice_hrn) slices = self.driver.shell.getSlices({}, []) @@ -204,7 +206,8 @@ class NitosSlices: return added_users - def verify_keys(self, persons, users, options={}): + def verify_keys(self, persons, users, options=None): + if options is None: options={} # existing keys key_ids = [] for person in persons: diff --git a/sfa/openstack/image.py b/sfa/openstack/image.py index 54aaf502..555b1b9b 100644 --- a/sfa/openstack/image.py +++ b/sfa/openstack/image.py @@ -4,7 +4,8 @@ from sfa.rspecs.elements.disk_image import DiskImage class Image: - def __init__(self, image={}): + def __init__(self, image=None): + if image is None: image={} self.id = None self.container_format = None self.kernel_id = None diff --git a/sfa/openstack/nova_driver.py b/sfa/openstack/nova_driver.py index e36946e6..39ea2f91 100644 --- a/sfa/openstack/nova_driver.py +++ b/sfa/openstack/nova_driver.py @@ -355,23 +355,27 @@ class NovaDriver(Driver): return {} # first 2 args are None in case of resource discovery - def list_resources (self, version=None, options={}): + def list_resources (self, version=None, options=None): + if options is None: options={} aggregate = OSAggregate(self) rspec = aggregate.list_resources(version=version, options=options) return rspec - def describe(self, urns, version=None, options={}): + def describe(self, urns, version=None, options=None): + if options is None: options={} aggregate = OSAggregate(self) return aggregate.describe(urns, version=version, options=options) - def status (self, urns, options={}): + def status (self, urns, options=None): + if options is None: options={} aggregate = OSAggregate(self) desc = aggregate.describe(urns) status = {'geni_urn': desc['geni_urn'], 'geni_slivers': desc['geni_slivers']} return status - def allocate (self, urn, rspec_string, expiration, options={}): + def allocate (self, urn, rspec_string, expiration, options=None): + if options is None: options={} xrn = Xrn(urn) aggregate = OSAggregate(self) @@ -401,7 +405,8 @@ class NovaDriver(Driver): return aggregate.describe(urns=[urn], version=rspec.version) - def provision(self, urns, options={}): + def provision(self, urns, options=None): + if options is None: options={} # update sliver allocation states and set them to geni_provisioned aggregate = OSAggregate(self) instances = aggregate.get_instances(urns) @@ -415,7 +420,8 @@ class NovaDriver(Driver): rspec_version = version_manager.get_version(options['geni_rspec_version']) return self.describe(urns, rspec_version, options=options) - def delete (self, urns, options={}): + def delete (self, urns, options=None): + if options is None: options={} # collect sliver ids so we can update sliver allocation states after # we remove the slivers. aggregate = OSAggregate(self) @@ -441,11 +447,13 @@ class NovaDriver(Driver): 'geni_expires': None}) return geni_slivers - def renew (self, urns, expiration_time, options={}): + def renew (self, urns, expiration_time, options=None): + if options is None: options={} description = self.describe(urns, None, options) return description['geni_slivers'] - def perform_operational_action (self, urns, action, options={}): + def perform_operational_action (self, urns, action, options=None): + if options is None: options={} aggregate = OSAggregate(self) action = action.lower() if action == 'geni_start': @@ -474,7 +482,8 @@ class NovaDriver(Driver): geni_slivers = self.describe(urns, None, options)['geni_slivers'] return geni_slivers - def shutdown(self, xrn, options={}): + def shutdown(self, xrn, options=None): + if options is None: options={} xrn = OSXrn(xrn=xrn, type='slice') tenant_name = xrn.get_tenant_name() name = xrn.get_slicename() diff --git a/sfa/openstack/osaggregate.py b/sfa/openstack/osaggregate.py index d6d73677..2b653997 100644 --- a/sfa/openstack/osaggregate.py +++ b/sfa/openstack/osaggregate.py @@ -8,7 +8,7 @@ import time from collections import defaultdict from nova.exception import ImageNotFound from nova.api.ec2.cloud import CloudController -from sfa.util.faults import SfaAPIError, SliverDoesNotExist +from sfa.util.faults import SliverDoesNotExist from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch from sfa.rspecs.rspec import RSpec from sfa.rspecs.elements.hardware_type import HardwareType @@ -58,7 +58,8 @@ class OSAggregate: zones = [zone.name for zone in zones] return zones - def list_resources(self, version=None, options={}): + def list_resources(self, version=None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version(version.type, version.version, 'ad') @@ -67,7 +68,8 @@ class OSAggregate: rspec.version.add_nodes(nodes) return rspec.toxml() - def describe(self, urns, version=None, options={}): + def describe(self, urns, version=None, options=None): + if options is None: options={} # update nova connection tenant_name = OSXrn(xrn=urns[0], type='slice').get_tenant_name() self.driver.shell.nova_manager.connect(tenant=tenant_name) @@ -211,7 +213,8 @@ class OSAggregate: 'storage': str(instance.disk)}) return sliver - def instance_to_geni_sliver(self, instance, sliver_allocations = {}): + def instance_to_geni_sliver(self, instance, sliver_allocations=None): + if sliver_allocations is None: sliver_allocations={} sliver_hrn = '%s.%s' % (self.driver.hrn, instance.id) sliver_id = Xrn(sliver_hrn, type='sliver').urn @@ -302,7 +305,8 @@ class OSAggregate: return key_name - def create_security_group(self, slicename, fw_rules=[]): + def create_security_group(self, slicename, fw_rules=None): + if fw_rules is None: fw_rules=[] # use default group by default group_name = 'default' if isinstance(fw_rules, list) and fw_rules: diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py index cae6495b..28766948 100644 --- a/sfa/planetlab/plaggregate.py +++ b/sfa/planetlab/plaggregate.py @@ -31,7 +31,8 @@ class PlAggregate: def __init__(self, driver): self.driver = driver - def get_nodes(self, options={}): + def get_nodes(self, options=None): + if options is None: options={} filter = {'peer_id': None} geni_available = options.get('geni_available') if geni_available == True: @@ -40,13 +41,15 @@ class PlAggregate: return nodes - def get_sites(self, filter={}): + def get_sites(self, filter=None): + if filter is None: filter={} sites = {} for site in self.driver.shell.GetSites(filter): sites[site['site_id']] = site return sites - def get_interfaces(self, filter={}): + def get_interfaces(self, filter=None): + if filter is None: filter={} interfaces = {} for interface in self.driver.shell.GetInterfaces(filter): iface = Interface() @@ -98,20 +101,23 @@ class PlAggregate: return links - def get_node_tags(self, filter={}): + def get_node_tags(self, filter=None): + if filter is None: filter={} node_tags = {} for node_tag in self.driver.shell.GetNodeTags(filter): node_tags[node_tag['node_tag_id']] = node_tag return node_tags - def get_pl_initscripts(self, filter={}): + def get_pl_initscripts(self, filter=None): + if filter is None: filter={} pl_initscripts = {} filter.update({'enabled': True}) for initscript in self.driver.shell.GetInitScripts(filter): pl_initscripts[initscript['initscript_id']] = initscript return pl_initscripts - def get_slivers(self, urns, options={}): + def get_slivers(self, urns, options=None): + if options is None: options={} names = set() slice_ids = set() node_ids = [] @@ -193,7 +199,9 @@ class PlAggregate: slivers.append(node) return slivers - def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}): + def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=None, grain=None, options=None): + if pl_initscripts is None: pl_initscripts=[] + if options is None: options={} rspec_node = NodeElement() # xxx how to retrieve site['login_base'] site=sites[node['site_id']] @@ -286,7 +294,8 @@ class PlAggregate: tags_dict[tag['node_id']] = tag return tags_dict - def get_slice_nodes(self, slice, options={}): + def get_slice_nodes(self, slice, options=None): + if options is None: options={} nodes_dict = {} filter = {'peer_id': None} tags_filter = {} @@ -304,7 +313,8 @@ class PlAggregate: nodes_dict[node['node_id']] = node return nodes_dict - def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}): + def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations=None): + if sliver_allocations is None: sliver_allocations={} if rspec_node['sliver_id'] in sliver_allocations: # set sliver allocation and operational status sliver_allocation = sliver_allocations[rspec_node['sliver_id']] @@ -333,7 +343,8 @@ class PlAggregate: } return geni_sliver - def get_leases(self, slice=None, options={}): + def get_leases(self, slice=None, options=None): + if options is None: options={} now = int(time.time()) filter={} @@ -370,7 +381,8 @@ class PlAggregate: return rspec_leases - def list_resources(self, version = None, options={}): + def list_resources(self, version = None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) @@ -410,7 +422,8 @@ class PlAggregate: return rspec.toxml() - def describe(self, urns, version=None, options={}): + def describe(self, urns, version=None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version(version.type, version.version, 'manifest') diff --git a/sfa/planetlab/pldriver.py b/sfa/planetlab/pldriver.py index 1c59fc9e..53f12563 100644 --- a/sfa/planetlab/pldriver.py +++ b/sfa/planetlab/pldriver.py @@ -627,23 +627,27 @@ class PlDriver (Driver): return {} # first 2 args are None in case of resource discovery - def list_resources (self, version=None, options={}): + def list_resources (self, version=None, options=None): + if options is None: options={} aggregate = PlAggregate(self) rspec = aggregate.list_resources(version=version, options=options) return rspec - def describe(self, urns, version, options={}): + def describe(self, urns, version, options=None): + if options is None: options={} aggregate = PlAggregate(self) return aggregate.describe(urns, version=version, options=options) - def status (self, urns, options={}): + def status (self, urns, options=None): + if options is None: options={} aggregate = PlAggregate(self) desc = aggregate.describe(urns, version='GENI 3') status = {'geni_urn': desc['geni_urn'], 'geni_slivers': desc['geni_slivers']} return status - def allocate (self, urn, rspec_string, expiration, options={}): + def allocate (self, urn, rspec_string, expiration, options=None): + if options is None: options={} xrn = Xrn(urn) aggregate = PlAggregate(self) slices = PlSlices(self) @@ -680,7 +684,8 @@ class PlDriver (Driver): return aggregate.describe([xrn.get_urn()], version=rspec.version) - def provision(self, urns, options={}): + def provision(self, urns, options=None): + if options is None: options={} # update users slices = PlSlices(self) aggregate = PlAggregate(self) @@ -715,7 +720,8 @@ class PlDriver (Driver): rspec_version = version_manager.get_version(options['geni_rspec_version']) return self.describe(urns, rspec_version, options=options) - def delete(self, urns, options={}): + def delete(self, urns, options=None): + if options is None: options={} # collect sliver ids so we can update sliver allocation states after # we remove the slivers. aggregate = PlAggregate(self) @@ -754,7 +760,8 @@ class PlDriver (Driver): 'geni_expires': datetime_to_string(utcparse(sliver['expires']))}) return geni_slivers - def renew (self, urns, expiration_time, options={}): + def renew (self, urns, expiration_time, options=None): + if options is None: options={} aggregate = PlAggregate(self) slivers = aggregate.get_slivers(urns) if not slivers: @@ -767,7 +774,8 @@ class PlDriver (Driver): return description['geni_slivers'] - def perform_operational_action (self, urns, action, options={}): + def perform_operational_action (self, urns, action, options=None): + if options is None: options={} # MyPLC doesn't support operational actions. Lets pretend like it # supports start, but reject everything else. action = action.lower() @@ -787,7 +795,8 @@ class PlDriver (Driver): return geni_slivers # set the 'enabled' tag to 0 - def shutdown (self, xrn, options={}): + def shutdown (self, xrn, options=None): + if options is None: options={} hrn, _ = urn_to_hrn(xrn) top_auth_hrn = top_auth(hrn) site_hrn = '.'.join(hrn.split('.')[:-1]) diff --git a/sfa/planetlab/plslices.py b/sfa/planetlab/plslices.py index b6cec880..1226f12f 100644 --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@ -322,7 +322,9 @@ class PlSlices: - def verify_site(self, slice_xrn, slice_record={}, sfa_peer=None, options={}): + def verify_site(self, slice_xrn, slice_record=None, sfa_peer=None, options=None): + if slice_record is None: slice_record={} + if options is None: options={} (slice_hrn, type) = urn_to_hrn(slice_xrn) top_auth_hrn = top_auth(slice_hrn) site_hrn = '.'.join(slice_hrn.split('.')[:-1]) @@ -367,7 +369,8 @@ class PlSlices: return site - def verify_slice(self, slice_hrn, slice_record, sfa_peer, expiration, options={}): + def verify_slice(self, slice_hrn, slice_record, sfa_peer, expiration, options=None): + if options is None: options={} top_auth_hrn = top_auth(slice_hrn) site_hrn = '.'.join(slice_hrn.split('.')[:-1]) slice_part = slice_hrn.split('.')[-1] @@ -416,7 +419,8 @@ class PlSlices: return self.driver.shell.GetSlices(int(slice['slice_id']))[0] - def verify_persons(self, slice_hrn, slice_record, users, sfa_peer, options={}): + def verify_persons(self, slice_hrn, slice_record, users, sfa_peer, options=None): + if options is None: options={} top_auth_hrn = top_auth(slice_hrn) site_hrn = '.'.join(slice_hrn.split('.')[:-1]) slice_part = slice_hrn.split('.')[-1] @@ -504,7 +508,8 @@ class PlSlices: return persons_to_add - def verify_keys(self, persons_to_verify_keys, options={}): + def verify_keys(self, persons_to_verify_keys, options=None): + if options is None: options={} # we only add keys that comes from sfa to persons in PL for person_id in persons_to_verify_keys: person_sfa_keys = persons_to_verify_keys[person_id].get('keys', []) @@ -518,7 +523,8 @@ class PlSlices: self.driver.shell.AddPersonKey(int(person_id), key) - def verify_slice_attributes(self, slice, requested_slice_attributes, options={}, admin=False): + def verify_slice_attributes(self, slice, requested_slice_attributes, options=None, admin=False): + if options is None: options={} append = options.get('append', True) # get list of attributes users ar able to manage filter = {'category': '*slice*'} diff --git a/sfa/rspecs/elements/element.py b/sfa/rspecs/elements/element.py index 7f79e810..36ad12f7 100644 --- a/sfa/rspecs/elements/element.py +++ b/sfa/rspecs/elements/element.py @@ -2,7 +2,8 @@ class Element(dict): fields = {} - def __init__(self, fields={}, element=None, keys=None): + def __init__(self, fields=None, element=None, keys=None): + if fields is None: fields={} self.element = element dict.__init__(self, dict.fromkeys(self.fields)) if not keys: diff --git a/sfa/rspecs/elements/versions/iotlabv1Lease.py b/sfa/rspecs/elements/versions/iotlabv1Lease.py index eebb1ae2..bfc503aa 100644 --- a/sfa/rspecs/elements/versions/iotlabv1Lease.py +++ b/sfa/rspecs/elements/versions/iotlabv1Lease.py @@ -29,7 +29,8 @@ class Iotlabv1Lease: @staticmethod - def get_leases(xml, filter={}): + def get_leases(xml, filter=None): + if filter is None: filter={} xpath = '//lease%s | //default:lease%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) lease_elems = xml.xpath(xpath) return Iotlabv1Lease.get_lease_objs(lease_elems) diff --git a/sfa/rspecs/elements/versions/iotlabv1Node.py b/sfa/rspecs/elements/versions/iotlabv1Node.py index af6fe2a4..49f7cfd2 100644 --- a/sfa/rspecs/elements/versions/iotlabv1Node.py +++ b/sfa/rspecs/elements/versions/iotlabv1Node.py @@ -160,14 +160,16 @@ class Iotlabv1Node: return node_elems @staticmethod - def get_nodes(xml, filter={}): + def get_nodes(xml, filter=None): + if filter is None: filter={} xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), \ XpathFilter.xpath(filter)) node_elems = xml.xpath(xpath) return Iotlabv1Node.get_node_objs(node_elems) @staticmethod - def get_nodes_with_slivers(xml, sliver_filter={}): + def get_nodes_with_slivers(xml, sliver_filter=None): + if sliver_filter is None: sliver_filter={} xpath = '//node[count(sliver)>0] | \ //default:node[count(default:sliver) > 0]' diff --git a/sfa/rspecs/elements/versions/iotlabv1Sliver.py b/sfa/rspecs/elements/versions/iotlabv1Sliver.py index f26ace6b..0f9fb014 100644 --- a/sfa/rspecs/elements/versions/iotlabv1Sliver.py +++ b/sfa/rspecs/elements/versions/iotlabv1Sliver.py @@ -35,7 +35,8 @@ class Iotlabv1Sliver: for (key, value) in attrib_dict.items(): attrib_elem.set(key, value) @staticmethod - def get_slivers(xml, filter={}): + def get_slivers(xml, filter=None): + if filter is None: filter={} xpath = './default:sliver | ./sliver' sliver_elems = xml.xpath(xpath) @@ -54,5 +55,6 @@ class Iotlabv1Sliver: return slivers @staticmethod - def get_sliver_attributes(xml, filter={}): - return [] \ No newline at end of file + def get_sliver_attributes(xml, filter=None): + if filter is None: filter={} + return [] diff --git a/sfa/rspecs/elements/versions/nitosv1Channel.py b/sfa/rspecs/elements/versions/nitosv1Channel.py index 60582e35..cf4a5f61 100644 --- a/sfa/rspecs/elements/versions/nitosv1Channel.py +++ b/sfa/rspecs/elements/versions/nitosv1Channel.py @@ -54,7 +54,8 @@ class NITOSv1Channel: @staticmethod - def get_channels(xml, filter={}): + def get_channels(xml, filter=None): + if filter is None: filter={} xpath = '//channel%s | //default:channel%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) channel_elems = xml.xpath(xpath) return NITOSv1Channel.get_channel_objs(channel_elems) diff --git a/sfa/rspecs/elements/versions/nitosv1Lease.py b/sfa/rspecs/elements/versions/nitosv1Lease.py index 99e815a5..dd3041c5 100644 --- a/sfa/rspecs/elements/versions/nitosv1Lease.py +++ b/sfa/rspecs/elements/versions/nitosv1Lease.py @@ -73,7 +73,8 @@ class NITOSv1Lease: @staticmethod - def get_leases(xml, filter={}): + def get_leases(xml, filter=None): + if filter is None: filter={} xpath = '//lease%s | //default:lease%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) lease_elems = xml.xpath(xpath) return NITOSv1Lease.get_lease_objs(lease_elems) diff --git a/sfa/rspecs/elements/versions/nitosv1Node.py b/sfa/rspecs/elements/versions/nitosv1Node.py index 44b9b523..ea59b3d5 100644 --- a/sfa/rspecs/elements/versions/nitosv1Node.py +++ b/sfa/rspecs/elements/versions/nitosv1Node.py @@ -135,7 +135,8 @@ class NITOSv1Node: node.element.remove(sliver.element) @staticmethod - def get_nodes(xml, filter={}): + def get_nodes(xml, filter=None): + if filter is None: filter={} xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) node_elems = xml.xpath(xpath) return NITOSv1Node.get_node_objs(node_elems) diff --git a/sfa/rspecs/elements/versions/nitosv1PLTag.py b/sfa/rspecs/elements/versions/nitosv1PLTag.py index 7d03fe06..ea34ff4e 100644 --- a/sfa/rspecs/elements/versions/nitosv1PLTag.py +++ b/sfa/rspecs/elements/versions/nitosv1PLTag.py @@ -9,7 +9,8 @@ class NITOSv1PLTag: pl_tag_elem.set_text(value) @staticmethod - def get_pl_tags(xml, ignore=[]): + def get_pl_tags(xml, ignore=None): + if ignore is None: ignore=[] pl_tags = [] for elem in xml.iterchildren(): if elem.tag not in ignore: diff --git a/sfa/rspecs/elements/versions/nitosv1Sliver.py b/sfa/rspecs/elements/versions/nitosv1Sliver.py index 0f40211d..feac8879 100644 --- a/sfa/rspecs/elements/versions/nitosv1Sliver.py +++ b/sfa/rspecs/elements/versions/nitosv1Sliver.py @@ -43,7 +43,8 @@ class NITOSv1Sliver: return attribs @staticmethod - def get_slivers(xml, filter={}): + def get_slivers(xml, filter=None): + if filter is None: filter={} xpath = './default:sliver | ./sliver' sliver_elems = xml.xpath(xpath) slivers = [] diff --git a/sfa/rspecs/elements/versions/ofeliav1Port.py b/sfa/rspecs/elements/versions/ofeliav1Port.py index 07520ef3..f4cf74dc 100644 --- a/sfa/rspecs/elements/versions/ofeliav1Port.py +++ b/sfa/rspecs/elements/versions/ofeliav1Port.py @@ -39,7 +39,8 @@ class Ofeliav1Port: return attribs @staticmethod - def get_ports(xml, filter={}): + def get_ports(xml, filter=None): + if filter is None: filter={} xpath = './openflow:port | ./port' port_elems = xml.xpath(xpath) ports = [] diff --git a/sfa/rspecs/elements/versions/ofeliav1datapath.py b/sfa/rspecs/elements/versions/ofeliav1datapath.py index 86a38002..a1849735 100644 --- a/sfa/rspecs/elements/versions/ofeliav1datapath.py +++ b/sfa/rspecs/elements/versions/ofeliav1datapath.py @@ -21,7 +21,8 @@ from sfa.rspecs.elements.versions.ofeliav1Port import Ofeliav1Port class Ofeliav1Datapath: @staticmethod - def get_datapaths(xml, filter={}): + def get_datapaths(xml, filter=None): + if filter is None: filter={} #xpath = '//datapath%s | //default:datapath%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) xpath = '//datapath%s | //openflow:datapath%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) datapath_elems = xml.xpath(xpath) @@ -144,7 +145,8 @@ class Ofeliav1Datapath: # node.element.remove(sliver.element) # # @staticmethod -# def get_nodes(xml, filter={}): +# def get_nodes(xml, filter=None): +# if filter is None: filter={} # xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) # node_elems = xml.xpath(xpath) # return SFAv1Node.get_node_objs(node_elems) diff --git a/sfa/rspecs/elements/versions/ofeliav1link.py b/sfa/rspecs/elements/versions/ofeliav1link.py index 3fc2eb24..83a20969 100644 --- a/sfa/rspecs/elements/versions/ofeliav1link.py +++ b/sfa/rspecs/elements/versions/ofeliav1link.py @@ -8,7 +8,8 @@ from sfa.rspecs.elements.link import Link class Ofeliav1Link: @staticmethod - def get_links(xml, filter={}): + def get_links(xml, filter=None): + if filter is None: filter={} xpath = '//link%s | //openflow:link%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) link_elems = xml.xpath(xpath) return Ofeliav1Link.get_link_objs(link_elems) diff --git a/sfa/rspecs/elements/versions/pgv2DiskImage.py b/sfa/rspecs/elements/versions/pgv2DiskImage.py index 51363de3..4a6df82e 100644 --- a/sfa/rspecs/elements/versions/pgv2DiskImage.py +++ b/sfa/rspecs/elements/versions/pgv2DiskImage.py @@ -13,7 +13,8 @@ class PGv2DiskImage: xml.add_instance('disk_image', image, DiskImage.fields) @staticmethod - def get_images(xml, filter={}): + def get_images(xml, filter=None): + if filter is None: filter={} xpath = './default:disk_image | ./disk_image' image_elems = xml.xpath(xpath) images = [] diff --git a/sfa/rspecs/elements/versions/pgv2Lease.py b/sfa/rspecs/elements/versions/pgv2Lease.py index 68c44f0c..b04f7dc2 100644 --- a/sfa/rspecs/elements/versions/pgv2Lease.py +++ b/sfa/rspecs/elements/versions/pgv2Lease.py @@ -51,7 +51,8 @@ class PGv2Lease: @staticmethod - def get_leases(xml, filter={}): + def get_leases(xml, filter=None): + if filter is None: filter={} xpath = '//lease%s | //default:lease%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) lease_elems = xml.xpath(xpath) return PGv2Lease.get_lease_objs(lease_elems) diff --git a/sfa/rspecs/elements/versions/pgv2Node.py b/sfa/rspecs/elements/versions/pgv2Node.py index d553d219..60447b03 100644 --- a/sfa/rspecs/elements/versions/pgv2Node.py +++ b/sfa/rspecs/elements/versions/pgv2Node.py @@ -82,13 +82,15 @@ class PGv2Node: @staticmethod - def get_nodes(xml, filter={}): + def get_nodes(xml, filter=None): + if filter is None: filter={} xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) node_elems = xml.xpath(xpath) return PGv2Node.get_node_objs(node_elems) @staticmethod - def get_nodes_with_slivers(xml, filter={}): + def get_nodes_with_slivers(xml, filter=None): + if filter is None: filter={} xpath = '//node[count(sliver_type)>0] | //default:node[count(default:sliver_type) > 0]' node_elems = xml.xpath(xpath) return PGv2Node.get_node_objs(node_elems) diff --git a/sfa/rspecs/elements/versions/pgv2SliverType.py b/sfa/rspecs/elements/versions/pgv2SliverType.py index 1f3ec0c7..3ad687f8 100644 --- a/sfa/rspecs/elements/versions/pgv2SliverType.py +++ b/sfa/rspecs/elements/versions/pgv2SliverType.py @@ -40,7 +40,8 @@ class PGv2SliverType: for (key, value) in attrib_dict.items(): attrib_elem.set(key, value) @staticmethod - def get_slivers(xml, filter={}): + def get_slivers(xml, filter=None): + if filter is None: filter={} xpath = './default:sliver_type | ./sliver_type' sliver_elems = xml.xpath(xpath) slivers = [] @@ -56,5 +57,6 @@ class PGv2SliverType: return slivers @staticmethod - def get_sliver_attributes(xml, filter={}): + def get_sliver_attributes(xml, filter=None): + if filter is None: filter={} return [] diff --git a/sfa/rspecs/elements/versions/sfav1Lease.py b/sfa/rspecs/elements/versions/sfav1Lease.py index 039d2d13..0c7cb26b 100644 --- a/sfa/rspecs/elements/versions/sfav1Lease.py +++ b/sfa/rspecs/elements/versions/sfav1Lease.py @@ -72,7 +72,8 @@ class SFAv1Lease: @staticmethod - def get_leases(xml, filter={}): + def get_leases(xml, filter=None): + if filter is None: filter={} xpath = '//lease%s | //default:lease%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) lease_elems = xml.xpath(xpath) return SFAv1Lease.get_lease_objs(lease_elems) diff --git a/sfa/rspecs/elements/versions/sfav1Node.py b/sfa/rspecs/elements/versions/sfav1Node.py index 51076985..1931a584 100644 --- a/sfa/rspecs/elements/versions/sfav1Node.py +++ b/sfa/rspecs/elements/versions/sfav1Node.py @@ -125,7 +125,8 @@ class SFAv1Node: node.element.remove(sliver.element) @staticmethod - def get_nodes(xml, filter={}): + def get_nodes(xml, filter=None): + if filter is None: filter={} xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) node_elems = xml.xpath(xpath) return SFAv1Node.get_node_objs(node_elems) diff --git a/sfa/rspecs/elements/versions/sfav1PLTag.py b/sfa/rspecs/elements/versions/sfav1PLTag.py index b523124f..907c962a 100644 --- a/sfa/rspecs/elements/versions/sfav1PLTag.py +++ b/sfa/rspecs/elements/versions/sfav1PLTag.py @@ -9,7 +9,8 @@ class SFAv1PLTag: pl_tag_elem.set_text(value) @staticmethod - def get_pl_tags(xml, ignore=[]): + def get_pl_tags(xml, ignore=None): + if ignore is None: ignore=[] pl_tags = [] for elem in xml.iterchildren(): if elem.tag not in ignore: diff --git a/sfa/rspecs/elements/versions/sfav1Sliver.py b/sfa/rspecs/elements/versions/sfav1Sliver.py index a2b07a13..7e9282f2 100644 --- a/sfa/rspecs/elements/versions/sfav1Sliver.py +++ b/sfa/rspecs/elements/versions/sfav1Sliver.py @@ -39,7 +39,8 @@ class SFAv1Sliver: return attribs @staticmethod - def get_slivers(xml, filter={}): + def get_slivers(xml, filter=None): + if filter is None: filter={} xpath = './default:sliver | ./sliver' sliver_elems = xml.xpath(xpath) slivers = [] diff --git a/sfa/rspecs/rspec.py b/sfa/rspecs/rspec.py index 7d7d007a..ce16a1d1 100755 --- a/sfa/rspecs/rspec.py +++ b/sfa/rspecs/rspec.py @@ -3,13 +3,15 @@ from datetime import datetime, timedelta from sfa.util.xml import XML, XpathFilter from sfa.util.faults import InvalidRSpecElement, InvalidRSpec +from sfa.util.sfatime import SFATIME_FORMAT from sfa.rspecs.rspec_elements import RSpecElement, RSpecElements from sfa.rspecs.version_manager import VersionManager class RSpec: - def __init__(self, rspec="", version=None, user_options={}, ttl=60): + def __init__(self, rspec="", version=None, user_options=None, ttl=60): + if user_options is None: user_options={} self.header = '\n' self.template = """""" self.version = None @@ -37,11 +39,9 @@ class RSpec: self.version = self.version_manager.get_version(version) self.namespaces = self.version.namespaces self.parse_xml(self.version.template, self.version) - # eg. 2011-03-23T19:53:28Z - date_format = '%Y-%m-%dT%H:%M:%SZ' now = datetime.utcnow() - generated_ts = now.strftime(date_format) - expires_ts = (now + timedelta(minutes=self.ttl)).strftime(date_format) + generated_ts = now.strftime(SFATIME_FORMAT) + expires_ts = (now + timedelta(minutes=self.ttl)).strftime(SFATIME_FORMAT) self.xml.set('expires', expires_ts) self.xml.set('generated', generated_ts) @@ -75,15 +75,17 @@ class RSpec: raise InvalidRSpecElement(element_type, extra=msg) return self.elements[element_type] - def get(self, element_type, filter={}, depth=0): + def get(self, element_type, filter=None, depth=0): + if filter is None: filter={} elements = self.get_elements(element_type, filter) elements = [self.xml.get_element_attributes(elem, depth=depth) for elem in elements] return elements - def get_elements(self, element_type, filter={}): + def get_elements(self, element_type, filter=None): """ search for a registered element """ + if filter is None: filter={} if element_type not in self.elements: msg = "Unable to search for element %s in rspec, expath expression not found." % \ element_type diff --git a/sfa/rspecs/versions/iotlabv1.py b/sfa/rspecs/versions/iotlabv1.py index 9d8045f7..ad491571 100644 --- a/sfa/rspecs/versions/iotlabv1.py +++ b/sfa/rspecs/versions/iotlabv1.py @@ -139,7 +139,8 @@ class Iotlabv1(RSpecVersion): def add_default_sliver_attribute(self, name, value, network=None): pass - def add_slivers(self, hostnames, attributes=[], sliver_urn=None, append=False): + def add_slivers(self, hostnames, attributes=None, sliver_urn=None, append=False): + if attributes is None: attributes=[] # all nodes hould already be present in the rspec. Remove all # nodes that done have slivers print>>sys.stderr, "\r\n \r\n \r\n \t\t\t Iotlabv1.PY add_slivers ----->get_node " diff --git a/sfa/rspecs/versions/nitosv1.py b/sfa/rspecs/versions/nitosv1.py index af60d8ed..3288b48b 100644 --- a/sfa/rspecs/versions/nitosv1.py +++ b/sfa/rspecs/versions/nitosv1.py @@ -60,7 +60,8 @@ class NITOSv1(RSpecVersion): # Slivers - def add_slivers(self, hostnames, attributes=[], sliver_urn=None, append=False): + def add_slivers(self, hostnames, attributes=None, sliver_urn=None, append=False): + if attributes is None: attributes=[] # add slice name to network tag network_tags = self.xml.xpath('//network') if network_tags: diff --git a/sfa/rspecs/versions/ofeliav1.py b/sfa/rspecs/versions/ofeliav1.py index cd206ffc..d074694b 100755 --- a/sfa/rspecs/versions/ofeliav1.py +++ b/sfa/rspecs/versions/ofeliav1.py @@ -86,7 +86,8 @@ class Ofelia(RSpecVersion): # Slivers - def add_slivers(self, hostnames, attributes=[], sliver_urn=None, append=False): + def add_slivers(self, hostnames, attributes=None, sliver_urn=None, append=False): + if attributes is None: attributes=[] # add slice name to network tag network_tags = self.xml.xpath('//network') if network_tags: diff --git a/sfa/rspecs/versions/pgv2.py b/sfa/rspecs/versions/pgv2.py index 80febc90..21183bf7 100644 --- a/sfa/rspecs/versions/pgv2.py +++ b/sfa/rspecs/versions/pgv2.py @@ -103,7 +103,8 @@ class PGv2(RSpecVersion): def add_default_sliver_attribute(self, name, value, network=None): pass - def add_slivers(self, hostnames, attributes=[], sliver_urn=None, append=False): + def add_slivers(self, hostnames, attributes=None, sliver_urn=None, append=False): + if attributes is None: attributes=[] # all nodes hould already be present in the rspec. Remove all # nodes that done have slivers for hostname in hostnames: diff --git a/sfa/rspecs/versions/sfav1.py b/sfa/rspecs/versions/sfav1.py index 4ee8a4c8..6e973e7e 100644 --- a/sfa/rspecs/versions/sfav1.py +++ b/sfa/rspecs/versions/sfav1.py @@ -59,7 +59,8 @@ class SFAv1(RSpecVersion): # Slivers - def add_slivers(self, hostnames, attributes=[], sliver_urn=None, append=False): + def add_slivers(self, hostnames, attributes=None, sliver_urn=None, append=False): + if attributes is None: attributes=[] # add slice name to network tag network_tags = self.xml.xpath('//network') if network_tags: diff --git a/sfa/storage/model.py b/sfa/storage/model.py index 46e2deed..7c749773 100644 --- a/sfa/storage/model.py +++ b/sfa/storage/model.py @@ -143,12 +143,12 @@ class RegRecord (Base,AlchemyObj): else: return GID(string=self.gid) def just_created (self): - now=datetime.now() + now=datetime.utcnow() self.date_created=now self.last_updated=now def just_updated (self): - now=datetime.now() + now=datetime.utcnow() self.last_updated=now #################### cross-relations tables @@ -410,7 +410,8 @@ def drop_tables(engine): ############################## # create a record of the right type from either a dict or an xml string -def make_record (dict={}, xml=""): +def make_record (dict=None, xml=""): + if dict is None: dict={} if dict: return make_record_dict (dict) elif xml: return make_record_xml (xml) else: raise Exception("make_record has no input") diff --git a/sfa/storage/record.py b/sfa/storage/record.py index 812efdeb..9fcab19f 100644 --- a/sfa/storage/record.py +++ b/sfa/storage/record.py @@ -35,7 +35,8 @@ class Record: # it may be important to exclude relationships, which fortunately # - def todict (self, exclude_types=[]): + def todict (self, exclude_types=None): + if exclude_types is None: exclude_types=[] d=self.__dict__ def exclude (k,v): if k.startswith('_'): return True diff --git a/sfa/trust/abac_credential.py b/sfa/trust/abac_credential.py new file mode 100644 index 00000000..407f405f --- /dev/null +++ b/sfa/trust/abac_credential.py @@ -0,0 +1,279 @@ +#---------------------------------------------------------------------- +# Copyright (c) 2014 Raytheon BBN Technologies +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and/or hardware specification (the "Work") to +# deal in the Work without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Work, and to permit persons to whom the Work +# is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Work. +# +# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS +# IN THE WORK. +#---------------------------------------------------------------------- + +from sfa.trust.credential import Credential, append_sub, DEFAULT_CREDENTIAL_LIFETIME +from sfa.util.sfalogging import logger +from sfa.util.sfatime import SFATIME_FORMAT + +from StringIO import StringIO +from xml.dom.minidom import Document, parseString + +HAVELXML = False +try: + from lxml import etree + HAVELXML = True +except: + pass + +# This module defines a subtype of sfa.trust,credential.Credential +# called an ABACCredential. An ABAC credential is a signed statement +# asserting a role representing the relationship between a subject and target +# or between a subject and a class of targets (all those satisfying a role). +# +# An ABAC credential is like a normal SFA credential in that it has +# a validated signature block and is checked for expiration. +# It does not, however, have 'privileges'. Rather it contains a 'head' and +# list of 'tails' of elements, each of which represents a principal and +# role. + +# A special case of an ABAC credential is a speaks_for credential. Such +# a credential is simply an ABAC credential in form, but has a single +# tail and fixed role 'speaks_for'. In ABAC notation, it asserts +# AGENT.speaks_for(AGENT)<-CLIENT, or "AGENT asserts that CLIENT may speak +# for AGENT". The AGENT in this case is the head and the CLIENT is the +# tail and 'speaks_for_AGENT' is the role on the head. These speaks-for +# Credentials are used to allow a tool to 'speak as' itself but be recognized +# as speaking for an individual and be authorized to the rights of that +# individual and not to the rights of the tool itself. + +# For more detail on the semantics and syntax and expected usage patterns +# of ABAC credentials, see http://groups.geni.net/geni/wiki/TIEDABACCredential. + + +# An ABAC element contains a principal (keyid and optional mnemonic) +# and optional role and linking_role element +class ABACElement: + def __init__(self, principal_keyid, principal_mnemonic=None, \ + role=None, linking_role=None): + self._principal_keyid = principal_keyid + self._principal_mnemonic = principal_mnemonic + self._role = role + self._linking_role = linking_role + + def get_principal_keyid(self): return self._principal_keyid + def get_principal_mnemonic(self): return self._principal_mnemonic + def get_role(self): return self._role + def get_linking_role(self): return self._linking_role + + def __str__(self): + ret = self._principal_keyid + if self._principal_mnemonic: + ret = "%s (%s)" % (self._principal_mnemonic, self._principal_keyid) + if self._linking_role: + ret += ".%s" % self._linking_role + if self._role: + ret += ".%s" % self._role + return ret + +# Subclass of Credential for handling ABAC credentials +# They have a different cred_type (geni_abac vs. geni_sfa) +# and they have a head and tail and role (as opposed to privileges) +class ABACCredential(Credential): + + ABAC_CREDENTIAL_TYPE = 'geni_abac' + + def __init__(self, create=False, subject=None, + string=None, filename=None): + self.head = None # An ABACElemenet + self.tails = [] # List of ABACElements + super(ABACCredential, self).__init__(create=create, + subject=subject, + string=string, + filename=filename) + self.cred_type = ABACCredential.ABAC_CREDENTIAL_TYPE + + def get_head(self) : + if not self.head: + self.decode() + return self.head + + def get_tails(self) : + if len(self.tails) == 0: + self.decode() + return self.tails + + def decode(self): + super(ABACCredential, self).decode() + # Pull out the ABAC-specific info + doc = parseString(self.xml) + rt0s = doc.getElementsByTagName('rt0') + if len(rt0s) != 1: + raise CredentialNotVerifiable("ABAC credential had no rt0 element") + rt0_root = rt0s[0] + heads = self._get_abac_elements(rt0_root, 'head') + if len(heads) != 1: + raise CredentialNotVerifiable("ABAC credential should have exactly 1 head element, had %d" % len(heads)) + + self.head = heads[0] + self.tails = self._get_abac_elements(rt0_root, 'tail') + + def _get_abac_elements(self, root, label): + abac_elements = [] + elements = root.getElementsByTagName(label) + for elt in elements: + keyids = elt.getElementsByTagName('keyid') + if len(keyids) != 1: + raise CredentialNotVerifiable("ABAC credential element '%s' should have exactly 1 keyid, had %d." % (label, len(keyids))) + keyid_elt = keyids[0] + keyid = keyid_elt.childNodes[0].nodeValue.strip() + + mnemonic = None + mnemonic_elts = elt.getElementsByTagName('mnemonic') + if len(mnemonic_elts) > 0: + mnemonic = mnemonic_elts[0].childNodes[0].nodeValue.strip() + + role = None + role_elts = elt.getElementsByTagName('role') + if len(role_elts) > 0: + role = role_elts[0].childNodes[0].nodeValue.strip() + + linking_role = None + linking_role_elts = elt.getElementsByTagName('linking_role') + if len(linking_role_elts) > 0: + linking_role = linking_role_elts[0].childNodes[0].nodeValue.strip() + + abac_element = ABACElement(keyid, mnemonic, role, linking_role) + abac_elements.append(abac_element) + + return abac_elements + + def dump_string(self, dump_parents=False, show_xml=False): + result = "ABAC Credential\n" + filename=self.get_filename() + if filename: result += "Filename %s\n"%filename + if self.expiration: + result += "\texpiration: %s \n" % self.expiration.strftime(SFATIME_FORMAT) + + result += "\tHead: %s\n" % self.get_head() + for tail in self.get_tails(): + result += "\tTail: %s\n" % tail + if self.get_signature(): + result += " gidIssuer:\n" + result += self.get_signature().get_issuer_gid().dump_string(8, dump_parents) + if show_xml and HAVELXML: + try: + tree = etree.parse(StringIO(self.xml)) + aside = etree.tostring(tree, pretty_print=True) + result += "\nXML:\n\n" + result += aside + result += "\nEnd XML\n" + except: + import traceback + print "exc. Credential.dump_string / XML" + traceback.print_exc() + return result + + # sounds like this should be __repr__ instead ?? + # Produce the ABAC assertion. Something like [ABAC cred: Me.role<-You] or similar + def get_summary_tostring(self): + result = "[ABAC cred: " + str(self.get_head()) + for tail in self.get_tails(): + result += "<-%s" % str(tail) + result += "]" + return result + + def createABACElement(self, doc, tagName, abacObj): + kid = abacObj.get_principal_keyid() + mnem = abacObj.get_principal_mnemonic() # may be None + role = abacObj.get_role() # may be None + link = abacObj.get_linking_role() # may be None + ele = doc.createElement(tagName) + prin = doc.createElement('ABACprincipal') + ele.appendChild(prin) + append_sub(doc, prin, "keyid", kid) + if mnem: + append_sub(doc, prin, "mnemonic", mnem) + if role: + append_sub(doc, ele, "role", role) + if link: + append_sub(doc, ele, "linking_role", link) + return ele + + ## + # Encode the attributes of the credential into an XML string + # This should be done immediately before signing the credential. + # WARNING: + # In general, a signed credential obtained externally should + # not be changed else the signature is no longer valid. So, once + # you have loaded an existing signed credential, do not call encode() or sign() on it. + + def encode(self): + # Create the XML document + doc = Document() + signed_cred = doc.createElement("signed-credential") + +# Declare namespaces +# Note that credential/policy.xsd are really the PG schemas +# in a PL namespace. +# Note that delegation of credentials between the 2 only really works +# cause those schemas are identical. +# Also note these PG schemas talk about PG tickets and CM policies. + signed_cred.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") + signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.geni.net/resources/credential/2/credential.xsd") + signed_cred.setAttribute("xsi:schemaLocation", "http://www.planet-lab.org/resources/sfa/ext/policy/1 http://www.planet-lab.org/resources/sfa/ext/policy/1/policy.xsd") + +# PG says for those last 2: +# signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd") +# signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd") + + doc.appendChild(signed_cred) + + # Fill in the bit + cred = doc.createElement("credential") + cred.setAttribute("xml:id", self.get_refid()) + signed_cred.appendChild(cred) + append_sub(doc, cred, "type", "abac") + + # Stub fields + append_sub(doc, cred, "serial", "8") + append_sub(doc, cred, "owner_gid", '') + append_sub(doc, cred, "owner_urn", '') + append_sub(doc, cred, "target_gid", '') + append_sub(doc, cred, "target_urn", '') + append_sub(doc, cred, "uuid", "") + + if not self.expiration: + self.set_expiration(datetime.datetime.utcnow() + datetime.timedelta(seconds=DEFAULT_CREDENTIAL_LIFETIME)) + self.expiration = self.expiration.replace(microsecond=0) + if self.expiration.tzinfo is not None and self.expiration.tzinfo.utcoffset(self.expiration) is not None: + # TZ aware. Make sure it is UTC + self.expiration = self.expiration.astimezone(tz.tzutc()) + append_sub(doc, cred, "expires", self.expiration.strftime(SFATIME_FORMAT)) # RFC3339 + + abac = doc.createElement("abac") + rt0 = doc.createElement("rt0") + abac.appendChild(rt0) + cred.appendChild(abac) + append_sub(doc, rt0, "version", "1.1") + head = self.createABACElement(doc, "head", self.get_head()) + rt0.appendChild(head) + for tail in self.get_tails(): + tailEle = self.createABACElement(doc, "tail", tail) + rt0.appendChild(tailEle) + + # Create the tag + signatures = doc.createElement("signatures") + signed_cred.appendChild(signatures) + + # Get the finished product + self.xml = doc.toxml("utf-8") diff --git a/sfa/trust/auth.py b/sfa/trust/auth.py index 18c3d615..59ca4c27 100644 --- a/sfa/trust/auth.py +++ b/sfa/trust/auth.py @@ -2,6 +2,7 @@ # SfaAPI authentication # import sys +from types import StringTypes from sfa.util.faults import InsufficientRights, MissingCallerGID, MissingTrustedRoots, PermissionError, \ BadRequestHash, ConnectionKeyGIDMismatch, SfaPermissionDenied, CredentialNotVerifiable, Forbidden, \ @@ -17,6 +18,7 @@ from sfa.trust.credential import Credential from sfa.trust.trustedroots import TrustedRoots from sfa.trust.hierarchy import Hierarchy from sfa.trust.sfaticket import SfaTicket +from sfa.trust.speaksfor_util import determine_speaks_for class Auth: @@ -35,12 +37,33 @@ class Auth: self.trusted_cert_list = TrustedRoots(self.config.get_trustedroots_dir()).get_list() self.trusted_cert_file_list = TrustedRoots(self.config.get_trustedroots_dir()).get_file_list() - def checkCredentials(self, creds, operation, xrns=[], check_sliver_callback=None, speaking_for_hrn=None): - + # this convenience methods extracts speaking_for_xrn from the passed options using 'geni_speaking_for' + def checkCredentialsSpeaksFor (self, *args, **kwds): + if 'options' not in kwds: + logger.error ("checkCredentialsSpeaksFor was not passed options=options") + return + # remove the options arg + options=kwds['options']; del kwds['options'] + # compute the speaking_for_xrn arg and pass it to checkCredentials + if options is None: speaking_for_xrn=None + else: speaking_for_xrn=options.get('geni_speaking_for',None) + kwds['speaking_for_xrn']=speaking_for_xrn + return self.checkCredentials (*args, **kwds) + + # do not use mutable as default argument + # http://docs.python-guide.org/en/latest/writing/gotchas/#mutable-default-arguments + def checkCredentials(self, creds, operation, xrns=None, + check_sliver_callback=None, + speaking_for_xrn=None): + if xrns is None: xrns=[] def log_invalid_cred(cred): - cred_obj=Credential(string=cred) - logger.debug("failed to validate credential - dump=%s"%cred_obj.dump_string(dump_parents=True)) - error = sys.exc_info()[:2] + if not isinstance (cred, StringTypes): + logger.info("cannot validate credential %s - expecting a string"%cred) + error="checkCredentials: expected a string, received %s"%(type(cred)) + else: + cred_obj=Credential(string=cred) + logger.info("failed to validate credential - dump=%s"%cred_obj.dump_string(dump_parents=True)) + error = sys.exc_info()[:2] return error # if xrns are specified they cannot be None or empty string @@ -60,7 +83,6 @@ class Auth: # we make sure not to include sliver urns/hrns in the core validation loop hrns = [Xrn(xrn).hrn for xrn in xrns if xrn not in sliver_xrns] valid = [] - speaks_for_cred = None if not isinstance(creds, list): creds = [creds] logger.debug("Auth.checkCredentials with %d creds on hrns=%s"%(len(creds),hrns)) @@ -68,22 +90,21 @@ class Auth: if not creds: raise Forbidden("no credential provided") if not hrns: hrns = [None] error=[None,None] - for cred in creds: - for hrn in hrns: - try: - self.check(cred, operation, hrn) - valid.append(cred) - except: - if speaking_for_hrn: - try: - self.check(cred, operation, speaking_for_hrn) - speaks_for_cred = cred - valid.append(cred) - except: - error = log_invalid_cred(cred) - else: - error = log_invalid_cred(cred) - continue + + speaks_for_gid = determine_speaks_for(logger, creds, self.peer_cert, + speaking_for_xrn, self.trusted_cert_list) + + if self.peer_cert and \ + not self.peer_cert.is_pubkey(speaks_for_gid.get_pubkey()): + valid = creds + else: + for cred in creds: + for hrn in hrns: + try: + self.check(cred, operation, hrn) + valid.append(cred) + except: + error = log_invalid_cred(cred) # make sure all sliver xrns are validated against the valid credentials if sliver_xrns: @@ -96,9 +117,6 @@ class Auth: if not len(valid): raise Forbidden("Invalid credential %s -- %s"%(error[0],error[1])) - if speaking_for_hrn and not speaks_for_cred: - raise InsufficientRights('Access denied: "geni_speaking_for" option specified but no valid speaks for credential found: %s -- %s' % (error[0],error[1])) - return valid diff --git a/sfa/trust/credential.py b/sfa/trust/credential.py index 9a45400e..9d0fd283 100644 --- a/sfa/trust/credential.py +++ b/sfa/trust/credential.py @@ -44,14 +44,13 @@ from xml.parsers.expat import ExpatError from sfa.util.faults import CredentialNotVerifiable, ChildRightsNotSubsetOfParent from sfa.util.sfalogging import logger -from sfa.util.sfatime import utcparse -from sfa.trust.credential_legacy import CredentialLegacy +from sfa.util.sfatime import utcparse, SFATIME_FORMAT from sfa.trust.rights import Right, Rights, determine_rights from sfa.trust.gid import GID from sfa.util.xrn import urn_to_hrn, hrn_authfor_hrn -# 2 weeks, in seconds -DEFAULT_CREDENTIAL_LIFETIME = 86400 * 31 +# 31 days, in seconds +DEFAULT_CREDENTIAL_LIFETIME = 86400 * 28 # TODO: @@ -196,14 +195,13 @@ class Signature(object): def encode(self): self.xml = signature_template % (self.get_refid(), self.get_refid()) - ## # A credential provides a caller gid with privileges to an object gid. # A signed credential is signed by the object's authority. # -# Credentials are encoded in one of two ways. The legacy style places -# it in the subjectAltName of an X509 certificate. The new credentials -# are placed in signed XML. +# Credentials are encoded in one of two ways. +# The legacy style (now unsupported) places it in the subjectAltName of an X509 certificate. +# The new credentials are placed in signed XML. # # WARNING: # In general, a signed credential obtained externally should @@ -249,7 +247,6 @@ class Credential(object): self.signature = None self.xml = None self.refid = None - self.legacy = None self.type = None self.version = None @@ -264,16 +261,16 @@ class Credential(object): self.version = cred['geni_version'] - # Check if this is a legacy credential, translate it if so if string or filename: if string: str = string elif filename: str = file(filename).read() - if str.strip().startswith("-----"): - self.legacy = CredentialLegacy(False,string=str) - self.translate_legacy(str) + # if this is a legacy credential, write error and bail out + if isinstance (str, StringTypes) and str.strip().startswith("-----"): + logger.error("Legacy credentials not supported any more - giving up with %s..."%str[:10]) + return else: self.xml = str self.decode() @@ -313,24 +310,6 @@ class Credential(object): self.signature = sig - ## - # Translate a legacy credential into a new one - # - # @param String of the legacy credential - - def translate_legacy(self, str): - legacy = CredentialLegacy(False,string=str) - self.gidCaller = legacy.get_gid_caller() - self.gidObject = legacy.get_gid_object() - lifetime = legacy.get_lifetime() - if not lifetime: - self.set_expiration(datetime.datetime.utcnow() + datetime.timedelta(seconds=DEFAULT_CREDENTIAL_LIFETIME)) - else: - self.set_expiration(int(lifetime)) - self.lifeTime = legacy.get_lifetime() - self.set_privileges(legacy.get_privileges()) - self.get_privileges().delegate_all_privileges(legacy.get_delegate()) - ## # Need the issuer's private key and name # @param key Keypair object containing the private key of the issuer @@ -385,15 +364,11 @@ class Credential(object): # Expiration: an absolute UTC time of expiration (as either an int or string or datetime) # def set_expiration(self, expiration): - if isinstance(expiration, (int, float)): - self.expiration = datetime.datetime.fromtimestamp(expiration) - elif isinstance (expiration, datetime.datetime): - self.expiration = expiration - elif isinstance (expiration, StringTypes): - self.expiration = utcparse (expiration) + expiration_datetime = utcparse (expiration) + if expiration_datetime is not None: + self.expiration = expiration_datetime else: - logger.error ("unexpected input type in Credential.set_expiration") - + logger.error ("unexpected input %s in Credential.set_expiration"%expiration) ## # get the lifetime of the credential (always in datetime format) @@ -404,11 +379,6 @@ class Credential(object): # at this point self.expiration is normalized as a datetime - DON'T call utcparse again return self.expiration - ## - # For legacy sake - def get_lifetime(self): - return self.get_expiration() - ## # set the privileges # @@ -456,19 +426,19 @@ class Credential(object): doc = Document() signed_cred = doc.createElement("signed-credential") -# Declare namespaces -# Note that credential/policy.xsd are really the PG schemas -# in a PL namespace. -# Note that delegation of credentials between the 2 only really works -# cause those schemas are identical. -# Also note these PG schemas talk about PG tickets and CM policies. + # Declare namespaces + # Note that credential/policy.xsd are really the PG schemas + # in a PL namespace. + # Note that delegation of credentials between the 2 only really works + # cause those schemas are identical. + # Also note these PG schemas talk about PG tickets and CM policies. signed_cred.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.planet-lab.org/resources/sfa/credential.xsd") signed_cred.setAttribute("xsi:schemaLocation", "http://www.planet-lab.org/resources/sfa/ext/policy/1 http://www.planet-lab.org/resources/sfa/ext/policy/1/policy.xsd") -# PG says for those last 2: -# signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd") -# signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd") + # PG says for those last 2: + #signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd") + # signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd") doc.appendChild(signed_cred) @@ -484,9 +454,10 @@ class Credential(object): append_sub(doc, cred, "target_urn", self.gidObject.get_urn()) append_sub(doc, cred, "uuid", "") if not self.expiration: + logger.debug("Creating credential valid for %s s"%DEFAULT_CREDENTIAL_LIFETIME) self.set_expiration(datetime.datetime.utcnow() + datetime.timedelta(seconds=DEFAULT_CREDENTIAL_LIFETIME)) self.expiration = self.expiration.replace(microsecond=0) - append_sub(doc, cred, "expires", self.expiration.isoformat()) + append_sub(doc, cred, "expires", self.expiration.strftime(SFATIME_FORMAT)) privileges = doc.createElement("privileges") cred.appendChild(privileges) @@ -508,10 +479,10 @@ class Credential(object): # and we need to include those again here or else their signature # no longer matches on the credential. # We expect three of these, but here we copy them all: -# signed_cred.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") -# and from PG (PL is equivalent, as shown above): -# signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd") -# signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd") + # signed_cred.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") + # and from PG (PL is equivalent, as shown above): + # signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd") + # signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd") # HOWEVER! # PL now also declares these, with different URLs, so @@ -680,10 +651,6 @@ class Credential(object): self.xml = signed - # This is no longer a legacy credential - if self.legacy: - self.legacy = None - # Update signatures self.decode() @@ -800,7 +767,7 @@ class Credential(object): self.decode() # validate against RelaxNG schema - if HAVELXML and not self.legacy: + if HAVELXML: if schema and os.path.exists(schema): tree = etree.parse(StringIO(self.xml)) schema_doc = etree.parse(schema) @@ -829,18 +796,9 @@ class Credential(object): logger.error("Failed to load trusted cert from %s: %r"%( f, exc)) trusted_certs = ok_trusted_certs - # Use legacy verification if this is a legacy credential - if self.legacy: - self.legacy.verify_chain(trusted_cert_objects) - if self.legacy.client_gid: - self.legacy.client_gid.verify_chain(trusted_cert_objects) - if self.legacy.object_gid: - self.legacy.object_gid.verify_chain(trusted_cert_objects) - return True - # make sure it is not expired if self.get_expiration() < datetime.datetime.utcnow(): - raise CredentialNotVerifiable("Credential %s expired at %s" % (self.get_summary_tostring(), self.expiration.isoformat())) + raise CredentialNotVerifiable("Credential %s expired at %s" % (self.get_summary_tostring(), self.expiration.strftime(SFATIME_FORMAT))) # Verify the signatures filename = self.save_to_random_tmp_file() @@ -1048,7 +1006,32 @@ class Credential(object): # only informative def get_filename(self): return getattr(self,'filename',None) - + + def actual_caller_hrn (self): + """a helper method used by some API calls like e.g. Allocate + to try and find out who really is the original caller + + This admittedly is a bit of a hack, please USE IN LAST RESORT + + This code uses a heuristic to identify a delegated credential + + A first known restriction if for traffic that gets through a slice manager + in this case the hrn reported is the one from the last SM in the call graph + which is not at all what is meant here""" + + caller_hrn = self.get_gid_caller().get_hrn() + issuer_hrn = self.get_signature().get_issuer_gid().get_hrn() + subject_hrn = self.get_gid_object().get_hrn() + # if we find that the caller_hrn is an immediate descendant of the issuer, then + # this seems to be a 'regular' credential + if caller_hrn.startswith(issuer_hrn): + actual_caller_hrn=caller_hrn + # else this looks like a delegated credential, and the real caller is the issuer + else: + actual_caller_hrn=issuer_hrn + logger.info("actual_caller_hrn: caller_hrn=%s, issuer_hrn=%s, returning %s"%(caller_hrn,issuer_hrn,actual_caller_hrn)) + return actual_caller_hrn + ## # Dump the contents of a credential to stdout in human-readable format # @@ -1077,7 +1060,7 @@ class Credential(object): self.get_signature().get_issuer_gid().dump(8, dump_parents) if self.expiration: - print " expiration:", self.expiration.isoformat() + print " expiration:", self.expiration.strftime(SFATIME_FORMAT) gidObject = self.get_gid_object() if gidObject: diff --git a/sfa/trust/credential_factory.py b/sfa/trust/credential_factory.py new file mode 100644 index 00000000..2fe37a70 --- /dev/null +++ b/sfa/trust/credential_factory.py @@ -0,0 +1,110 @@ +#---------------------------------------------------------------------- +# Copyright (c) 2014 Raytheon BBN Technologies +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and/or hardware specification (the "Work") to +# deal in the Work without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Work, and to permit persons to whom the Work +# is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Work. +# +# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS +# IN THE WORK. +#---------------------------------------------------------------------- + +from sfa.util.sfalogging import logger +from sfa.trust.credential import Credential +from sfa.trust.abac_credential import ABACCredential + +import json +import re + +# Factory for creating credentials of different sorts by type. +# Specifically, this factory can create standard SFA credentials +# and ABAC credentials from XML strings based on their identifying content + +class CredentialFactory: + + UNKNOWN_CREDENTIAL_TYPE = 'geni_unknown' + + # Static Credential class method to determine the type of a credential + # string depending on its contents + @staticmethod + def getType(credString): + credString_nowhitespace = re.sub('\s', '', credString) + if credString_nowhitespace.find('abac') > -1: + return ABACCredential.ABAC_CREDENTIAL_TYPE + elif credString_nowhitespace.find('privilege') > -1: + return Credential.SFA_CREDENTIAL_TYPE + else: + st = credString_nowhitespace.find('') + end = credString_nowhitespace.find('', st) + return credString_nowhitespace[st + len(''):end] +# return CredentialFactory.UNKNOWN_CREDENTIAL_TYPE + + # Static Credential class method to create the appropriate credential + # (SFA or ABAC) depending on its type + @staticmethod + def createCred(credString=None, credFile=None): + if not credString and not credFile: + raise Exception("CredentialFactory.createCred called with no argument") + if credFile: + try: + credString = open(credFile).read() + except Exception, e: + logger.info("Error opening credential file %s: %s" % credFile, e) + return None + + # Try to treat the file as JSON, getting the cred_type from the struct + try: + credO = json.loads(credString, encoding='ascii') + if credO.has_key('geni_value') and credO.has_key('geni_type'): + cred_type = credO['geni_type'] + credString = credO['geni_value'] + except Exception, e: + # It wasn't a struct. So the credString is XML. Pull the type directly from the string + logger.debug("Credential string not JSON: %s" % e) + cred_type = CredentialFactory.getType(credString) + + if cred_type == Credential.SFA_CREDENTIAL_TYPE: + try: + cred = Credential(string=credString) + return cred + except Exception, e: + if credFile: + msg = "credString started: %s" % credString[:50] + raise Exception("%s not a parsable SFA credential: %s. " % (credFile, e) + msg) + else: + raise Exception("SFA Credential not parsable: %s. Cred start: %s..." % (e, credString[:50])) + + elif cred_type == ABACCredential.ABAC_CREDENTIAL_TYPE: + try: + cred = ABACCredential(string=credString) + return cred + except Exception, e: + if credFile: + raise Exception("%s not a parsable ABAC credential: %s" % (credFile, e)) + else: + raise Exception("ABAC Credential not parsable: %s. Cred start: %s..." % (e, credString[:50])) + else: + raise Exception("Unknown credential type '%s'" % cred_type) + +if __name__ == "__main__": + c2 = open('/tmp/sfa.xml').read() + cred1 = CredentialFactory.createCred(credFile='/tmp/cred.xml') + cred2 = CredentialFactory.createCred(credString=c2) + + print "C1 = %s" % cred1 + print "C2 = %s" % cred2 + c1s = cred1.dump_string() + print "C1 = %s" % c1s +# print "C2 = %s" % cred2.dump_string() diff --git a/sfa/trust/credential_legacy.py b/sfa/trust/credential_legacy.py deleted file mode 100644 index b5fc449a..00000000 --- a/sfa/trust/credential_legacy.py +++ /dev/null @@ -1,270 +0,0 @@ -#---------------------------------------------------------------------- -# Copyright (c) 2008 Board of Trustees, Princeton University -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and/or hardware specification (the "Work") to -# deal in the Work without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Work, and to permit persons to whom the Work -# is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Work. -# -# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS -# IN THE WORK. -#---------------------------------------------------------------------- -## -# Implements SFA Credentials -# -# Credentials are layered on top of certificates, and are essentially a -# certificate that stores a tuple of parameters. -## - - -import xmlrpclib - -from sfa.util.faults import MissingDelegateBit, ChildRightsNotSubsetOfParent -from sfa.trust.certificate import Certificate -from sfa.trust.gid import GID - -## -# Credential is a tuple: -# (GIDCaller, GIDObject, LifeTime, Privileges, Delegate) -# -# These fields are encoded using xmlrpc into the subjectAltName field of the -# x509 certificate. Note: Call encode() once the fields have been filled in -# to perform this encoding. - -class CredentialLegacy(Certificate): - gidCaller = None - gidObject = None - lifeTime = None - privileges = None - delegate = False - - ## - # Create a Credential object - # - # @param create If true, create a blank x509 certificate - # @param subject If subject!=None, create an x509 cert with the subject name - # @param string If string!=None, load the credential from the string - # @param filename If filename!=None, load the credential from the file - - def __init__(self, create=False, subject=None, string=None, filename=None): - Certificate.__init__(self, create, subject, string, filename) - - ## - # set the GID of the caller - # - # @param gid GID object of the caller - - def set_gid_caller(self, gid): - self.gidCaller = gid - # gid origin caller is the caller's gid by default - self.gidOriginCaller = gid - - ## - # get the GID of the object - - def get_gid_caller(self): - if not self.gidCaller: - self.decode() - return self.gidCaller - - ## - # set the GID of the object - # - # @param gid GID object of the object - - def set_gid_object(self, gid): - self.gidObject = gid - - ## - # get the GID of the object - - def get_gid_object(self): - if not self.gidObject: - self.decode() - return self.gidObject - - ## - # set the lifetime of this credential - # - # @param lifetime lifetime of credential - - def set_lifetime(self, lifeTime): - self.lifeTime = lifeTime - - ## - # get the lifetime of the credential - - def get_lifetime(self): - if not self.lifeTime: - self.decode() - return self.lifeTime - - ## - # set the delegate bit - # - # @param delegate boolean (True or False) - - def set_delegate(self, delegate): - self.delegate = delegate - - ## - # get the delegate bit - - def get_delegate(self): - if not self.delegate: - self.decode() - return self.delegate - - ## - # set the privileges - # - # @param privs either a comma-separated list of privileges of a Rights object - - def set_privileges(self, privs): - if isinstance(privs, str): - self.privileges = Rights(string = privs) - else: - self.privileges = privs - - ## - # return the privileges as a Rights object - - def get_privileges(self): - if not self.privileges: - self.decode() - return self.privileges - - ## - # determine whether the credential allows a particular operation to be - # performed - # - # @param op_name string specifying name of operation ("lookup", "update", etc) - - def can_perform(self, op_name): - rights = self.get_privileges() - if not rights: - return False - return rights.can_perform(op_name) - - ## - # Encode the attributes of the credential into a string and store that - # string in the alt-subject-name field of the X509 object. This should be - # done immediately before signing the credential. - - def encode(self): - dict = {"gidCaller": None, - "gidObject": None, - "lifeTime": self.lifeTime, - "privileges": None, - "delegate": self.delegate} - if self.gidCaller: - dict["gidCaller"] = self.gidCaller.save_to_string(save_parents=True) - if self.gidObject: - dict["gidObject"] = self.gidObject.save_to_string(save_parents=True) - if self.privileges: - dict["privileges"] = self.privileges.save_to_string() - str = xmlrpclib.dumps((dict,), allow_none=True) - self.set_data('URI:http://' + str) - - ## - # Retrieve the attributes of the credential from the alt-subject-name field - # of the X509 certificate. This is automatically done by the various - # get_* methods of this class and should not need to be called explicitly. - - def decode(self): - data = self.get_data().lstrip('URI:http://') - - if data: - dict = xmlrpclib.loads(data)[0][0] - else: - dict = {} - - self.lifeTime = dict.get("lifeTime", None) - self.delegate = dict.get("delegate", None) - - privStr = dict.get("privileges", None) - if privStr: - self.privileges = Rights(string = privStr) - else: - self.privileges = None - - gidCallerStr = dict.get("gidCaller", None) - if gidCallerStr: - self.gidCaller = GID(string=gidCallerStr) - else: - self.gidCaller = None - - gidObjectStr = dict.get("gidObject", None) - if gidObjectStr: - self.gidObject = GID(string=gidObjectStr) - else: - self.gidObject = None - - ## - # Verify that a chain of credentials is valid (see cert.py:verify). In - # addition to the checks for ordinary certificates, verification also - # ensures that the delegate bit was set by each parent in the chain. If - # a delegate bit was not set, then an exception is thrown. - # - # Each credential must be a subset of the rights of the parent. - - def verify_chain(self, trusted_certs = None): - # do the normal certificate verification stuff - Certificate.verify_chain(self, trusted_certs) - - if self.parent: - # make sure the parent delegated rights to the child - if not self.parent.get_delegate(): - raise MissingDelegateBit(self.parent.get_subject()) - - # make sure the rights given to the child are a subset of the - # parents rights - if not self.parent.get_privileges().is_superset(self.get_privileges()): - raise ChildRightsNotSubsetOfParent(self.get_subject() - + " " + self.parent.get_privileges().save_to_string() - + " " + self.get_privileges().save_to_string()) - - return - - ## - # Dump the contents of a credential to stdout in human-readable format - # - # @param dump_parents If true, also dump the parent certificates - - def dump(self, *args, **kwargs): - print self.dump_string(*args,**kwargs) - - def dump_string(self, dump_parents=False): - result="" - result += "CREDENTIAL %s\n" % self.get_subject() - - result += " privs: %s\n" % self.get_privileges().save_to_string() - - gidCaller = self.get_gid_caller() - if gidCaller: - result += " gidCaller:\n" - gidCaller.dump(8, dump_parents) - - gidObject = self.get_gid_object() - if gidObject: - result += " gidObject:\n" - result += gidObject.dump_string(8, dump_parents) - - result += " delegate: %s" % self.get_delegate() - - if self.parent and dump_parents: - result += "PARENT\n" - result += self.parent.dump_string(dump_parents) - - return result diff --git a/sfa/trust/speaksfor_util.py b/sfa/trust/speaksfor_util.py new file mode 100644 index 00000000..2c56a47c --- /dev/null +++ b/sfa/trust/speaksfor_util.py @@ -0,0 +1,451 @@ +#---------------------------------------------------------------------- +# Copyright (c) 2014 Raytheon BBN Technologies +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and/or hardware specification (the "Work") to +# deal in the Work without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Work, and to permit persons to whom the Work +# is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Work. +# +# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS +# IN THE WORK. +#---------------------------------------------------------------------- + +import datetime +from dateutil import parser as du_parser, tz as du_tz +import optparse +import os +import subprocess +import sys +import tempfile +from xml.dom.minidom import * +from StringIO import StringIO + +from sfa.util.sfatime import SFATIME_FORMAT + +from sfa.trust.certificate import Certificate +from sfa.trust.credential import Credential, signature_template, HAVELXML +from sfa.trust.abac_credential import ABACCredential, ABACElement +from sfa.trust.credential_factory import CredentialFactory +from sfa.trust.gid import GID + +# Routine to validate that a speaks-for credential +# says what it claims to say: +# It is a signed credential wherein the signer S is attesting to the +# ABAC statement: +# S.speaks_for(S)<-T Or "S says that T speaks for S" + +# Requires that openssl be installed and in the path +# create_speaks_for requires that xmlsec1 be on the path + +# Simple XML helper functions + +# Find the text associated with first child text node +def findTextChildValue(root): + child = findChildNamed(root, '#text') + if child: return str(child.nodeValue) + return None + +# Find first child with given name +def findChildNamed(root, name): + for child in root.childNodes: + if child.nodeName == name: + return child + return None + +# Write a string to a tempfile, returning name of tempfile +def write_to_tempfile(str): + str_fd, str_file = tempfile.mkstemp() + if str: + os.write(str_fd, str) + os.close(str_fd) + return str_file + +# Run a subprocess and return output +def run_subprocess(cmd, stdout, stderr): + try: + proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr) + proc.wait() + if stdout: + output = proc.stdout.read() + else: + output = proc.returncode + return output + except Exception as e: + raise Exception("Failed call to subprocess '%s': %s" % (" ".join(cmd), e)) + +def get_cert_keyid(gid): + """Extract the subject key identifier from the given certificate. + Return they key id as lowercase string with no colon separators + between pairs. The key id as shown in the text output of a + certificate are in uppercase with colon separators. + + """ + raw_key_id = gid.get_extension('subjectKeyIdentifier') + # Raw has colons separating pairs, and all characters are upper case. + # Remove the colons and convert to lower case. + keyid = raw_key_id.replace(':', '').lower() + return keyid + +# Pull the cert out of a list of certs in a PEM formatted cert string +def grab_toplevel_cert(cert): + start_label = '-----BEGIN CERTIFICATE-----' + if cert.find(start_label) > -1: + start_index = cert.find(start_label) + len(start_label) + else: + start_index = 0 + end_label = '-----END CERTIFICATE-----' + end_index = cert.find(end_label) + first_cert = cert[start_index:end_index] + pieces = first_cert.split('\n') + first_cert = "".join(pieces) + return first_cert + +# Validate that the given speaks-for credential represents the +# statement User.speaks_for(User)<-Tool for the given user and tool certs +# and was signed by the user +# Return: +# Boolean indicating whether the given credential +# is not expired +# is an ABAC credential +# was signed by the user associated with the speaking_for_urn +# is verified by xmlsec1 +# asserts U.speaks_for(U)<-T ("user says that T may speak for user") +# If schema provided, validate against schema +# is trusted by given set of trusted roots (both user cert and tool cert) +# String user certificate of speaking_for user if the above tests succeed +# (None otherwise) +# Error message indicating why the speaks_for call failed ("" otherwise) +def verify_speaks_for(cred, tool_gid, speaking_for_urn, + trusted_roots, schema=None, logger=None): + + # Credential has not expired + if cred.expiration and cred.expiration < datetime.datetime.utcnow(): + return False, None, "ABAC Credential expired at %s (%s)" % (cred.expiration.strftime(SFATIME_FORMAT), cred.get_summary_tostring()) + + # Must be ABAC + if cred.get_cred_type() != ABACCredential.ABAC_CREDENTIAL_TYPE: + return False, None, "Credential not of type ABAC but %s" % cred.get_cred_type + + if cred.signature is None or cred.signature.gid is None: + return False, None, "Credential malformed: missing signature or signer cert. Cred: %s" % cred.get_summary_tostring() + user_gid = cred.signature.gid + user_urn = user_gid.get_urn() + + # URN of signer from cert must match URN of 'speaking-for' argument + if user_urn != speaking_for_urn: + return False, None, "User URN from cred doesn't match speaking_for URN: %s != %s (cred %s)" % \ + (user_urn, speaking_for_urn, cred.get_summary_tostring()) + + tails = cred.get_tails() + if len(tails) != 1: + return False, None, "Invalid ABAC-SF credential: Need exactly 1 tail element, got %d (%s)" % \ + (len(tails), cred.get_summary_tostring()) + + user_keyid = get_cert_keyid(user_gid) + tool_keyid = get_cert_keyid(tool_gid) + subject_keyid = tails[0].get_principal_keyid() + + head = cred.get_head() + principal_keyid = head.get_principal_keyid() + role = head.get_role() + + # Credential must pass xmlsec1 verify + cred_file = write_to_tempfile(cred.save_to_string()) + cert_args = [] + if trusted_roots: + for x in trusted_roots: + cert_args += ['--trusted-pem', x.filename] + # FIXME: Why do we not need to specify the --node-id option as credential.py does? + xmlsec1_args = [cred.xmlsec_path, '--verify'] + cert_args + [ cred_file] + output = run_subprocess(xmlsec1_args, stdout=None, stderr=subprocess.PIPE) + os.unlink(cred_file) + if output != 0: + # FIXME + # xmlsec errors have a msg= which is the interesting bit. + # But does this go to stderr or stdout? Do we have it here? + mstart = verified.find("msg=") + msg = "" + if mstart > -1 and len(verified) > 4: + mstart = mstart + 4 + mend = verified.find('\\', mstart) + msg = verified[mstart:mend] + if msg == "": + msg = output + return False, None, "ABAC credential failed to xmlsec1 verify: %s" % msg + + # Must say U.speaks_for(U)<-T + if user_keyid != principal_keyid or \ + tool_keyid != subject_keyid or \ + role != ('speaks_for_%s' % user_keyid): + return False, None, "ABAC statement doesn't assert U.speaks_for(U)<-T (%s)" % cred.get_summary_tostring() + + # If schema provided, validate against schema + if HAVELXML and schema and os.path.exists(schema): + from lxml import etree + tree = etree.parse(StringIO(cred.xml)) + schema_doc = etree.parse(schema) + xmlschema = etree.XMLSchema(schema_doc) + if not xmlschema.validate(tree): + error = xmlschema.error_log.last_error + message = "%s: %s (line %s)" % (cred.get_summary_tostring(), error.message, error.line) + return False, None, ("XML Credential schema invalid: %s" % message) + + if trusted_roots: + # User certificate must validate against trusted roots + try: + user_gid.verify_chain(trusted_roots) + except Exception, e: + return False, None, \ + "Cred signer (user) cert not trusted: %s" % e + + # Tool certificate must validate against trusted roots + try: + tool_gid.verify_chain(trusted_roots) + except Exception, e: + return False, None, \ + "Tool cert not trusted: %s" % e + + return True, user_gid, "" + +# Determine if this is a speaks-for context. If so, validate +# And return either the tool_cert (not speaks-for or not validated) +# or the user cert (validated speaks-for) +# +# credentials is a list of GENI-style credentials: +# Either a cred string xml string, or Credential object of a tuple +# [{'geni_type' : geni_type, 'geni_value : cred_value, +# 'geni_version' : version}] +# caller_gid is the raw X509 cert gid +# options is the dictionary of API-provided options +# trusted_roots is a list of Certificate objects from the system +# trusted_root directory +# Optionally, provide an XML schema against which to validate the credential +def determine_speaks_for(logger, credentials, caller_gid, speaking_for_xrn, trusted_roots, schema=None): + if speaking_for_xrn: + speaking_for_urn = Xrn (speaking_for_xrn.strip()).get_urn() + for cred in credentials: + # Skip things that aren't ABAC credentials + if type(cred) == dict: + if cred['geni_type'] != ABACCredential.ABAC_CREDENTIAL_TYPE: continue + cred_value = cred['geni_value'] + elif isinstance(cred, Credential): + if not isinstance(cred, ABACCredential): + continue + else: + cred_value = cred + else: + if CredentialFactory.getType(cred) != ABACCredential.ABAC_CREDENTIAL_TYPE: continue + cred_value = cred + + # If the cred_value is xml, create the object + if not isinstance(cred_value, ABACCredential): + cred = CredentialFactory.createCred(cred_value) + +# print "Got a cred to check speaksfor for: %s" % cred.get_summary_tostring() +# #cred.dump(True, True) +# print "Caller: %s" % caller_gid.dump_string(2, True) + # See if this is a valid speaks_for + is_valid_speaks_for, user_gid, msg = \ + verify_speaks_for(cred, + caller_gid, speaking_for_urn, \ + trusted_roots, schema, logger=logger) + logger.info(msg) + if is_valid_speaks_for: + return user_gid # speaks-for + else: + if logger: + logger.info("Got speaks-for option but not a valid speaks_for with this credential: %s" % msg) + else: + print "Got a speaks-for option but not a valid speaks_for with this credential: " + msg + return caller_gid # Not speaks-for + +# Create an ABAC Speaks For credential using the ABACCredential object and it's encode&sign methods +def create_sign_abaccred(tool_gid, user_gid, ma_gid, user_key_file, cred_filename, dur_days=365): + print "Creating ABAC SpeaksFor using ABACCredential...\n" + # Write out the user cert + from tempfile import mkstemp + ma_str = ma_gid.save_to_string() + user_cert_str = user_gid.save_to_string() + if not user_cert_str.endswith(ma_str): + user_cert_str += ma_str + fp, user_cert_filename = mkstemp(suffix='cred', text=True) + fp = os.fdopen(fp, "w") + fp.write(user_cert_str) + fp.close() + + # Create the cred + cred = ABACCredential() + cred.set_issuer_keys(user_key_file, user_cert_filename) + tool_urn = tool_gid.get_urn() + user_urn = user_gid.get_urn() + user_keyid = get_cert_keyid(user_gid) + tool_keyid = get_cert_keyid(tool_gid) + cred.head = ABACElement(user_keyid, user_urn, "speaks_for_%s" % user_keyid) + cred.tails.append(ABACElement(tool_keyid, tool_urn)) + cred.set_expiration(datetime.datetime.utcnow() + datetime.timedelta(days=dur_days)) + cred.expiration = cred.expiration.replace(microsecond=0) + + # Produce the cred XML + cred.encode() + + # Sign it + cred.sign() + # Save it + cred.save_to_file(cred_filename) + print "Created ABAC credential: '%s' in file %s" % \ + (cred.get_summary_tostring(), cred_filename) + +# FIXME: Assumes xmlsec1 is on path +# FIXME: Assumes signer is itself signed by an 'ma_gid' that can be trusted +def create_speaks_for(tool_gid, user_gid, ma_gid, \ + user_key_file, cred_filename, dur_days=365): + tool_urn = tool_gid.get_urn() + user_urn = user_gid.get_urn() + + header = '' + reference = "ref0" + signature_block = \ + '\n' + \ + signature_template + \ + '' + template = header + '\n' + \ + '\n' + \ + 'abac\n' + \ + '\n' +\ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '%s' +\ + '\n' + \ + '\n' + \ + '%s\n' + \ + '\n' + \ + '%s%s\n' +\ + 'speaks_for_%s\n' + \ + '\n' + \ + '\n' +\ + '%s%s\n' +\ + '\n' +\ + '\n' + \ + '\n' + \ + '\n' + \ + signature_block + \ + '\n' + + + credential_duration = datetime.timedelta(days=dur_days) + expiration = datetime.datetime.utcnow() + credential_duration + expiration_str = expiration.strftime(SFATIME_FORMAT) + version = "1.1" + + user_keyid = get_cert_keyid(user_gid) + tool_keyid = get_cert_keyid(tool_gid) + unsigned_cred = template % (reference, expiration_str, version, \ + user_keyid, user_urn, user_keyid, tool_keyid, tool_urn, \ + reference, reference) + unsigned_cred_filename = write_to_tempfile(unsigned_cred) + + # Now sign the file with xmlsec1 + # xmlsec1 --sign --privkey-pem privkey.pem,cert.pem + # --output signed.xml tosign.xml + pems = "%s,%s,%s" % (user_key_file, user_gid.get_filename(), + ma_gid.get_filename()) + # FIXME: assumes xmlsec1 is on path + cmd = ['xmlsec1', '--sign', '--privkey-pem', pems, + '--output', cred_filename, unsigned_cred_filename] + +# print " ".join(cmd) + sign_proc_output = run_subprocess(cmd, stdout=subprocess.PIPE, stderr=None) + if sign_proc_output == None: + print "OUTPUT = %s" % sign_proc_output + else: + print "Created ABAC credential: '%s speaks_for %s' in file %s" % \ + (tool_urn, user_urn, cred_filename) + os.unlink(unsigned_cred_filename) + + +# Test procedure +if __name__ == "__main__": + + parser = optparse.OptionParser() + parser.add_option('--cred_file', + help='Name of credential file') + parser.add_option('--tool_cert_file', + help='Name of file containing tool certificate') + parser.add_option('--user_urn', + help='URN of speaks-for user') + parser.add_option('--user_cert_file', + help="filename of x509 certificate of signing user") + parser.add_option('--ma_cert_file', + help="filename of x509 cert of MA that signed user cert") + parser.add_option('--user_key_file', + help="filename of private key of signing user") + parser.add_option('--trusted_roots_directory', + help='Directory of trusted root certs') + parser.add_option('--create', + help="name of file of ABAC speaksfor cred to create") + parser.add_option('--useObject', action='store_true', default=False, + help='Use the ABACCredential object to create the credential (default False)') + + options, args = parser.parse_args(sys.argv) + + tool_gid = GID(filename=options.tool_cert_file) + + if options.create: + if options.user_cert_file and options.user_key_file \ + and options.ma_cert_file: + user_gid = GID(filename=options.user_cert_file) + ma_gid = GID(filename=options.ma_cert_file) + if options.useObject: + create_sign_abaccred(tool_gid, user_gid, ma_gid, \ + options.user_key_file, \ + options.create) + else: + create_speaks_for(tool_gid, user_gid, ma_gid, \ + options.user_key_file, \ + options.create) + else: + print "Usage: --create cred_file " + \ + "--user_cert_file user_cert_file" + \ + " --user_key_file user_key_file --ma_cert_file ma_cert_file" + sys.exit() + + user_urn = options.user_urn + + # Get list of trusted rootcerts + if options.cred_file and not options.trusted_roots_directory: + sys.exit("Must supply --trusted_roots_directory to validate a credential") + + trusted_roots_directory = options.trusted_roots_directory + trusted_roots = \ + [Certificate(filename=os.path.join(trusted_roots_directory, file)) \ + for file in os.listdir(trusted_roots_directory) \ + if file.endswith('.pem') and file != 'CATedCACerts.pem'] + + cred = open(options.cred_file).read() + + creds = [{'geni_type' : ABACCredential.ABAC_CREDENTIAL_TYPE, 'geni_value' : cred, + 'geni_version' : '1'}] + gid = determine_speaks_for(None, creds, tool_gid, \ + {'geni_speaking_for' : user_urn}, \ + trusted_roots) + + + print 'SPEAKS_FOR = %s' % (gid != tool_gid) + print "CERT URN = %s" % gid.get_urn() diff --git a/sfa/util/config.py b/sfa/util/config.py index c25ec442..797bed73 100644 --- a/sfa/util/config.py +++ b/sfa/util/config.py @@ -194,7 +194,8 @@ DO NOT EDIT. This file was automatically generated at return False - def dump(self, sections = []): + def dump(self, sections=None): + if sections is None: sections=[] sys.stdout.write(output_python()) def output_python(self, encoding = "utf-8"): diff --git a/sfa/util/sfatime.py b/sfa/util/sfatime.py index 992785e1..75a2e4a3 100644 --- a/sfa/util/sfatime.py +++ b/sfa/util/sfatime.py @@ -21,13 +21,15 @@ # IN THE WORK. #---------------------------------------------------------------------- from types import StringTypes -import dateutil.parser -import datetime import time +import datetime +import dateutil.parser +import calendar +import re from sfa.util.sfalogging import logger -DATEFORMAT = "%Y-%m-%dT%H:%M:%SZ" +SFATIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" def utcparse(input): """ Translate a string into a time using dateutil.parser.parse but make sure it's in UTC time and strip @@ -35,16 +37,36 @@ the timezone, so that it's compatible with normal datetime.datetime objects. For safety this can also handle inputs that are either timestamps, or datetimes """ + + def handle_shorthands (input): + """recognize string like +5d or +3w or +2m as + 2 days, 3 weeks or 2 months from now""" + if input.startswith('+'): + match=re.match (r"([0-9]+)([dwm])",input[1:]) + if match: + how_many=int(match.group(1)) + what=match.group(2) + if what == 'd': d=datetime.timedelta(days=how_many) + elif what == 'w': d=datetime.timedelta(weeks=how_many) + elif what == 'm': d=datetime.timedelta(weeks=4*how_many) + return datetime.datetime.utcnow()+d + # prepare the input for the checks below by # casting strings ('1327098335') to ints if isinstance(input, StringTypes): try: input = int(input) except ValueError: - pass + try: + new_input=handle_shorthands(input) + if new_input is not None: input=new_input + except: + import traceback + traceback.print_exc() + #################### here we go if isinstance (input, datetime.datetime): - logger.warn ("argument to utcparse already a datetime - doing nothing") + #logger.info ("argument to utcparse already a datetime - doing nothing") return input elif isinstance (input, StringTypes): t = dateutil.parser.parse(input) @@ -56,18 +78,38 @@ For safety this can also handle inputs that are either timestamps, or datetimes else: logger.error("Unexpected type in utcparse [%s]"%type(input)) -def datetime_to_string(input): - return datetime.datetime.strftime(input, DATEFORMAT) +def datetime_to_string(dt): + return datetime.datetime.strftime(dt, SFATIME_FORMAT) -def datetime_to_utc(input): - return time.gmtime(datetime_to_epoch(input)) +def datetime_to_utc(dt): + return time.gmtime(datetime_to_epoch(dt)) -def datetime_to_epoch(input): - return int(time.mktime(input.timetuple())) +# see https://docs.python.org/2/library/time.html +# all timestamps are in UTC so time.mktime() would be *wrong* +def datetime_to_epoch(dt): + return int(calendar.timegm(dt.timetuple())) -def adjust_datetime(input, days=0, hours=0, minutes=0, seconds=0): +def add_datetime(input, days=0, hours=0, minutes=0, seconds=0): """ Adjust the input date by the specified delta (in seconds). """ dt = utcparse(input) return dt + datetime.timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds) + +if __name__ == '__main__': + # checking consistency + print 20*'X' + print ("Should be close to zero: %s"%(datetime_to_epoch(datetime.datetime.utcnow())-time.time())) + print 20*'X' + for input in [ + '+2d', + '+3w', + '+2m', + 1401282977.575632, + 1401282977, + '1401282977', + '2014-05-28', + '2014-05-28T15:18', + '2014-05-28T15:18:30', + ]: + print "input=%20s -> parsed %s"%(input,datetime_to_string(utcparse(input))) diff --git a/sfa/util/storage.py b/sfa/util/storage.py index 793a38eb..89a25099 100644 --- a/sfa/util/storage.py +++ b/sfa/util/storage.py @@ -9,8 +9,8 @@ class SimpleStorage(dict): db_filename = None type = 'dict' - def __init__(self, db_filename, db = {}): - + def __init__(self, db_filename, db = None): + if db is None: db={} dict.__init__(self, db) self.db_filename = db_filename diff --git a/sfa/util/version.py.in b/sfa/util/version.py.in index 46b31d39..97ae6c40 100644 --- a/sfa/util/version.py.in +++ b/sfa/util/version.py.in @@ -3,7 +3,8 @@ version_tag="@VERSIONTAG@" scm_url="@SCMURL@" import socket -def version_core (more={}): +def version_core (more=None): + if more is None: more={} core = { 'code_tag' : version_tag, 'code_url' : scm_url, 'hostname' : socket.gethostname(), diff --git a/sfa/util/xml.py b/sfa/util/xml.py index d6734e63..ba324c7b 100755 --- a/sfa/util/xml.py +++ b/sfa/util/xml.py @@ -20,7 +20,8 @@ class XpathFilter: return xpath @staticmethod - def xpath(filter={}): + def xpath(filter=None): + if filter is None: filter={} xpath = "" if filter: filter_list = [] @@ -78,11 +79,12 @@ class XmlElement: def getparent(self): return XmlElement(self.element.getparent(), self.namespaces) - def get_instance(self, instance_class=None, fields=[]): + def get_instance(self, instance_class=None, fields=None): """ Returns an instance (dict) of this xml element. The instance holds a reference to this xml element. """ + if fields is None: fields=[] if not instance_class: instance_class = Element if not fields and hasattr(instance_class, 'fields'): @@ -97,11 +99,12 @@ class XmlElement: instance[field] = self.attrib[field] return instance - def add_instance(self, name, instance, fields=[]): + def add_instance(self, name, instance, fields=None): """ Adds the specifed instance(s) as a child element of this xml element. """ + if fields is None: fields=[] if not fields and hasattr(instance, 'keys'): fields = instance.keys() elem = self.add_element(name) diff --git a/sfa/util/xrn.py b/sfa/util/xrn.py index b0db4c1d..b16ea511 100644 --- a/sfa/util/xrn.py +++ b/sfa/util/xrn.py @@ -110,7 +110,8 @@ class Xrn: return Xrn.urn_meaningful(urn).split('+') @staticmethod - def filter_type(urns=[], type=None): + def filter_type(urns=None, type=None): + if urns is None: urns=[] urn_list = [] if not type: return urns diff --git a/testbeds/iotlab/tests/driver_tests.py b/testbeds/iotlab/tests/driver_tests.py index 77b813b0..0a473e21 100644 --- a/testbeds/iotlab/tests/driver_tests.py +++ b/testbeds/iotlab/tests/driver_tests.py @@ -14,6 +14,7 @@ from sfa.util.sfalogging import logger #OAR imports from datetime import datetime +from sfa.util.sfatime import SFATIME_FORMAT from sfa.iotlab.OARrestapi import OARrestapi #Test iotlabdriver @@ -240,11 +241,9 @@ def TestOAR(job_id = None): message_and_wait("\r\n Get server's date and timezone") - time_format = "%Y-%m-%d %H:%M:%S" server_timestamp, server_tz = oar.parser.SendRequest("GET_timezone") print "\r\n OAR GetTimezone ", server_timestamp, server_tz - print(datetime.fromtimestamp(int(server_timestamp)).strftime( - time_format)) + print(datetime.fromtimestamp(int(server_timestamp)).strftime(SFATIME_FORMAT)) message_and_wait("\r\n Get all the resources with details from OAR") uri = '/oarapi/resources/full.json' @@ -380,4 +379,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec index dd582056..72b1e80f 100644 --- a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec @@ -1,5 +1,5 @@ - + wsn430-12.devlille.iot-lab.info @@ -16,7 +16,7 @@ - +