From: Sandrine Avakian Date: Mon, 8 Oct 2012 11:28:08 +0000 (+0200) Subject: Merge branch 'master' into senslab2 X-Git-Tag: sfa-2.1-24~3^2~80 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=dbd15819d950b69d0c6ba5527660969d0754ad29;hp=-c;p=sfa.git Merge branch 'master' into senslab2 Conflicts: sfa/client/client_helper.py sfa/managers/registry_manager.py --- dbd15819d950b69d0c6ba5527660969d0754ad29 diff --combined setup.py index 69a9e3c0,041904ca..cb3e98e8 --- a/setup.py +++ b/setup.py @@@ -31,21 -31,13 +31,22 @@@ packages = 'sfa/generic', 'sfa/managers', 'sfa/importer', + + + + 'sfa/senslab', + + + + + 'sfa/rspecs', 'sfa/rspecs/elements', 'sfa/rspecs/elements/versions', 'sfa/rspecs/versions', 'sfa/client', 'sfa/planetlab', + 'sfa/nitos', 'sfa/openstack', 'sfa/federica', 'sfatables', diff --combined sfa/client/sfi.py index e8372700,b0f23eb3..cc8ef3f4 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@@ -197,6 -197,54 +197,54 @@@ def save_record_to_file(filename, recor f.close() return + # used in sfi list + def terminal_render (records,options): + # sort records by type + grouped_by_type={} + for record in records: + type=record['type'] + if type not in grouped_by_type: grouped_by_type[type]=[] + grouped_by_type[type].append(record) + group_types=grouped_by_type.keys() + group_types.sort() + for type in group_types: + group=grouped_by_type[type] + # print 20 * '-', type + try: renderer=eval('terminal_render_'+type) + except: renderer=terminal_render_default + for record in group: renderer(record,options) + + def render_plural (how_many, name,names=None): + if not names: names="%ss"%name + if how_many<=0: return "No %s"%name + elif how_many==1: return "1 %s"%name + else: return "%d %s"%(how_many,names) + + def terminal_render_default (record,options): + print "%s (%s)" % (record['hrn'], record['type']) + def terminal_render_user (record, options): + print "%s (User)"%record['hrn'], + if record.get('reg-pi-authorities',None): print " [PI at %s]"%(" and ".join(record['reg-pi-authorities'])), + if record.get('reg-slices',None): print " [IN slices %s]"%(" and ".join(record['reg-slices'])), + user_keys=record.get('reg-keys',[]) + if not options.verbose: + print " [has %s]"%(render_plural(len(user_keys),"key")) + else: + print "" + for key in user_keys: print 8*' ',key.strip("\n") + + def terminal_render_slice (record, options): + print "%s (Slice)"%record['hrn'], + if record.get('reg-researchers',None): print " [USERS %s]"%(" and ".join(record['reg-researchers'])), + # print record.keys() + print "" + def terminal_render_authority (record, options): + print "%s (Authority)"%record['hrn'], + if record.get('reg-pis',None): print " [PIS %s]"%(" and ".join(record['reg-pis'])), + print "" + def terminal_render_node (record, options): + print "%s (Node)"%record['hrn'] + # minimally check a key argument def check_ssh_key (key): good_ssh_key = r'^.*(?:ssh-dss|ssh-rsa)[ ]+[A-Za-z0-9+/=]+(?: .*)?$' @@@ -418,6 -466,8 +466,8 @@@ class Sfi if command == 'list': parser.add_option("-r", "--recursive", dest="recursive", action='store_true', help="list all child records", default=False) + parser.add_option("-v", "--verbose", dest="verbose", action='store_true', + help="gives details, like user keys", default=False) if command in ("delegate"): parser.add_option("-u", "--user", action="store_true", dest="delegate_user", default=False, @@@ -488,7 -538,11 +538,11 @@@ # Main: parse arguments and dispatch to command # def dispatch(self, command, command_options, command_args): - return getattr(self, command)(command_options, command_args) + method=getattr(self, command,None) + if not method: + print "Unknown command %s"%command + return + return method(command_options, command_args) def main(self): self.sfi_parser = self.create_parser() @@@ -523,8 -577,8 +577,8 @@@ try: self.dispatch(command, command_options, command_args) - except KeyError: - self.logger.critical ("Unknown command %s"%command) + except: + self.logger.log_exc ("sfi command %s failed"%command) sys.exit(1) return @@@ -856,10 -910,9 +910,9 @@@ or version information about sfi itsel raise Exception, "Not enough parameters for the 'list' command" # filter on person, slice, site, node, etc. - # THis really should be in the self.filter_records funct def comment... + # This really should be in the self.filter_records funct def comment... list = filter_records(options.type, list) - for record in list: - print "%s (%s)" % (record['hrn'], record['type']) + terminal_render (list, options) if options.file: save_records_to_file(options.file, list, options.fileformat) return @@@ -872,7 -925,8 +925,8 @@@ self.print_help() sys.exit(1) hrn = args[0] - record_dicts = self.registry().Resolve(hrn, self.my_credential_string) + # explicitly require Resolve to run in details mode + record_dicts = self.registry().Resolve(hrn, self.my_credential_string, {'details':True}) record_dicts = filter_records(options.type, record_dicts) if not record_dicts: self.logger.error("No record of type %s"% options.type) @@@ -1097,10 -1151,14 +1151,14 @@@ or with an slice hrn, shows currently p # keys: [, ] # }] users = [] + # xxx Thierry 2012 sept. 21 + # contrary to what I was first thinking, calling Resolve with details=False does not yet work properly here + # I am turning details=True on again on a - hopefully - temporary basis, just to get this whole thing to work again slice_records = self.registry().Resolve(slice_urn, [self.my_credential_string]) - if slice_records and 'researcher' in slice_records[0] and slice_records[0]['researcher']!=[]: + # slice_records = self.registry().Resolve(slice_urn, [self.my_credential_string], {'details':True}) + if slice_records and 'reg-researchers' in slice_records[0] and slice_records[0]['reg-researchers']: slice_record = slice_records[0] - user_hrns = slice_record['researcher'] + user_hrns = slice_record['reg-researchers'] user_urns = [hrn_to_urn(hrn, 'user') for hrn in user_hrns] user_records = self.registry().Resolve(user_urns, [self.my_credential_string]) @@@ -1110,7 -1168,6 +1168,7 @@@ rspec.filter({'component_manager_id': server_version['urn']}) rspec = RSpecConverter.to_pg_rspec(rspec.toxml(), content_type='request') else: + print >>sys.stderr, "\r\n \r\n \r\n WOOOOOO" users = sfa_users_arg(user_records, slice_record) # do not append users, keys, or slice tags. Anything diff --combined sfa/managers/driver.py index 6b2681ca,f049f549..ab3b2165 --- a/sfa/managers/driver.py +++ b/sfa/managers/driver.py @@@ -2,7 -2,7 +2,7 @@@ # an attempt to document what a driver class should provide, # and implement reasonable defaults # - +import sys class Driver: def __init__ (self, config): @@@ -20,13 -20,10 +20,11 @@@ # the following is used in Resolve (registry) when run in full mode # after looking up the sfa db, we wish to be able to display # testbed-specific info as well - # this at minima should fill in the 'researcher' field for slice records - # as this information is then used to compute rights - # roadmap: there is an intention to redesign the SFA database so as to clear up - # this constraint, based on the principle that SFA should not rely on the - # testbed database to perform such a core operation (i.e. getting rights right) + # based on the principle that SFA should not rely on the testbed database + # to perform such a core operation (i.e. getting rights right) + # this is no longer in use when performing other SFA operations def augment_records_with_testbed_info (self, sfa_records): + print >>sys.stderr, " \r\n \r\n DRIVER.PY augment_records_with_testbed_info sfa_records ",sfa_records return sfa_records # incoming record, as provided by the client to the Register API call diff --combined sfa/managers/registry_manager.py index 2f530904,564fb44c..d1a4c6f5 --- a/sfa/managers/registry_manager.py +++ b/sfa/managers/registry_manager.py @@@ -17,8 -17,12 +17,12 @@@ from sfa.trust.credential import Creden from sfa.trust.certificate import Certificate, Keypair, convert_public_key from sfa.trust.gid import create_uuid - from sfa.storage.model import make_record, RegRecord, RegAuthority, RegUser, RegSlice, RegKey + from sfa.storage.model import make_record, RegRecord, RegAuthority, RegUser, RegSlice, RegKey, \ + augment_with_sfa_builtins from sfa.storage.alchemy import dbsession + ### the types that we need to exclude from sqlobjects before being able to dump + # them on the xmlrpc wire + from sqlalchemy.orm.collections import InstrumentedList class RegistryManager: @@@ -53,13 -57,6 +57,6 @@@ if not record: raise RecordNotFound("hrn=%s, type=%s"%(hrn,type)) - # xxx for the record only - # used to call this, which was wrong, now all needed data is natively is our DB - # self.driver.augment_records_with_testbed_info (record.__dict__) - # likewise, we deprecate is_enabled which was not really useful - # if not self.driver.is_enabled (record.__dict__): ... - # xxx for the record only - # get the callers gid # if caller_xrn is not specified assume the caller is the record # object itself. @@@ -104,7 -101,8 +101,8 @@@ return new_cred.save_to_string(save_parents=True) - def Resolve(self, api, xrns, type=None, full=True): + # the default for full, which means 'dig into the testbed as well', should be false + def Resolve(self, api, xrns, type=None, details=False): if not isinstance(xrns, types.ListType): # try to infer type if not set and we get a single input @@@ -142,6 -140,8 +140,8 @@@ credential = api.getCredential() interface = api.registries[registry_hrn] server_proxy = api.server_proxy(interface, credential) + # should propagate the details flag but that's not supported in the xmlrpc interface yet + #peer_records = server_proxy.Resolve(xrns, credential,type, details=details) peer_records = server_proxy.Resolve(xrns, credential,type) # pass foreign records as-is # previous code used to read @@@ -155,19 -155,21 +155,21 @@@ local_records = dbsession.query(RegRecord).filter(RegRecord.hrn.in_(local_hrns)) if type: local_records = local_records.filter_by(type=type) - local_records=local_records.all() + local_records=local_records.all() - logger.info("Resolve: local_records=%s (type=%s)"%(local_records,type)) + + for local_record in local_records: + augment_with_sfa_builtins (local_record) + + logger.info("Resolve, (details=%s,type=%s) local_records=%s "%(details,type,local_records)) local_dicts = [ record.__dict__ for record in local_records ] - if full: - # in full mode we get as much info as we can, which involves contacting the + if details: + # in details mode we get as much info as we can, which involves contacting the # testbed for getting implementation details about the record self.driver.augment_records_with_testbed_info(local_dicts) - #logger.debug("Resolve: local_dicts =%s "%(local_dicts)) # also we fill the 'url' field for known authorities # used to be in the driver code, sounds like a poorman thing though def solve_neighbour_url (record): - logger.debug("\r\n \t\t solve_neighbour_url: record = %s "%(record)) if not record.type.startswith('authority'): return hrn=record.hrn for neighbour_dict in [ api.aggregates, api.registries ]: @@@ -175,13 -177,13 +177,12 @@@ record.url=neighbour_dict[hrn].get_url() return for record in local_records: solve_neighbour_url (record) - #logger.debug("\solve_neighbour_url = OK ") - # convert local record objects to dicts for xmlrpc # xxx somehow here calling dict(record) issues a weird error # however record.todict() seems to work fine # records.extend( [ dict(record) for record in local_records ] ) - records.extend( [ record.todict() for record in local_records ] ) - #logger.debug("\RESOLVE = records %s " %(records)) + records.extend( [ record.todict(exclude_types=[InstrumentedList]) for record in local_records ] ) + if not records: raise RecordNotFound(str(hrns)) @@@ -227,7 -229,9 +228,9 @@@ records = dbsession.query(RegRecord).filter(RegRecord.hrn.startswith(hrn)) else: records = dbsession.query(RegRecord).filter_by(authority=hrn) - record_dicts=[ record.todict() for record in records ] + # so that sfi list can show more than plain names... + for record in records: augment_with_sfa_builtins (record) + record_dicts=[ record.todict(exclude_types=[InstrumentedList]) for record in records ] return record_dicts @@@ -246,12 -250,10 +249,10 @@@ #################### # utility for handling relationships among the SFA objects - # given that the SFA db does not handle this sort of relationsships - # it will rely on side-effects in the testbed to keep this persistent # subject_record describes the subject of the relationships # ref_record contains the target values for the various relationships we need to manage - # (to begin with, this is just the slice x person relationship) + # (to begin with, this is just the slice x person (researcher) and authority x person (pi) relationships) def update_driver_relations (self, subject_obj, ref_obj): type=subject_obj.type #for (k,v) in subject_obj.__dict__.items(): print k,'=',v diff --combined sfa/planetlab/plslices.py index 58ee4f42,0d76b593..2de659eb --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@@ -1,6 -1,5 +1,6 @@@ from types import StringTypes from collections import defaultdict +import sys from sfa.util.sfatime import utcparse, datetime_to_epoch from sfa.util.sfalogging import logger @@@ -9,7 -8,9 +9,9 @@@ from sfa.util.xrn import Xrn, get_leaf from sfa.rspecs.rspec import RSpec from sfa.planetlab.vlink import VLink - from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename + from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename, xrn_to_hostname + + import time MAXINT = 2L**31-1 @@@ -131,11 -132,13 +133,11 @@@ class PlSlices # slice belongs to out local plc or a myplc peer. We will assume it # is a local site, unless we find out otherwise peer = None - # get this slice's authority (site) slice_authority = get_authority(hrn) # get this site's authority (sfa root authority or sub authority) site_authority = get_authority(slice_authority).lower() - # check if we are already peered with this site_authority, if so peers = self.driver.shell.GetPeers({}, ['peer_id', 'peername', 'shortname', 'hrn_root']) for peer_record in peers: @@@ -158,19 -161,54 +160,54 @@@ return sfa_peer - def verify_slice_leases(self, slice, requested_leases, kept_leases, peer): - - leases = self.driver.shell.GetLeases({'name':slice['name']}, ['lease_id']) + def verify_slice_leases(self, slice, rspec_requested_leases, peer): + + leases = self.driver.shell.GetLeases({'name':slice['name'], 'clip':int(time.time())}, ['lease_id','name', 'hostname', 't_from', 't_until']) grain = self.driver.shell.GetLeaseGranularity() - current_leases = [lease['lease_id'] for lease in leases] - deleted_leases = list(set(current_leases).difference(kept_leases)) + + requested_leases = [] + for lease in rspec_requested_leases: + requested_lease = {} + slice_name = hrn_to_pl_slicename(lease['slice_id']) + if slice_name != slice['name']: + continue + elif Xrn(lease['component_id']).get_authority_urn().split(':')[0] != self.driver.hrn: + continue + + hostname = xrn_to_hostname(lease['component_id']) + # fill the requested node with nitos ids + requested_lease['name'] = slice['name'] + requested_lease['hostname'] = hostname + requested_lease['t_from'] = int(lease['start_time']) + requested_lease['t_until'] = int(lease['duration']) * grain + int(lease['start_time']) + requested_leases.append(requested_lease) + + + + # prepare actual slice leases by lease_id + leases_by_id = {} + for lease in leases: + leases_by_id[lease['lease_id']] = {'name': lease['name'], 'hostname': lease['hostname'], \ + 't_from': lease['t_from'], 't_until': lease['t_until']} + + added_leases = [] + kept_leases_id = [] + deleted_leases_id = [] + for lease_id in leases_by_id: + if leases_by_id[lease_id] not in requested_leases: + deleted_leases_id.append(lease_id) + else: + kept_leases_id.append(lease_id) + requested_leases.remove(leases_by_id[lease_id]) + added_leases = requested_leases + try: if peer: self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) - deleted=self.driver.shell.DeleteLeases(deleted_leases) - for lease in requested_leases: - added=self.driver.shell.AddLeases(lease['hostname'], slice['name'], int(lease['start_time']), int(lease['duration']) * grain + int(lease['start_time'])) + self.driver.shell.DeleteLeases(deleted_leases_id) + for lease in added_leases: + self.driver.shell.AddLeases(lease['hostname'], slice['name'], lease['t_from'], lease['t_until']) except: logger.log_exc('Failed to add/remove slice leases') @@@ -178,11 -216,27 +215,27 @@@ return leases - def verify_slice_nodes(self, slice, requested_slivers, peer): + def verify_slice_nodes(self, slice, slivers, peer): nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids']) current_slivers = [node['hostname'] for node in nodes] + requested_slivers = [] + tags = [] + for node in slivers: + hostname = None + if node.get('component_name'): + hostname = node.get('component_name').strip() + elif node.get('component_id'): + hostname = xrn_to_hostname(node.get('component_id').strip()) + if node.get('client_id'): + tags.append({'slicename': slice['name'], + 'tagname': 'client_id', + 'value': node['client_id'], + 'node': hostname}) + if hostname: + requested_slivers.append(hostname) + # remove nodes not in rspec deleted_nodes = list(set(current_slivers).difference(requested_slivers)) @@@ -194,9 -248,16 +247,16 @@@ self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) self.driver.shell.AddSliceToNodes(slice['name'], added_nodes) self.driver.shell.DeleteSliceFromNodes(slice['name'], deleted_nodes) - + except: logger.log_exc('Failed to add/remove slice from nodes') + + # add tags + for tag in tags: + try: + self.driver.shell.AddSliceTag(tag['slicename'], tag['tagname'], tag['value'], tag['node']) + except: + logger.log_exc('Failed to add slice tag') return nodes def free_egre_key(self): diff --combined sfa/rspecs/elements/versions/sfav1Lease.py index f36418d8,03a43422..6275121b --- a/sfa/rspecs/elements/versions/sfav1Lease.py +++ b/sfa/rspecs/elements/versions/sfav1Lease.py @@@ -22,7 -22,7 +22,7 @@@ class SFAv1Lease @staticmethod def add_leases(xml, leases): - + logger.debug("SFAV1LEASE \t add_lease ") network_elems = xml.xpath('//network') if len(network_elems) > 0: network_elem = network_elems[0] @@@ -32,13 -32,43 +32,44 @@@ else: network_elem = xml - lease_elems = [] - for lease in leases: - lease_fields = ['lease_id', 'component_id', 'slice_id', 'start_time', 'duration'] - lease_elem = network_elem.add_instance('lease', lease, lease_fields) + # group the leases by slice and timeslots + grouped_leases = [] + + while leases: + slice_id = leases[0]['slice_id'] + start_time = leases[0]['start_time'] + duration = leases[0]['duration'] + group = [] + + for lease in leases: + if slice_id == lease['slice_id'] and start_time == lease['start_time'] and duration == lease['duration']: + group.append(lease) + + grouped_leases.append(group) + + for lease1 in group: + leases.remove(lease1) + + lease_elems = [] + for lease in grouped_leases: + #lease_fields = ['lease_id', 'component_id', 'slice_id', 'start_time', 'duration'] + lease_fields = ['slice_id', 'start_time', 'duration'] + lease_elem = network_elem.add_instance('lease', lease[0], lease_fields) lease_elems.append(lease_elem) + logger.debug("SFAV1LEASE \t add_lease lease %s" %(lease)) + # add nodes of this lease + for node in lease: + lease_elem.add_instance('node', node, ['component_id']) + + + + # lease_elems = [] + # for lease in leases: + # lease_fields = ['lease_id', 'component_id', 'slice_id', 'start_time', 'duration'] + # lease_elem = network_elem.add_instance('lease', lease, lease_fields) + # lease_elems.append(lease_elem) + @staticmethod def get_leases(xml, filter={}): @@@ -48,16 -78,34 +79,34 @@@ @staticmethod def get_lease_objs(lease_elems): - leases = [] + leases = [] for lease_elem in lease_elems: - lease = Lease(lease_elem.attrib, lease_elem) - if lease.get('lease_id'): - lease['lease_id'] = lease_elem.attrib['lease_id'] - lease['component_id'] = lease_elem.attrib['component_id'] - lease['slice_id'] = lease_elem.attrib['slice_id'] - lease['start_time'] = lease_elem.attrib['start_time'] - lease['duration'] = lease_elem.attrib['duration'] - - leases.append(lease) - return leases + #get nodes + node_elems = lease_elem.xpath('./default:node | ./node') + for node_elem in node_elems: + lease = Lease(lease_elem.attrib, lease_elem) + lease['slice_id'] = lease_elem.attrib['slice_id'] + lease['start_time'] = lease_elem.attrib['start_time'] + lease['duration'] = lease_elem.attrib['duration'] + lease['component_id'] = node_elem.attrib['component_id'] + leases.append(lease) + + return leases + + + + + + # leases = [] + # for lease_elem in lease_elems: + # lease = Lease(lease_elem.attrib, lease_elem) + # if lease.get('lease_id'): + # lease['lease_id'] = lease_elem.attrib['lease_id'] + # lease['component_id'] = lease_elem.attrib['component_id'] + # lease['slice_id'] = lease_elem.attrib['slice_id'] + # lease['start_time'] = lease_elem.attrib['start_time'] + # lease['duration'] = lease_elem.attrib['duration'] + + # leases.append(lease) + # return leases