From: Mohamed Larabi Date: Mon, 17 Sep 2012 12:09:09 +0000 (+0200) Subject: bugs fix X-Git-Tag: sfa-2.1-15~28 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=65140a14a72e7009d8fd2f1d1f35e42ec98edc21 bugs fix --- diff --git a/sfa/importer/nitosimporter.py b/sfa/importer/nitosimporter.py index 3a8f90b1..080417df 100644 --- a/sfa/importer/nitosimporter.py +++ b/sfa/importer/nitosimporter.py @@ -122,7 +122,7 @@ class NitosImporter: # Get all nitos nodes nodes = shell.getNodes({}, []) # create hash by node_id - nodes_by_id = dict ( [ (node['id'], node) for node in nodes ] ) + nodes_by_id = dict ( [ (node['node_id'], node) for node in nodes ] ) # Get all nitos slices slices = shell.getSlices({}, []) # create hash by slice_id @@ -164,7 +164,7 @@ class NitosImporter: for node in nodes: site_auth = get_authority(site_hrn) site_name = site['name'] - node_hrn = hostname_to_hrn(site_auth, site_name, node['name']) + node_hrn = hostname_to_hrn(site_auth, site_name, node['hostname']) # xxx this sounds suspicious if len(node_hrn) > 64: node_hrn = node_hrn[:64] node_record = self.locate_by_type_hrn ( 'node', node_hrn ) @@ -174,7 +174,7 @@ class NitosImporter: urn = hrn_to_urn(node_hrn, 'node') node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) node_record = RegNode (hrn=node_hrn, gid=node_gid, - pointer =node['id'], + pointer =node['node_id'], authority=get_authority(node_hrn)) node_record.just_created() dbsession.add(node_record) @@ -294,7 +294,7 @@ class NitosImporter: pass # record current users affiliated with the slice slice_record.reg_researchers = \ - [ self.locate_by_type_pointer ('user',int(user_id)) for user_id in slice['user_id'] ] + [ self.locate_by_type_pointer ('user',int(user_id)) for user_id in slice['user_ids'] ] dbsession.commit() slice_record.stale=False diff --git a/sfa/nitos/nitosaggregate.py b/sfa/nitos/nitosaggregate.py index 2799a30f..e37624fe 100644 --- a/sfa/nitos/nitosaggregate.py +++ b/sfa/nitos/nitosaggregate.py @@ -29,77 +29,6 @@ class NitosAggregate: def __init__(self, driver): self.driver = driver - - def get_sites(self, filter={}): - sites = {} - for site in self.driver.shell.GetSites(filter): - sites[site['site_id']] = site - return sites - - def get_interfaces(self, filter={}): - interfaces = {} - for interface in self.driver.shell.GetInterfaces(filter): - iface = Interface() - if interface['bwlimit']: - interface['bwlimit'] = str(int(interface['bwlimit'])/1000) - interfaces[interface['interface_id']] = interface - return interfaces - - def get_links(self, sites, nodes, interfaces): - - topology = Topology() - links = [] - for (site_id1, site_id2) in topology: - site_id1 = int(site_id1) - site_id2 = int(site_id2) - link = Link() - if not site_id1 in sites or site_id2 not in sites: - continue - site1 = sites[site_id1] - site2 = sites[site_id2] - # get hrns - site1_hrn = self.driver.hrn + '.' + site1['login_base'] - site2_hrn = self.driver.hrn + '.' + site2['login_base'] - - for s1_node_id in site1['node_ids']: - for s2_node_id in site2['node_ids']: - if s1_node_id not in nodes or s2_node_id not in nodes: - continue - node1 = nodes[s1_node_id] - node2 = nodes[s2_node_id] - # set interfaces - # just get first interface of the first node - if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id'])) - if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip'] - if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id'])) - if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip'] - - if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} ) - if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} ) - - # set link - link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'}) - link['interface1'] = if1 - link['interface2'] = if2 - link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base']) - link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn() - link['component_manager_id'] = hrn_to_urn(self.driver.hrn, 'authority+am') - links.append(link) - - return links - - def get_node_tags(self, filter={}): - node_tags = {} - for node_tag in self.driver.shell.GetNodeTags(filter): - node_tags[node_tag['node_tag_id']] = node_tag - return node_tags - - def get_pl_initscripts(self, filter={}): - pl_initscripts = {} - filter.update({'enabled': True}) - for initscript in self.driver.shell.GetInitScripts(filter): - pl_initscripts[initscript['initscript_id']] = initscript - return pl_initscripts def get_slice_and_slivers(self, slice_xrn): @@ -114,30 +43,38 @@ class NitosAggregate: slice_hrn, _ = urn_to_hrn(slice_xrn) slice_name = hrn_to_nitos_slicename(slice_hrn) slices = self.driver.shell.getSlices({'slice_name': slice_name}, []) - # filter results - #for slc in slices: - # if slc['slice_name'] == slice_name: - # slice = slc - # break + #filter results + for slc in slices: + if slc['slice_name'] == slice_name: + slice = slc + break if not slice: return (slice, slivers) reserved_nodes = self.driver.shell.getReservedNodes({'slice_id': slice['slice_id']}, []) - + reserved_node_ids = [] + # filter on the slice for node in reserved_nodes: - slivers[node['id']] = node - + if node['slice_id'] == slice['slice_id']: + reserved_node_ids.append(node['node_id']) + #get all the nodes + all_nodes = self.driver.shell.getNodes({}, []) + + for node in all_nodes: + if node['node_id'] in reserved_node_ids: + slivers[node['node_id']] = node + return (slice, slivers) - def get_nodes_and_links(self, slice_xrn, slice=None,slivers={}, options={}): + def get_nodes(self, slice_xrn, slice=None,slivers={}, options={}): # if we are dealing with a slice that has no node just return # and empty list if slice_xrn: if not slice or not slivers: - return ([],[]) + return [] else: nodes = [slivers[sliver] for sliver in slivers] else: @@ -147,14 +84,13 @@ class NitosAggregate: grain = self.driver.testbedInfo['grain'] #grain = 1800 - rspec_nodes = [] for node in nodes: rspec_node = Node() site_name = self.driver.testbedInfo['name'] - rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site_name, node['name']) - rspec_node['component_name'] = node['name'] + rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site_name, node['hostname']) + rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn() rspec_node['authority_id'] = hrn_to_urn(NitosXrn.site_hrn(self.driver.hrn, site_name), 'authority+sa') # do not include boot state ( element) in the manifest rspec @@ -168,7 +104,7 @@ class NitosAggregate: location = Location({'longitude': longitude, 'latitude': latitude, 'country': 'unknown'}) rspec_node['location'] = location # 3D position - position_3d = Position3D({'x': node['X'], 'y': node['Y'], 'z': node['Z']}) + position_3d = Position3D({'x': node['position']['X'], 'y': node['position']['Y'], 'z': node['position']['Z']}) #position_3d = Position3D({'x': 1, 'y': 2, 'z': 3}) rspec_node['position_3d'] = position_3d # Granularity @@ -176,12 +112,20 @@ class NitosAggregate: rspec_node['granularity'] = granularity # HardwareType - rspec_node['hardware_type'] = node['type'] + rspec_node['hardware_type'] = node['node_type'] #rspec_node['hardware_type'] = "orbit" + + #slivers + if node['node_id'] in slivers: + # add sliver info + sliver = slivers[node['node_id']] + rspec_node['sliver_id'] = sliver['node_id'] + rspec_node['client_id'] = node['hostname'] + rspec_node['slivers'] = [sliver] rspec_nodes.append(rspec_node) - return (rspec_nodes, []) + return rspec_nodes def get_leases_and_channels(self, slice=None, options={}): @@ -193,10 +137,14 @@ class NitosAggregate: grain = self.driver.testbedInfo['grain'] if slice: - for lease in leases: + all_leases = [] + all_leases.extend(leases) + all_reserved_channels = [] + all_reserved_channels.extend(reserved_channels) + for lease in all_leases: if lease['slice_id'] != slice['slice_id']: leases.remove(lease) - for channel in reserved_channels: + for channel in all_reserved_channels: if channel['slice_id'] != slice['slice_id']: reserved_channels.remove(channel) @@ -234,8 +182,8 @@ class NitosAggregate: rspec_lease['lease_id'] = lease['reservation_id'] # retreive node name for node in nodes: - if node['id'] == lease['node_id']: - nodename = node['name'] + if node['node_id'] == lease['node_id']: + nodename = node['hostname'] break rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, self.driver.testbedInfo['name'], nodename) @@ -255,10 +203,23 @@ class NitosAggregate: return (rspec_leases, rspec_channels) - def get_channels(self, options={}): - - filter = {} - channels = self.driver.shell.getChannels({}, []) + def get_channels(self, slice=None, options={}): + + all_channels = self.driver.shell.getChannels({}, []) + channels = [] + if slice: + reserved_channels = self.driver.shell.getReservedChannels() + reserved_channel_ids = [] + for channel in reserved_channels: + if channel['slice_id'] == slice['slice_id']: + reserved_channel_ids.append(channel['channel_id']) + + for channel in all_channels: + if channel['channel_id'] in reserved_channel_ids: + channels.append(channel) + else: + channels = all_channels + rspec_channels = [] for channel in channels: rspec_channel = Channel() @@ -288,9 +249,8 @@ class NitosAggregate: rspec.xml.set('expires', datetime_to_string(utcparse(slice['expires']))) if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'leases': - nodes, links = self.get_nodes_and_links(slice_xrn, slice, slivers, options) + nodes = self.get_nodes(slice_xrn, slice, slivers, options) rspec.version.add_nodes(nodes) - rspec.version.add_links(links) # add sliver defaults default_sliver = slivers.get(None, []) if default_sliver: @@ -299,7 +259,7 @@ class NitosAggregate: logger.info(attrib) rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value']) # add wifi channels - channels = self.get_channels() + channels = self.get_channels(slice, options) rspec.version.add_channels(channels) if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources': diff --git a/sfa/nitos/nitosdriver.py b/sfa/nitos/nitosdriver.py index b917ee12..3b1f30e1 100644 --- a/sfa/nitos/nitosdriver.py +++ b/sfa/nitos/nitosdriver.py @@ -7,7 +7,7 @@ from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \ from sfa.util.sfalogging import logger from sfa.util.defaultdict import defaultdict from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch -from sfa.util.xrn import hrn_to_urn, get_leaf +from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf, urn_to_hrn from sfa.util.cache import Cache # one would think the driver should not need to mess with the SFA db, but.. @@ -29,7 +29,6 @@ from sfa.nitos.nitosslices import NitosSlices from sfa.nitos.nitosxrn import NitosXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_nitos_slicename, xrn_to_hostname - def list_to_dict(recs, key): """ convert a list of dictionaries into a dictionary keyed on the @@ -52,11 +51,47 @@ class NitosDriver (Driver): self.shell = NitosShell (config) self.cache=None self.testbedInfo = self.shell.getTestbedInfo() - if config.SFA_AGGREGATE_CACHING: - if NitosDriver.cache is None: - NitosDriver.cache = Cache() - self.cache = NitosDriver.cache +# un-comment below lines to enable caching +# if config.SFA_AGGREGATE_CACHING: +# if NitosDriver.cache is None: +# NitosDriver.cache = Cache() +# self.cache = NitosDriver.cache + ########################################### + ########## utility methods for NITOS driver + ########################################### + + + def filter_nitos_results (self, listo, filters_dict): + """ + the Nitos scheduler API does not provide a get result filtring so we do it here + """ + mylist = [] + mylist.extend(listo) + for dicto in mylist: + for filter in filters_dict: + if filter not in dicto or dicto[filter] != filters_dict[filter]: + listo.remove(dicto) + break + return listo + + def convert_id (self, list_of_dict): + """ + convert object id retrived in string format to int format + """ + for dicto in list_of_dict: + for key in dicto: + if key in ['node_id', 'slice_id', 'user_id', 'channel_id', 'reservation_id'] and isinstance(dicto[key], str): + dicto[key] = int(dicto[key]) + elif key in ['user_ids']: + user_ids2 = [] + for user_id in dicto['user_ids']: + user_ids2.append(int(user_id)) + dicto['user_ids'] = user_ids2 + return list_of_dict + + + ######################################## ########## registry oriented ######################################## @@ -70,7 +105,7 @@ class NitosDriver (Driver): nitos_record = self.sfa_fields_to_nitos_fields(type, hrn, sfa_record) if type == 'authority': - pointer = 1 + pointer = -1 elif type == 'slice': slices = self.shell.getSlices() @@ -81,7 +116,7 @@ class NitosDriver (Driver): break if not slice_id: - pointer = self.shell.addSlice({slice_name : nitos_record['name']}) + pointer = self.shell.addSlice({'slice_name' : nitos_record['name']}) else: pointer = slice_id @@ -93,22 +128,20 @@ class NitosDriver (Driver): user_id = user['user_id'] break if not user_id: - pointer = self.shell.addUser({username : nitos_record['name'], email : nitos_record['email']}) + pointer = self.shell.addUser({'username' : nitos_record['name'], 'email' : nitos_record['email']}) else: pointer = user_id - # What roles should this user have? # Add the user's key if pub_key: - self.shell.addUserKey({user_id : pointer,'key' : pub_key}) + self.shell.addUserKey({'user_id' : pointer,'key' : pub_key}) elif type == 'node': - login_base = PlXrn(xrn=sfa_record['authority'],type='node').pl_login_base() - nodes = self.shell.GetNodes([pl_record['hostname']]) + nodes = self.shell.GetNodes({}, []) # filter nodes for node in nodes: - if node['node_name'] == nitos_record['name']: + if node['hostname'] == nitos_record['name']: node_id = node['node_id'] break @@ -121,96 +154,59 @@ class NitosDriver (Driver): ########## def update (self, old_sfa_record, new_sfa_record, hrn, new_key): - """ + pointer = old_sfa_record['pointer'] type = old_sfa_record['type'] + new_nitos_record = self.sfa_fields_to_nitos_fields(type, hrn, new_sfa_record) # new_key implemented for users only if new_key and type not in [ 'user' ]: raise UnknownSfaType(type) - if (type == "authority"): - #self.shell.UpdateSite(pointer, new_sfa_record) - pass - - elif type == "slice": - nitos_record=self.sfa_fields_to_nitos_fields(type, hrn, new_sfa_record) - if 'name' in nitos_record: - nitos_record.pop('name') - self.shell.updateSlice(pointer, nitos_record) + if type == "slice": + if 'name' in new_sfa_record: + self.shell.updateSlice({'slice_id': pointer, 'fields': {'slice_name': new_sfa_record['name']}}) elif type == "user": - # SMBAKER: UpdatePerson only allows a limited set of fields to be - # updated. Ideally we should have a more generic way of doing - # this. I copied the field names from UpdatePerson.py... update_fields = {} - all_fields = new_sfa_record - for key in all_fields.keys(): - if key in ['first_name', 'last_name', 'title', 'email', - 'password', 'phone', 'url', 'bio', 'accepted_aup', - 'enabled']: - update_fields[key] = all_fields[key] - # when updating a user, we always get a 'email' field at this point - # this is because 'email' is a native field in the RegUser object... - if 'email' in update_fields and not update_fields['email']: - del update_fields['email'] - self.shell.UpdatePerson(pointer, update_fields) + if 'name' in new_sfa_record: + update_fields['username'] = new_sfa_record['name'] + if 'email' in new_sfa_record: + update_fields['email'] = new_sfa_record['email'] + + self.shell.updateUser({'user_id': pointer, 'fields': update_fields}) if new_key: - # must check this key against the previous one if it exists - persons = self.shell.getUsers([pointer], ['key_ids']) - person = persons[0] - keys = person['key_ids'] - keys = self.shell.GetKeys(person['key_ids']) - - # Delete all stale keys - key_exists = False - for key in keys: - if new_key != key['key']: - self.shell.DeleteKey(key['key_id']) - else: - key_exists = True - if not key_exists: - self.shell.AddPersonKey(pointer, {'key_type': 'ssh', 'key': new_key}) + # needs to be improved + self.shell.addUserKey({'user_id': pointer, 'key': new_key}) elif type == "node": - self.shell.UpdateNode(pointer, new_sfa_record) + self.shell.updateNode({'node_id': pointer, 'fields': new_sfa_record}) return True - """ - pass + ########## def remove (self, sfa_record): - """ + type=sfa_record['type'] pointer=sfa_record['pointer'] if type == 'user': - persons = self.shell.getUsers(pointer) - # only delete this person if he has site ids. if he doesnt, it probably means - # he was just removed from a site, not actually deleted - if persons and persons[0]['site_ids']: - self.shell.DeletePerson(pointer) + self.shell.deleteUser({'user_id': pointer}) elif type == 'slice': - if self.shell.GetSlices(pointer): - self.shell.DeleteSlice(pointer) + self.shell.deleteSlice({'slice_id': pointer}) elif type == 'node': - if self.shell.GetNodes(pointer): - self.shell.DeleteNode(pointer) - elif type == 'authority': - if self.shell.GetSites(pointer): - self.shell.DeleteSite(pointer) + self.shell.deleteNode({'node_id': pointer}) return True - """ - pass + ## # Convert SFA fields to NITOS fields for use when registering or updating - # registry record in the PLC database + # registry record in the NITOS Scheduler database # def sfa_fields_to_nitos_fields(self, type, hrn, sfa_record): @@ -218,136 +214,114 @@ class NitosDriver (Driver): nitos_record = {} if type == "slice": - nitos_record["name"] = hrn_to_nitos_slicename(hrn) + nitos_record["slice_name"] = hrn_to_nitos_slicename(hrn) elif type == "node": - if not "hostname" in nitos_record: - # fetch from sfa_record - if "hostname" not in sfa_record: - raise MissingSfaInfo("hostname") - nitos_record["name"] = sfa_record["hostname"] - elif type == "authority": - nitos_record["name"] = NitosXrn(xrn=hrn,type='authority').nitos_login_base() - if "name" not in sfa_record: - nitos_record["name"] = hrn + if "hostname" not in sfa_record: + raise MissingSfaInfo("hostname") + nitos_record["node_name"] = sfa_record["hostname"] return nitos_record #################### def fill_record_info(self, records): """ - Given a (list of) SFA record, fill in the PLC specific + Given a (list of) SFA record, fill in the NITOS specific and SFA specific fields in the record. """ if not isinstance(records, list): records = [records] - self.fill_record_pl_info(records) + self.fill_record_nitos_info(records) self.fill_record_hrns(records) self.fill_record_sfa_info(records) return records - def fill_record_pl_info(self, records): + def fill_record_nitos_info(self, records): """ - Fill in the planetlab specific fields of a SFA record. This - involves calling the appropriate PLC method to retrieve the + Fill in the nitos specific fields of a SFA record. This + involves calling the appropriate NITOS API method to retrieve the database record for the object. @param record: record to fill in field (in/out param) """ - """ + # get ids by type - node_ids, site_ids, slice_ids = [], [], [] - person_ids, key_ids = [], [] - type_map = {'node': node_ids, 'authority': site_ids, - 'slice': slice_ids, 'user': person_ids} + node_ids, slice_ids = [], [] + user_ids, key_ids = [], [] + type_map = {'node': node_ids, 'slice': slice_ids, 'user': user_ids} for record in records: for type in type_map: if type == record['type']: type_map[type].append(record['pointer']) - # get pl records - nodes, sites, slices, persons, keys = {}, {}, {}, {}, {} + # get nitos records + nodes, slices, users, keys = {}, {}, {}, {} if node_ids: - node_list = self.shell.GetNodes(node_ids) + all_nodes = self.convert_id(self.shell.getNodes({}, [])) + node_list = [node for node in all_nodes if node['node_id'] in node_ids] nodes = list_to_dict(node_list, 'node_id') - if site_ids: - site_list = self.shell.GetSites(site_ids) - sites = list_to_dict(site_list, 'site_id') if slice_ids: - slice_list = self.shell.GetSlices(slice_ids) + all_slices = self.convert_id(self.shell.getSlices({}, [])) + slice_list = [slice for slice in all_slices if slice['slice_id'] in slice_ids] slices = list_to_dict(slice_list, 'slice_id') - if person_ids: - person_list = self.shell.getUsers(person_ids) - persons = list_to_dict(person_list, 'person_id') - for person in persons: - key_ids.extend(persons[person]['key_ids']) + if user_ids: + all_users = self.convert_id(self.shell.getUsers()) + user_list = [user for user in all_users if user['user_id'] in user_ids] + users = list_to_dict(user_list, 'user_id') - pl_records = {'node': nodes, 'authority': sites, - 'slice': slices, 'user': persons} + nitos_records = {'node': nodes, 'slice': slices, 'user': users} - if key_ids: - key_list = self.shell.GetKeys(key_ids) - keys = list_to_dict(key_list, 'key_id') # fill record info for record in records: - # records with pointer==-1 do not have plc info. - # for example, the top level authority records which are - # authorities, but not PL "sites" if record['pointer'] == -1: continue - for type in pl_records: + for type in nitos_records: if record['type'] == type: - if record['pointer'] in pl_records[type]: - record.update(pl_records[type][record['pointer']]) + if record['pointer'] in nitos_records[type]: + record.update(nitos_records[type][record['pointer']]) break # fill in key info if record['type'] == 'user': - if 'key_ids' not in record: - logger.info("user record has no 'key_ids' - need to import from myplc ?") - else: - pubkeys = [keys[key_id]['key'] for key_id in record['key_ids'] if key_id in keys] - record['keys'] = pubkeys + if record['pointer'] in nitos_records['user']: + record['keys'] = nitos_records['user'][record['pointer']]['keys'] return records - """ - pass + + def fill_record_hrns(self, records): """ - convert pl ids to hrns - """ + convert nitos ids to hrns """ + # get ids - slice_ids, person_ids, site_ids, node_ids = [], [], [], [] + slice_ids, user_ids, node_ids = [], [], [] for record in records: - if 'site_id' in record: - site_ids.append(record['site_id']) - if 'site_ids' in record: - site_ids.extend(record['site_ids']) - if 'person_ids' in record: - person_ids.extend(record['person_ids']) + if 'user_ids' in record: + user_ids.extend(record['user_ids']) if 'slice_ids' in record: slice_ids.extend(record['slice_ids']) if 'node_ids' in record: node_ids.extend(record['node_ids']) - # get pl records - slices, persons, sites, nodes = {}, {}, {}, {} - if site_ids: - site_list = self.shell.GetSites(site_ids, ['site_id', 'login_base']) - sites = list_to_dict(site_list, 'site_id') - if person_ids: - person_list = self.shell.getUsers(person_ids, ['person_id', 'email']) - persons = list_to_dict(person_list, 'person_id') - if slice_ids: - slice_list = self.shell.GetSlices(slice_ids, ['slice_id', 'name']) - slices = list_to_dict(slice_list, 'slice_id') + # get nitos records + slices, users, nodes = {}, {}, {} if node_ids: - node_list = self.shell.GetNodes(node_ids, ['node_id', 'hostname']) + all_nodes = self.convert_id(self.shell.getNodes({}, [])) + node_list = [node for node in all_nodes if node['node_id'] in node_ids] nodes = list_to_dict(node_list, 'node_id') + if slice_ids: + all_slices = self.convert_id(self.shell.getSlices({}, [])) + slice_list = [slice for slice in all_slices if slice['slice_id'] in slice_ids] + slices = list_to_dict(slice_list, 'slice_id') + if user_ids: + all_users = self.convert_id(self.shell.getUsers()) + user_list = [user for user in all_users if user['user_id'] in user_ids] + users = list_to_dict(user_list, 'user_id') + # convert ids to hrns for record in records: @@ -355,22 +329,16 @@ class NitosDriver (Driver): type = record['type'] pointer = record['pointer'] auth_hrn = self.hrn - login_base = '' + testbed_name = self.testbedInfo['name'] if pointer == -1: continue - - if 'site_id' in record: - site = sites[record['site_id']] - login_base = site['login_base'] - record['site'] = ".".join([auth_hrn, login_base]) - if 'person_ids' in record: - emails = [persons[person_id]['email'] for person_id in record['person_ids'] \ - if person_id in persons] - usernames = [email.split('@')[0] for email in emails] - person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames] - record['persons'] = person_hrns + if 'user_ids' in record: + usernames = [users[user_id]['username'] for user_id in record['user_ids'] \ + if user_id in users] + user_hrns = [".".join([auth_hrn, testbed_name, username]) for username in usernames] + record['users'] = user_hrns if 'slice_ids' in record: - slicenames = [slices[slice_id]['name'] for slice_id in record['slice_ids'] \ + slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids'] \ if slice_id in slices] slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames] record['slices'] = slice_hrns @@ -379,11 +347,6 @@ class NitosDriver (Driver): if node_id in nodes] node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames] record['nodes'] = node_hrns - if 'site_ids' in record: - login_bases = [sites[site_id]['login_base'] for site_id in record['site_ids'] \ - if site_id in sites] - site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases] - record['sites'] = site_hrns if 'expires' in record: date = utcparse(record['expires']) @@ -391,106 +354,50 @@ class NitosDriver (Driver): record['expires'] = datestring return records - """ - pass def fill_record_sfa_info(self, records): - """ + def startswith(prefix, values): return [value for value in values if value.startswith(prefix)] - # get person ids - person_ids = [] - site_ids = [] + # get user ids + user_ids = [] for record in records: - person_ids.extend(record.get("person_ids", [])) - site_ids.extend(record.get("site_ids", [])) - if 'site_id' in record: - site_ids.append(record['site_id']) - - # get all pis from the sites we've encountered - # and store them in a dictionary keyed on site_id - site_pis = {} - if site_ids: - pi_filter = {'|roles': ['pi'], '|site_ids': site_ids} - pi_list = self.shell.getUsers(pi_filter, ['person_id', 'site_ids']) - for pi in pi_list: - # we will need the pi's hrns also - person_ids.append(pi['person_id']) - - # we also need to keep track of the sites these pis - # belong to - for site_id in pi['site_ids']: - if site_id in site_pis: - site_pis[site_id].append(pi) - else: - site_pis[site_id] = [pi] - - # get sfa records for all records associated with these records. - # we'll replace pl ids (person_ids) with hrns from the sfa records - # we obtain + user_ids.extend(record.get("user_ids", [])) # get the registry records - person_list, persons = [], {} - person_list = dbsession.query (RegRecord).filter(RegRecord.pointer.in_(person_ids)) + user_list, users = [], {} + user_list = dbsession.query(RegRecord).filter(RegRecord.pointer.in_(user_ids)).all() # create a hrns keyed on the sfa record's pointer. # Its possible for multiple records to have the same pointer so # the dict's value will be a list of hrns. - persons = defaultdict(list) - for person in person_list: - persons[person.pointer].append(person) + users = defaultdict(list) + for user in user_list: + users[user.pointer].append(user) + + # get the nitos records + nitos_user_list, nitos_users = [], {} + nitos_all_users = self.convert_id(self.shell.getUsers()) + nitos_user_list = [user for user in nitos_all_users if user['user_id'] in user_ids] + nitos_users = list_to_dict(nitos_user_list, 'user_id') - # get the pl records - pl_person_list, pl_persons = [], {} - pl_person_list = self.shell.getUsers(person_ids, ['person_id', 'roles']) - pl_persons = list_to_dict(pl_person_list, 'person_id') # fill sfa info for record in records: - # skip records with no pl info (top level authorities) - #if record['pointer'] == -1: - # continue + if record['pointer'] == -1: + continue + sfa_info = {} type = record['type'] logger.info("fill_record_sfa_info - incoming record typed %s"%type) if (type == "slice"): # all slice users are researchers record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice') - record['PI'] = [] record['researcher'] = [] - for person_id in record.get('person_ids', []): - hrns = [person.hrn for person in persons[person_id]] + for user_id in record.get('user_ids', []): + hrns = [user.hrn for user in users[user_id]] record['researcher'].extend(hrns) - - # pis at the slice's site - if 'site_id' in record and record['site_id'] in site_pis: - pl_pis = site_pis[record['site_id']] - pi_ids = [pi['person_id'] for pi in pl_pis] - for person_id in pi_ids: - hrns = [person.hrn for person in persons[person_id]] - record['PI'].extend(hrns) - record['geni_creator'] = record['PI'] - elif (type.startswith("authority")): - record['url'] = None - logger.info("fill_record_sfa_info - authority xherex") - if record['pointer'] != -1: - record['PI'] = [] - record['operator'] = [] - record['owner'] = [] - for pointer in record.get('person_ids', []): - if pointer not in persons or pointer not in pl_persons: - # this means there is not sfa or pl record for this user - continue - hrns = [person.hrn for person in persons[pointer]] - roles = pl_persons[pointer]['roles'] - if 'pi' in roles: - record['PI'].extend(hrns) - if 'tech' in roles: - record['operator'].extend(hrns) - if 'admin' in roles: - record['owner'].extend(hrns) - # xxx TODO: OrganizationName elif (type == "node"): sfa_info['dns'] = record.get("hostname", "") # xxx TODO: URI, LatLong, IP, DNS @@ -502,38 +409,26 @@ class NitosDriver (Driver): sfa_info['geni_certificate'] = record['gid'] # xxx TODO: PostalAddress, Phone record.update(sfa_info) - """ - pass #################### - # plcapi works by changes, compute what needs to be added/deleted def update_relation (self, subject_type, target_type, relation_name, subject_id, target_ids): - """ - # hard-wire the code for slice/user for now, could be smarter if needed + if subject_type =='slice' and target_type == 'user' and relation_name == 'researcher': - subject=self.shell.GetSlices (subject_id)[0] - current_target_ids = subject['person_ids'] + subject=self.shell.getSlices ({'slice_id': subject_id}, [])[0] + current_target_ids = subject['user_ids'] add_target_ids = list ( set (target_ids).difference(current_target_ids)) del_target_ids = list ( set (current_target_ids).difference(target_ids)) logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id))) for target_id in add_target_ids: - self.shell.AddPersonToSlice (target_id,subject_id) + self.shell.addUserToSlice ({'user_id': target_id, 'slice_id': subject_id}) logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id))) for target_id in del_target_ids: logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id))) - self.shell.DeletePersonFromSlice (target_id, subject_id) - elif subject_type == 'authority' and target_type == 'user' and relation_name == 'pi': - # due to the plcapi limitations this means essentially adding pi role to all people in the list - # it's tricky to remove any pi role here, although it might be desirable - persons = self.shell.getUsers (target_ids) - for person in persons: - if 'pi' not in person['roles']: - self.shell.AddRoleToPerson('pi',person['person_id']) + self.shell.deleteUserFromSlice ({'user_id': target_id, 'slice_id': subject_id}) else: logger.info('unexpected relation %s to maintain, %s -> %s'%(relation_name,subject_type,target_type)) - """ - pass + ######################################## ########## aggregate oriented ######################################## @@ -566,10 +461,8 @@ class NitosDriver (Driver): # get data from db slices = self.shell.getSlices({}, []) - # get site name - #site_name = self.shell.getTestbedInfo()['site_name'] - site_name = "nitos" - slice_hrns = [slicename_to_hrn(self.hrn, site_name, slice['slice_name']) for slice in slices] + testbed_name = self.testbedInfo['name'] + slice_hrns = [slicename_to_hrn(self.hrn, testbed_name, slice['slice_name']) for slice in slices] slice_urns = [hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns] # cache the result @@ -637,66 +530,57 @@ class NitosDriver (Driver): raise SliverDoesNotExist("%s (used %s as slicename internally)" % (slice_hrn, slicename)) # report about the reserved nodes only - reserved_nodes = self.shell.getReservedNodes() - nodes = self.shell.getNodes() + reserved_nodes = self.shell.getReservedNodes({}, []) + nodes = self.shell.getNodes({}, []) - user_reserved_nodes = [] + slice_reserved_nodes = [] for r_node in reserved_nodes: if r_node['slice_id'] == slice['slice_id']: for node in nodes: - if node['id'] == r_node['node_id']: - user_reserved_nodes.append(node) + if node['node_id'] == r_node['node_id']: + slice_reserved_nodes.append(node) - if len(user_reserved_nodes) == 0: + if len(slice_reserved_nodes) == 0: raise SliverDoesNotExist("You have not allocated any slivers here") ##### continue from here # get login info user = {} - if slice['person_ids']: - persons = self.shell.GetPersons(slice['person_ids'], ['key_ids']) - key_ids = [key_id for person in persons for key_id in person['key_ids']] - person_keys = self.shell.GetKeys(key_ids) - keys = [key['key'] for key in person_keys] + keys = [] + if slice['user_id']: + users = self.shell.getUsers() + # filter users on slice['user_ids'] + for usr in users: + if usr['user_id'] in slice['user_id']: + keys.extend(usr['keys']) + user.update({'urn': slice_urn, - 'login': slice['name'], + 'login': slice['slice_name'], 'protocol': ['ssh'], 'port': ['22'], 'keys': keys}) - site_ids = [node['site_id'] for node in nodes] result = {} top_level_status = 'unknown' - if nodes: + if slice_reserved_nodes: top_level_status = 'ready' result['geni_urn'] = slice_urn - result['pl_login'] = slice['name'] - result['pl_expires'] = datetime_to_string(utcparse(slice['expires'])) - result['geni_expires'] = datetime_to_string(utcparse(slice['expires'])) + result['nitos_gateway_login'] = slice['slice_name'] + #result['pl_expires'] = datetime_to_string(utcparse(slice['expires'])) + #result['geni_expires'] = datetime_to_string(utcparse(slice['expires'])) resources = [] - for node in nodes: + for node in slice_reserved_nodes: res = {} - res['pl_hostname'] = node['hostname'] - res['pl_boot_state'] = node['boot_state'] - res['pl_last_contact'] = node['last_contact'] - res['geni_expires'] = datetime_to_string(utcparse(slice['expires'])) - if node['last_contact'] is not None: - - res['pl_last_contact'] = datetime_to_string(utcparse(node['last_contact'])) - sliver_id = Xrn(slice_urn, type='slice', id=node['node_id'], authority=self.hrn).urn + res['nitos_hostname'] = node['hostname'] + sliver_id = Xrn(slice_urn, type='slice', id=node['node_id']).urn res['geni_urn'] = sliver_id - if node['boot_state'] == 'boot': - res['geni_status'] = 'ready' - else: - res['geni_status'] = 'failed' - top_level_status = 'failed' - + res['geni_status'] = 'ready' res['geni_error'] = '' res['users'] = [user] @@ -711,74 +595,54 @@ class NitosDriver (Driver): aggregate = NitosAggregate(self) slices = NitosSlices(self) - peer = slices.get_peer(slice_hrn) sfa_peer = slices.get_sfa_peer(slice_hrn) slice_record=None if users: slice_record = users[0].get('slice_record', {}) # parse rspec - rspec = RSpec(rspec_string) - requested_attributes = rspec.version.get_slice_attributes() + rspec = RSpec(rspec_string, version='NITOS 1') - # ensure site record exists - site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer, options=options) # ensure slice record exists - slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer, options=options) - # ensure person records exists - persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer, options=options) - # ensure slice attributes exists - slices.verify_slice_attributes(slice, requested_attributes, options=options) + slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer, options=options) + # ensure user records exists + #users = slices.verify_users(slice_hrn, slice, users, sfa_peer, options=options) # add/remove slice from nodes - requested_slivers = [] - for node in rspec.version.get_nodes_with_slivers(): - hostname = None - if node.get('component_name'): - hostname = node.get('component_name').strip() - elif node.get('component_id'): - hostname = xrn_to_hostname(node.get('component_id').strip()) - if hostname: - requested_slivers.append(hostname) - nodes = slices.verify_slice_nodes(slice, requested_slivers, peer) + #requested_slivers = [] + #for node in rspec.version.get_nodes_with_slivers(): + # hostname = None + # if node.get('component_name'): + # hostname = node.get('component_name').strip() + # elif node.get('component_id'): + # hostname = xrn_to_hostname(node.get('component_id').strip()) + # if hostname: + # requested_slivers.append(hostname) + #nodes = slices.verify_slice_nodes(slice, requested_slivers, peer) - # add/remove links links - slices.verify_slice_links(slice, rspec.version.get_link_requests(), nodes) + # add/remove channels + # add/remove leases - requested_leases = [] - kept_leases = [] - for lease in rspec.version.get_leases(): - requested_lease = {} - if not lease.get('lease_id'): - requested_lease['hostname'] = xrn_to_hostname(lease.get('component_id').strip()) - requested_lease['start_time'] = lease.get('start_time') - requested_lease['duration'] = lease.get('duration') - else: - kept_leases.append(int(lease['lease_id'])) - if requested_lease.get('hostname'): - requested_leases.append(requested_lease) + # a lease in Nitos RSpec case is a reservation of nodes and channels grouped by (slice,timeslot) + rspec_requested_nodes, rspec_requested_channels = rspec.version.get_leases() + print rspec_requested_nodes, rspec_requested_channels + + nodes = slices.verify_slice_leases_nodes(slice, rspec_requested_nodes) + channels = slices.verify_slice_leases_channels(slice, rspec_requested_channels) - leases = slices.verify_slice_leases(slice, requested_leases, kept_leases, peer) - return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version) def delete_sliver (self, slice_urn, slice_hrn, creds, options): slicename = hrn_to_nitos_slicename(slice_hrn) - slices = self.shell.GetSlices({'name': slicename}) + slices = self.shell.getSlices({'slice_name': slicename}) if not slices: return 1 slice = slices[0] - # determine if this is a peer slice - # xxx I wonder if this would not need to use PlSlices.get_peer instead - # in which case plc.peers could be deprecated as this here - # is the only/last call to this last method in plc.peers - peer = peers.get_peer(self, slice_hrn) try: - if peer: - self.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer) - self.shell.DeleteSliceFromNodes(slicename, slice['node_ids']) + pass + #self.shell.DeleteSliceFromNodes({'slice_name': slicename, slice['node_ids']}) finally: if peer: self.shell.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id']) @@ -786,7 +650,7 @@ class NitosDriver (Driver): def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options): slicename = hrn_to_nitos_slicename(slice_hrn) - slices = self.shell.GetSlices({'name': slicename}, ['slice_id']) + slices = self.shell.GetSlices({'slicename': slicename}, ['slice_id']) if not slices: raise RecordNotFound(slice_hrn) slice = slices[0] @@ -794,6 +658,7 @@ class NitosDriver (Driver): record = {'expires': int(datetime_to_epoch(requested_time))} try: self.shell.UpdateSlice(slice['slice_id'], record) + return True except: return False diff --git a/sfa/nitos/nitosshell.py b/sfa/nitos/nitosshell.py index afbc4111..269e3676 100644 --- a/sfa/nitos/nitosshell.py +++ b/sfa/nitos/nitosshell.py @@ -7,13 +7,13 @@ from sfa.util.sfalogging import logger class NitosShell: """ - A simple xmlrpc shell to a myplc instance + A simple xmlrpc shell to a NITOS Scheduler instance This class can receive all NITOS API calls to the underlying testbed For safety this is limited to a set of hard-coded calls """ direct_calls = ['getNodes','getChannels','getSlices','getUsers','getReservedNodes', - 'getReservedChannels','getTestbedInfo' + 'getReservedChannels','getTestbedInfo', 'reserveNodes','reserveChannels','addSlice','addUser','addUserToSlice', 'addUserKey','addNode', 'addChannel', 'updateReservedNodes','updateReservedChannels','updateSlice','updateUser', @@ -26,52 +26,12 @@ class NitosShell: # use the 'capability' auth mechanism for higher performance when the PLC db is local def __init__ ( self, config ) : url = config.SFA_NITOS_URL -# url = "http://195.251.17.239:8080/RPC2" - # try to figure if the url is local - hostname=urlparse(url).hostname - is_local=False - if hostname == 'localhost': is_local=True - # otherwise compare IP addresses; - # this might fail for any number of reasons, so let's harden that - try: - # xxx todo this seems to result in a DNS request for each incoming request to the AM - # should be cached or improved - url_ip=socket.gethostbyname(hostname) - local_ip=socket.gethostbyname(socket.gethostname()) - if url_ip==local_ip: is_local=True - except: - pass - - if is_local: - try: - # too bad this is not installed properly - plcapi_path="/usr/share/plc_api" - if plcapi_path not in sys.path: sys.path.append(plcapi_path) - import PLC.Shell - plc_direct_access=True - except: - plc_direct_access=False - if is_local and plc_direct_access: - logger.debug('plshell access - capability') - #self.plauth = { 'AuthMethod': 'capability', - # 'Username': config.SFA_PLC_USER, - # 'AuthString': config.SFA_PLC_PASSWORD, - # } - self.proxy = PLC.Shell.Shell () - - else: - logger.debug('nitosshell access - xmlrpc') - #self.plauth = { 'AuthMethod': 'password', - # 'Username': config.SFA_PLC_USER, - # 'AuthString': config.SFA_PLC_PASSWORD, - # } - self.proxy = xmlrpclib.Server(url, verbose = False, allow_none = True) + self.proxy = xmlrpclib.Server(url, verbose = False, allow_none = True) def __getattr__(self, name): def func(*args, **kwds): actual_name=None if name in NitosShell.direct_calls: actual_name=name -# if name in NitosShell.alias_calls: actual_name=NitosShell.alias_calls[name] if not actual_name: raise Exception, "Illegal method call %s for NITOS driver"%(name) actual_name = "scheduler.server." + actual_name diff --git a/sfa/nitos/nitosslices.py b/sfa/nitos/nitosslices.py index c7902c8a..93ee2afa 100644 --- a/sfa/nitos/nitosslices.py +++ b/sfa/nitos/nitosslices.py @@ -7,37 +7,15 @@ from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn from sfa.rspecs.rspec import RSpec -from sfa.planetlab.vlink import VLink -from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename +from sfa.nitos.nitosxrn import NitosXrn, hrn_to_nitos_slicename, xrn_to_hostname MAXINT = 2L**31-1 class NitosSlices: - rspec_to_slice_tag = {'max_rate':'net_max_rate'} - def __init__(self, driver): self.driver = driver - def get_peer(self, xrn): - hrn, type = urn_to_hrn(xrn) - #Does this slice belong to a local site or a peer NITOS site? - peer = None - - # get this slice's authority (site) - slice_authority = get_authority(hrn) - - # get this site's authority (sfa root authority or sub authority) - site_authority = get_authority(slice_authority).lower() - - # check if we are already peered with this site_authority, if so - peers = self.driver.shell.GetPeers({}, ['peer_id', 'peername', 'shortname', 'hrn_root']) - for peer_record in peers: - names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)] - if site_authority in names: - peer = peer_record - - return peer def get_sfa_peer(self, xrn): hrn, type = urn_to_hrn(xrn) @@ -52,9 +30,114 @@ class NitosSlices: return sfa_peer - def verify_slice_leases(self, slice, requested_leases, kept_leases, peer): + def verify_slice_leases_nodes(self, slice, rspec_requested_nodes): + nodes = self.driver.shell.getNodes({}, []) + + requested_nodes = [] + for node in rspec_requested_nodes: + requested_node = {} + nitos_nodes = [] + nitos_nodes.extend(nodes) + slice_name = hrn_to_nitos_slicename(node['slice_id']) + if slice_name != slice['slice_name']: + continue + hostname = xrn_to_hostname(node['component_id']) + nitos_node = self.driver.filter_nitos_results(nitos_nodes, {'hostname': hostname})[0] + # fill the requested node with nitos ids + requested_node['slice_id'] = slice['slice_id'] + requested_node['node_id'] = nitos_node['node_id'] + requested_node['start_time'] = node['start_time'] + requested_node['end_time'] = str(int(node['duration']) * int(self.driver.testbedInfo['grain']) + int(node['start_time'])) + requested_nodes.append(requested_node) + + # get actual nodes reservation data for the slice + reserved_nodes = self.driver.filter_nitos_results(self.driver.shell.getReservedNodes({}, []), {'slice_id': slice['slice_id']}) + + reserved_nodes_by_id = {} + for node in reserved_nodes: + reserved_nodes_by_id[node['reservation_id']] = {'slice_id': node['slice_id'], \ + 'node_id': node['node_id'], 'start_time': node['start_time'], \ + 'end_time': node['end_time']} + + added_nodes = [] + kept_nodes_id = [] + deleted_nodes_id = [] + for reservation_id in reserved_nodes_by_id: + if reserved_nodes_by_id[reservation_id] not in requested_nodes: + deleted_nodes_id.append(reservation_id) + else: + kept_nodes_id.append(reservation_id) + requested_nodes.remove(reserved_nodes_by_id[reservation_id]) + added_nodes = requested_nodes + + print "NODES: \nAdded: %s \nDeleted: %s\nKept: %s" %(added_nodes,deleted_nodes_id,kept_nodes_id) + + try: + deleted=self.driver.shell.releaseNodes({'reservation_id': deleted_nodes_id}) + for node in added_nodes: + added=self.driver.shell.reserveNodes({'slice_id': slice['slice_id'], 'start_time': node['start_time'], 'end_time': node['end_time'], 'node_id': [node['node_id']]}) + + except: + logger.log_exc('Failed to add/remove slice leases nodes') + + return added_nodes + - leases = self.driver.shell.GetLeases({'name':slice['name']}, ['lease_id']) + def verify_slice_leases_channels(self, slice, rspec_requested_channels): + channels = self.driver.shell.getChannels({}, []) + + requested_channels = [] + for channel in rspec_requested_channels: + requested_channel = {} + nitos_channels = [] + nitos_channels.extend(channels) + slice_name = hrn_to_nitos_slicename(channel['slice_id']) + if slice_name != slice['slice_name']: + continue + channel_num = channel['channel_num'] + nitos_channel = self.driver.filter_nitos_results(nitos_channels, {'channel': channel_num})[0] + # fill the requested channel with nitos ids + requested_channel['slice_id'] = slice['slice_id'] + requested_channel['channel_id'] = nitos_channel['channel_id'] + requested_channel['start_time'] = channel['start_time'] + requested_channel['end_time'] = str(int(channel['duration']) * int(self.driver.testbedInfo['grain']) + int(channel['start_time'])) + requested_channels.append(requested_channel) + + # get actual channel reservation data for the slice + reserved_channels = self.driver.filter_nitos_results(self.driver.shell.getReservedChannels(), {'slice_id': slice['slice_id']}) + + reserved_channels_by_id = {} + for channel in reserved_channels: + reserved_channels_by_id[channel['reservation_id']] = {'slice_id': channel['slice_id'], \ + 'channel_id': channel['channel_id'], 'start_time': channel['start_time'], \ + 'end_time': channel['end_time']} + + added_channels = [] + kept_channels_id = [] + deleted_channels_id = [] + for reservation_id in reserved_channels_by_id: + if reserved_channels_by_id[reservation_id] not in requested_channels: + deleted_channels_id.append(reservation_id) + else: + kept_channels_id.append(reservation_id) + requested_channels.remove(reserved_channels_by_id[reservation_id]) + added_channels = requested_channels + + print "CHANNELS: \nAdded: %s \nDeleted: %s\nKept: %s" %(added_channels,deleted_channels_id,kept_channels_id) + + try: + deleted=self.driver.shell.releaseChannels({'reservation_id': deleted_channels_id}) + for channel in added_channels: + added=self.driver.shell.reserveChannels({'slice_id': slice['slice_id'], 'start_time': channel['start_time'], 'end_time': channel['end_time'], 'channel_id': [channel['channel_id']]}) + + except: + logger.log_exc('Failed to add/remove slice leases channels') + + return added_channels + + def verify_slice_leases(self, slice, requested_leases, kept_leases): + + leases = self.driver.shell.getLeases({'name':slice['name']}, ['lease_id']) grain = self.driver.shell.GetLeaseGranularity() current_leases = [lease['lease_id'] for lease in leases] deleted_leases = list(set(current_leases).difference(kept_leases)) @@ -72,7 +155,7 @@ class NitosSlices: return leases - def verify_slice_nodes(self, slice, requested_slivers, peer): + def verify_slice_nodes(self, slice, requested_slivers): nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids']) current_slivers = [node['hostname'] for node in nodes] @@ -107,163 +190,36 @@ class NitosSlices: return str(key) - def verify_slice_links(self, slice, requested_links, nodes): - # nodes is undefined here - if not requested_links: - return - - # build dict of nodes - nodes_dict = {} - interface_ids = [] - for node in nodes: - nodes_dict[node['node_id']] = node - interface_ids.extend(node['interface_ids']) - # build dict of interfaces - interfaces = self.driver.shell.GetInterfaces(interface_ids) - interfaces_dict = {} - for interface in interfaces: - interfaces_dict[interface['interface_id']] = interface - - slice_tags = [] - - # set egre key - slice_tags.append({'name': 'egre_key', 'value': self.free_egre_key()}) - - # set netns - slice_tags.append({'name': 'netns', 'value': '1'}) - - # set cap_net_admin - # need to update the attribute string? - slice_tags.append({'name': 'capabilities', 'value': 'CAP_NET_ADMIN'}) - - for link in requested_links: - # get the ip address of the first node in the link - ifname1 = Xrn(link['interface1']['component_id']).get_leaf() - (node_raw, device) = ifname1.split(':') - node_id = int(node_raw.replace('node', '')) - node = nodes_dict[node_id] - if1 = interfaces_dict[node['interface_ids'][0]] - ipaddr = if1['ip'] - topo_rspec = VLink.get_topo_rspec(link, ipaddr) - # set topo_rspec tag - slice_tags.append({'name': 'topo_rspec', 'value': str([topo_rspec]), 'node_id': node_id}) - # set vini_topo tag - slice_tags.append({'name': 'vini_topo', 'value': 'manual', 'node_id': node_id}) - #self.driver.shell.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) - - self.verify_slice_attributes(slice, slice_tags, {'append': True}, admin=True) - - def handle_peer(self, site, slice, persons, peer): - if peer: - # bind site - try: - if site: - self.driver.shell.BindObjectToPeer('site', site['site_id'], peer['shortname'], slice['site_id']) - except Exception,e: - self.driver.shell.DeleteSite(site['site_id']) - raise e - - # bind slice - try: - if slice: - self.driver.shell.BindObjectToPeer('slice', slice['slice_id'], peer['shortname'], slice['slice_id']) - except Exception,e: - self.driver.shell.DeleteSlice(slice['slice_id']) - raise e - - # bind persons - for person in persons: - try: - self.driver.shell.BindObjectToPeer('person', - person['person_id'], peer['shortname'], person['peer_person_id']) - - for (key, remote_key_id) in zip(person['keys'], person['key_ids']): - try: - self.driver.shell.BindObjectToPeer( 'key', key['key_id'], peer['shortname'], remote_key_id) - except: - self.driver.shell.DeleteKey(key['key_id']) - logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname'])) - except Exception,e: - self.driver.shell.DeletePerson(person['person_id']) - raise e - - return slice - - def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}): - (slice_hrn, type) = urn_to_hrn(slice_xrn) - site_hrn = get_authority(slice_hrn) - # login base can't be longer than 20 characters - slicename = hrn_to_pl_slicename(slice_hrn) - authority_name = slicename.split('_')[0] - login_base = authority_name[:20] - sites = self.driver.shell.GetSites(login_base) - if not sites: - # create new site record - site = {'name': 'geni.%s' % authority_name, - 'abbreviated_name': authority_name, - 'login_base': login_base, - 'max_slices': 100, - 'max_slivers': 1000, - 'enabled': True, - 'peer_site_id': None} - if peer: - site['peer_site_id'] = slice_record.get('site_id', None) - site['site_id'] = self.driver.shell.AddSite(site) - # exempt federated sites from monitor policies - self.driver.shell.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101") - -# # is this still necessary? -# # add record to the local registry -# if sfa_peer and slice_record: -# peer_dict = {'type': 'authority', 'hrn': site_hrn, \ -# 'peer_authority': sfa_peer, 'pointer': site['site_id']} -# self.registry.register_peer_object(self.credential, peer_dict) - else: - site = sites[0] - if peer: - # unbind from peer so we can modify if necessary. Will bind back later - self.driver.shell.UnBindObjectFromPeer('site', site['site_id'], peer['shortname']) - - return site - - def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, options={}): - slicename = hrn_to_pl_slicename(slice_hrn) - parts = slicename.split("_") - login_base = parts[0] - slices = self.driver.shell.GetSlices([slicename]) + def verify_slice(self, slice_hrn, slice_record, sfa_peer, options={}): + slicename = hrn_to_nitos_slicename(slice_hrn) + slices = self.driver.shell.getSlices({}, []) + slices = self.driver.filter_nitos_results(slices, {'slice_name': slicename}) if not slices: - slice = {'name': slicename, - 'url': slice_record.get('url', slice_hrn), - 'description': slice_record.get('description', slice_hrn)} + slice = {'name': slicename} # add the slice - slice['slice_id'] = self.driver.shell.AddSlice(slice) + slice['slice_id'] = self.driver.shell.addSlice(slice) slice['node_ids'] = [] - slice['person_ids'] = [] - if peer: - slice['peer_slice_id'] = slice_record.get('slice_id', None) - # mark this slice as an sfa peer record -# if sfa_peer: -# peer_dict = {'type': 'slice', 'hrn': slice_hrn, -# 'peer_authority': sfa_peer, 'pointer': slice['slice_id']} -# self.registry.register_peer_object(self.credential, peer_dict) + slice['user_ids'] = [] else: slice = slices[0] - if peer: - slice['peer_slice_id'] = slice_record.get('slice_id', None) - # unbind from peer so we can modify if necessary. Will bind back later - self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) - #Update existing record (e.g. expires field) it with the latest info. - if slice_record.get('expires'): - requested_expires = int(datetime_to_epoch(utcparse(slice_record['expires']))) - if requested_expires and slice['expires'] != requested_expires: - self.driver.shell.UpdateSlice( slice['slice_id'], {'expires' : requested_expires}) return slice #def get_existing_persons(self, users): - def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}): + def verify_users(self, slice_hrn, slice_record, users, sfa_peer, options={}): + + slice_user_ids = slice_record['user_ids'] + all_users = self.driver.shell.getUsers() + # filter slice users + slice_users = [user for user in all_users if user['user_id'] in slice_user_ids] + + slicename = hrn_to_nitos_slicename(slice_hrn) + slices = self.driver.shell.getSlices({}, []) + slices = self.driver.filter_nitos_results(slices, {'slice_name': slicename}) + + slice_user users_by_email = {} users_by_site = defaultdict(list) users_dict = {} @@ -405,7 +361,7 @@ class NitosSlices: return added_persons - def verify_keys(self, persons, users, peer, options={}): + def verify_keys(self, persons, users, options={}): # existing keys key_ids = [] for person in persons: @@ -456,67 +412,4 @@ class NitosSlices: except: pass - def verify_slice_attributes(self, slice, requested_slice_attributes, options={}, admin=False): - append = options.get('append', True) - # get list of attributes users ar able to manage - filter = {'category': '*slice*'} - if not admin: - filter['|roles'] = ['user'] - slice_attributes = self.driver.shell.GetTagTypes(filter) - valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes] - - # get sliver attributes - added_slice_attributes = [] - removed_slice_attributes = [] - ignored_slice_attribute_names = [] - existing_slice_attributes = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']}) - - # get attributes that should be removed - for slice_tag in existing_slice_attributes: - if slice_tag['tagname'] in ignored_slice_attribute_names: - # If a slice already has a admin only role it was probably given to them by an - # admin, so we should ignore it. - ignored_slice_attribute_names.append(slice_tag['tagname']) - else: - # If an existing slice attribute was not found in the request it should - # be removed - attribute_found=False - for requested_attribute in requested_slice_attributes: - if requested_attribute['name'] == slice_tag['tagname'] and \ - requested_attribute['value'] == slice_tag['value']: - attribute_found=True - break - - if not attribute_found and not append: - removed_slice_attributes.append(slice_tag) - - # get attributes that should be added: - for requested_attribute in requested_slice_attributes: - # if the requested attribute wasn't found we should add it - if requested_attribute['name'] in valid_slice_attribute_names: - attribute_found = False - for existing_attribute in existing_slice_attributes: - if requested_attribute['name'] == existing_attribute['tagname'] and \ - requested_attribute['value'] == existing_attribute['value']: - attribute_found=True - break - if not attribute_found: - added_slice_attributes.append(requested_attribute) - - - # remove stale attributes - for attribute in removed_slice_attributes: - try: - self.driver.shell.DeleteSliceTag(attribute['slice_tag_id']) - except Exception, e: - logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ - % (slice['name'], attribute['value'], attribute.get('node_id'), str(e))) - - # add requested_attributes - for attribute in added_slice_attributes: - try: - self.driver.shell.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None)) - except Exception, e: - logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ - % (slice['name'], attribute['value'], attribute.get('node_id'), str(e))) diff --git a/sfa/rspecs/elements/channel.py b/sfa/rspecs/elements/channel.py index eab3bead..6ed328ed 100644 --- a/sfa/rspecs/elements/channel.py +++ b/sfa/rspecs/elements/channel.py @@ -3,7 +3,11 @@ from sfa.rspecs.elements.element import Element class Channel(Element): fields = [ + 'reservation_id', 'channel_num', 'frequency', 'standard', + 'slice_id', + 'start_time', + 'duration', ] diff --git a/sfa/rspecs/elements/versions/nitosv1Lease.py b/sfa/rspecs/elements/versions/nitosv1Lease.py index ec3cd7ed..5d175e6c 100644 --- a/sfa/rspecs/elements/versions/nitosv1Lease.py +++ b/sfa/rspecs/elements/versions/nitosv1Lease.py @@ -15,6 +15,8 @@ from sfa.rspecs.elements.versions.nitosv1Sliver import NITOSv1Sliver from sfa.rspecs.elements.versions.nitosv1PLTag import NITOSv1PLTag from sfa.rspecs.elements.versions.pgv2Services import PGv2Services from sfa.rspecs.elements.lease import Lease +from sfa.rspecs.elements.channel import Channel + from sfa.nitos.nitosxrn import xrn_to_hostname @@ -77,15 +79,26 @@ class NITOSv1Lease: @staticmethod def get_lease_objs(lease_elems): leases = [] + channels = [] for lease_elem in lease_elems: - lease = Lease(lease_elem.attrib, lease_elem) - if lease.get('lease_id'): - lease['lease_id'] = lease_elem.attrib['lease_id'] - lease['component_id'] = lease_elem.attrib['component_id'] - lease['slice_id'] = lease_elem.attrib['slice_id'] - lease['start_time'] = lease_elem.attrib['start_time'] - lease['duration'] = lease_elem.attrib['duration'] + #get nodes + node_elems = lease_elem.xpath('./default:node | ./node') + for node_elem in node_elems: + lease = Lease(lease_elem.attrib, lease_elem) + lease['slice_id'] = lease_elem.attrib['slice_id'] + lease['start_time'] = lease_elem.attrib['start_time'] + lease['duration'] = lease_elem.attrib['duration'] + lease['component_id'] = node_elem.attrib['component_id'] + leases.append(lease) + #get channels + channel_elems = lease_elem.xpath('./default:channel | ./channel') + for channel_elem in channel_elems: + channel = Channel(channel_elem.attrib, channel_elem) + channel['slice_id'] = lease_elem.attrib['slice_id'] + channel['start_time'] = lease_elem.attrib['start_time'] + channel['duration'] = lease_elem.attrib['duration'] + channel['channel_num'] = channel_elem.attrib['channel_num'] + channels.append(channel) - leases.append(lease) - return leases + return (leases, channels)