X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=sfa%2Fplanetlab%2Fplslices.py;h=65e2164a907218564434d561bd196da90dba50ab;hb=f58accad0a467774fa451ef690147d0b93671f44;hp=be540bb6d1a6a70484047e340e62c978e1401277;hpb=30b26f8f644e2c99673d15c96f2722b746f6fbe2;p=sfa.git diff --git a/sfa/planetlab/plslices.py b/sfa/planetlab/plslices.py index be540bb6..65e2164a 100644 --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@ -1,14 +1,16 @@ +import time from types import StringTypes from collections import defaultdict from sfa.util.sfatime import utcparse, datetime_to_epoch from sfa.util.sfalogging import logger from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn - from sfa.rspecs.rspec import RSpec - from sfa.planetlab.vlink import VLink -from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename +from sfa.planetlab.topology import Topology +from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename, xrn_to_hostname, xrn_to_ext_slicename, hrn_to_ext_loginbase, top_auth +from sfa.storage.model import SliverAllocation +from sfa.storage.alchemy import dbsession MAXINT = 2L**31-1 @@ -159,19 +161,59 @@ class PlSlices: return sfa_peer - def verify_slice_leases(self, slice, requested_leases, kept_leases, peer): - - leases = self.driver.shell.GetLeases({'name':slice['name']}, ['lease_id']) + def verify_slice_leases(self, slice, rspec_requested_leases, peer): + + leases = self.driver.shell.GetLeases({'name':slice['name'], 'clip':int(time.time())}, ['lease_id','name', 'hostname', 't_from', 't_until']) grain = self.driver.shell.GetLeaseGranularity() - current_leases = [lease['lease_id'] for lease in leases] - deleted_leases = list(set(current_leases).difference(kept_leases)) + + requested_leases = [] + for lease in rspec_requested_leases: + requested_lease = {} + slice_hrn, _ = urn_to_hrn(lease['slice_id']) + top_auth_hrn = top_auth(slice_hrn) + if top_auth_hrn == self.driver.hrn: + slice_name = hrn_to_pl_slicename(lease['slice_id']) + else: + slice_name = xrn_to_ext_slicename(lease['slice_id']) + if slice_name != slice['name']: + continue + elif Xrn(lease['component_id']).get_authority_urn().split(':')[0] != self.driver.hrn: + continue + + hostname = xrn_to_hostname(lease['component_id']) + # fill the requested node with nitos ids + requested_lease['name'] = slice['name'] + requested_lease['hostname'] = hostname + requested_lease['t_from'] = int(lease['start_time']) + requested_lease['t_until'] = int(lease['duration']) * grain + int(lease['start_time']) + requested_leases.append(requested_lease) + + + + # prepare actual slice leases by lease_id + leases_by_id = {} + for lease in leases: + leases_by_id[lease['lease_id']] = {'name': lease['name'], 'hostname': lease['hostname'], \ + 't_from': lease['t_from'], 't_until': lease['t_until']} + + added_leases = [] + kept_leases_id = [] + deleted_leases_id = [] + for lease_id in leases_by_id: + if leases_by_id[lease_id] not in requested_leases: + deleted_leases_id.append(lease_id) + else: + kept_leases_id.append(lease_id) + requested_leases.remove(leases_by_id[lease_id]) + added_leases = requested_leases + try: if peer: self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) - deleted=self.driver.shell.DeleteLeases(deleted_leases) - for lease in requested_leases: - added=self.driver.shell.AddLeases(lease['hostname'], slice['name'], int(lease['start_time']), int(lease['duration']) * grain + int(lease['start_time'])) + self.driver.shell.DeleteLeases(deleted_leases_id) + for lease in added_leases: + self.driver.shell.AddLeases(lease['hostname'], slice['name'], lease['t_from'], lease['t_until']) except: logger.log_exc('Failed to add/remove slice leases') @@ -179,32 +221,28 @@ class PlSlices: return leases - def verify_slice_nodes(self, slice, slivers, peer): + def verify_slice_nodes(self, slice_urn, slice, rspec_nodes, peer): + + slivers = {} + for node in rspec_nodes: + hostname = node.get('component_name') + client_id = node.get('client_id') + component_id = node.get('component_id').strip() + if hostname: + hostname = hostname.strip() + elif component_id: + hostname = xrn_to_hostname(component_id) + if hostname: + slivers[hostname] = {'client_id': client_id, 'component_id': component_id} nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids']) current_slivers = [node['hostname'] for node in nodes] - requested_slivers = [] - tags = [] - for node in slivers: - hostname = None - if node.get('component_name'): - hostname = node.get('component_name').strip() - elif node.get('component_id'): - hostname = xrn_to_hostname(node.get('component_id').strip()) - if node.get('client_id'): - tags.append({'slicename': slice['name'], - 'tagname': 'client_id', - 'value': node['client_id'], - 'node': hostname}) - if hostname: - requested_slivers.append(hostname) - # remove nodes not in rspec - deleted_nodes = list(set(current_slivers).difference(requested_slivers)) + deleted_nodes = list(set(current_slivers).difference(slivers.keys())) # add nodes from rspec - added_nodes = list(set(requested_slivers).difference(current_slivers)) + added_nodes = list(set(slivers.keys()).difference(current_slivers)) try: if peer: @@ -215,13 +253,21 @@ class PlSlices: except: logger.log_exc('Failed to add/remove slice from nodes') - # add tags - for tag in tags: - try: - self.driver.shell.AddSliceTag(tag['slicename'], tag['tagname'], tag['value'], tag['node']) - except: - logger.log_exc('Failed to add slice tag') - return nodes + slices = self.driver.shell.GetSlices(slice['name'], ['node_ids']) + resulting_nodes = self.driver.shell.GetNodes(slices[0]['node_ids']) + + # update sliver allocations + for node in resulting_nodes: + client_id = slivers[node['hostname']]['client_id'] + component_id = slivers[node['hostname']]['component_id'] + sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id']) + sliver_id = Xrn(sliver_hrn, type='sliver').urn + record = SliverAllocation(sliver_id=sliver_id, client_id=client_id, + component_id=component_id, + slice_urn = slice_urn, + allocation_state='geni_allocated') + record.sync() + return resulting_nodes def free_egre_key(self): used = set() @@ -238,10 +284,15 @@ class PlSlices: return str(key) def verify_slice_links(self, slice, requested_links, nodes): - # nodes is undefined here + if not requested_links: return - + + # exit if links are not supported here + topology = Topology() + if not topology: + return + # build dict of nodes nodes_dict = {} interface_ids = [] @@ -269,17 +320,22 @@ class PlSlices: for link in requested_links: # get the ip address of the first node in the link ifname1 = Xrn(link['interface1']['component_id']).get_leaf() - (node_raw, device) = ifname1.split(':') - node_id = int(node_raw.replace('node', '')) - node = nodes_dict[node_id] - if1 = interfaces_dict[node['interface_ids'][0]] - ipaddr = if1['ip'] - topo_rspec = VLink.get_topo_rspec(link, ipaddr) - # set topo_rspec tag - slice_tags.append({'name': 'topo_rspec', 'value': str([topo_rspec]), 'node_id': node_id}) - # set vini_topo tag - slice_tags.append({'name': 'vini_topo', 'value': 'manual', 'node_id': node_id}) - #self.driver.shell.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) + if ifname1: + ifname_parts = ifname1.split(':') + node_raw = ifname_parts[0] + device = None + if len(ifname_parts) > 1: + device = ifname_parts[1] + node_id = int(node_raw.replace('node', '')) + node = nodes_dict[node_id] + if1 = interfaces_dict[node['interface_ids'][0]] + ipaddr = if1['ip'] + topo_rspec = VLink.get_topo_rspec(link, ipaddr) + # set topo_rspec tag + slice_tags.append({'name': 'topo_rspec', 'value': str([topo_rspec]), 'node_id': node_id}) + # set vini_topo tag + slice_tags.append({'name': 'vini_topo', 'value': 'manual', 'node_id': node_id}) + #self.driver.shell.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) self.verify_slice_attributes(slice, slice_tags, {'append': True}, admin=True) @@ -323,15 +379,20 @@ class PlSlices: def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}): (slice_hrn, type) = urn_to_hrn(slice_xrn) - site_hrn = get_authority(slice_hrn) - # login base can't be longer than 20 characters - slicename = hrn_to_pl_slicename(slice_hrn) - authority_name = slicename.split('_')[0] - login_base = authority_name[:20] + top_auth_hrn = top_auth(slice_hrn) + if top_auth_hrn == self.driver.hrn: + # login base can't be longer than 20 characters + slicename = hrn_to_pl_slicename(slice_hrn) + authority_name = slicename.split('_')[0] + login_base = authority_name[:20] + else: + login_base = hrn_to_ext_loginbase(slice_hrn) + authority_name = login_base + sites = self.driver.shell.GetSites(login_base) if not sites: # create new site record - site = {'name': 'geni.%s' % authority_name, + site = {'name': 'sfa.%s' % authority_name, 'abbreviated_name': authority_name, 'login_base': login_base, 'max_slices': 100, @@ -343,7 +404,7 @@ class PlSlices: site['site_id'] = self.driver.shell.AddSite(site) # exempt federated sites from monitor policies self.driver.shell.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101") - + # # is this still necessary? # # add record to the local registry # if sfa_peer and slice_record: @@ -354,15 +415,23 @@ class PlSlices: site = sites[0] if peer: # unbind from peer so we can modify if necessary. Will bind back later - self.driver.shell.UnBindObjectFromPeer('site', site['site_id'], peer['shortname']) - - return site + self.driver.shell.UnBindObjectFromPeer('site', site['site_id'], peer['shortname']) + + return site + + + def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, expiration, options={}): + top_auth_hrn = top_auth(slice_hrn) + if top_auth_hrn == self.driver.hrn: + slicename = hrn_to_pl_slicename(slice_hrn) + parts = slicename.split("_") + login_base = parts[0] + else: + login_base = hrn_to_ext_loginbase(slice_hrn) + slicename = xrn_to_ext_slicename(slice_hrn) - def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, options={}): - slicename = hrn_to_pl_slicename(slice_hrn) - parts = slicename.split("_") - login_base = parts[0] slices = self.driver.shell.GetSlices([slicename]) + expires = int(datetime_to_epoch(utcparse(expiration))) if not slices: slice = {'name': slicename, 'url': slice_record.get('url', slice_hrn), @@ -371,24 +440,27 @@ class PlSlices: slice['slice_id'] = self.driver.shell.AddSlice(slice) slice['node_ids'] = [] slice['person_ids'] = [] - if peer: - slice['peer_slice_id'] = slice_record.get('slice_id', None) - # mark this slice as an sfa peer record -# if sfa_peer: -# peer_dict = {'type': 'slice', 'hrn': slice_hrn, -# 'peer_authority': sfa_peer, 'pointer': slice['slice_id']} -# self.registry.register_peer_object(self.credential, peer_dict) + # set the slice HRN + self.driver.shell.SetSliceHrn(int(slice['slice_id']), slice_hrn) + + if peer and slice_record: + slice['peer_slice_id'] = slice_record.get('slice_id', None) + # set the expiration + self.driver.shell.UpdateSlice(slice['slice_id'], {'expires': expires}) else: slice = slices[0] - if peer: + # Check slice HRN + if self.driver.shell.GetSliceHrn(slice['slice_id']) != slice_hrn: + self.driver.shell.SetSliceHrn(slice['slice_id'], slice_hrn) + + if peer and slice_record: slice['peer_slice_id'] = slice_record.get('slice_id', None) # unbind from peer so we can modify if necessary. Will bind back later self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) - #Update existing record (e.g. expires field) it with the latest info. - if slice_record.get('expires'): - requested_expires = int(datetime_to_epoch(utcparse(slice_record['expires']))) - if requested_expires and slice['expires'] != requested_expires: - self.driver.shell.UpdateSlice( slice['slice_id'], {'expires' : requested_expires}) + + #Update expiration if necessary + if slice['expires'] != expires: + self.driver.shell.UpdateSlice( slice['slice_id'], {'expires' : expires}) return slice @@ -396,17 +468,23 @@ class PlSlices: def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}): users_by_email = {} users_by_site = defaultdict(list) - users_dict = {} + users_dict = {} for user in users: user['urn'] = user['urn'].lower() hrn, type = urn_to_hrn(user['urn']) username = get_leaf(hrn) - login_base = PlXrn(xrn=user['urn']).pl_login_base() user['username'] = username - user['site'] = login_base + top_auth_hrn = top_auth(hrn) + + if top_auth_hrn == self.driver.hrn: + login_base = PlXrn(xrn=user['urn']).pl_login_base() + else: + login_base = hrn_to_ext_loginbase(hrn) + + user['site'] = login_base if 'email' in user: - user['email'] = user['email'].lower() + user['email'] = user['email'].lower() users_by_email[user['email']] = user users_dict[user['email']] = user else: @@ -420,14 +498,15 @@ class PlSlices: if users_by_site: for login_base in users_by_site: users = users_by_site[login_base] - for user in users: - existing_user_ids_filter.append(user['username']+'@geni.net') - if existing_user_ids_filter: + for user in users: + existing_user_ids_filter.append(user['username']+'@geni.net') + + if existing_user_ids_filter: # get existing users by email - existing_users = self.driver.shell.GetPersons({'email': existing_user_ids_filter}, + existing_users = self.driver.shell.GetPersons({'email': existing_user_ids_filter}, ['person_id', 'key_ids', 'email']) existing_user_ids.extend([user['email'] for user in existing_users]) - + if users_by_site: # get a list of user sites (based on requeste user urns site_list = self.driver.shell.GetSites(users_by_site.keys(), \ @@ -439,7 +518,7 @@ class PlSlices: sites[site['site_id']] = site site_user_ids.extend(site['person_ids']) - existing_site_persons_list = self.driver.shell.GetPersons(site_user_ids, + existing_site_persons_list = self.driver.shell.GetPersons(site_user_ids, ['person_id', 'key_ids', 'email', 'site_ids']) # all requested users are either existing users or new (added) users @@ -460,12 +539,12 @@ class PlSlices: break if user_found: break - + if user_found == False: fake_email = requested_user['username'] + '@geni.net' requested_user['email'] = fake_email users_dict[fake_email] = requested_user - + # requested slice users requested_user_ids = users_dict.keys() # existing slice users @@ -473,7 +552,7 @@ class PlSlices: existing_slice_users = self.driver.shell.GetPersons(existing_slice_users_filter, ['person_id', 'key_ids', 'email']) existing_slice_user_ids = [user['email'] for user in existing_slice_users] - + # users to be added, removed or updated added_user_ids = set(requested_user_ids).difference(existing_user_ids) added_slice_user_ids = set(requested_user_ids).difference(existing_slice_user_ids) @@ -495,29 +574,36 @@ class PlSlices: # add new users for added_user_id in added_user_ids: added_user = users_dict[added_user_id] - hrn, type = urn_to_hrn(added_user['urn']) + hrn, type = urn_to_hrn(added_user['urn']) person = { 'first_name': added_user.get('first_name', hrn), 'last_name': added_user.get('last_name', hrn), 'email': added_user_id, - 'peer_person_id': None, - 'keys': [], - 'key_ids': added_user.get('key_ids', []), + #'peer_person_id': None, + #'keys': [], + #'key_ids': added_user.get('key_ids', []), } person['person_id'] = self.driver.shell.AddPerson(person) + self.driver.shell.AddRoleToPerson('user', int(person['person_id'])) + # check user HRN + if self.driver.shell.GetPersonHrn(int(person['person_id'])) != hrn: + self.driver.shell.SetPersonHrn(int(person['person_id']), hrn) + if peer: person['peer_person_id'] = added_user['person_id'] added_persons.append(person) - + # enable the account self.driver.shell.UpdatePerson(person['person_id'], {'enabled': True}) - + # add person to site self.driver.shell.AddPersonToSite(added_user_id, added_user['site']) for key_string in added_user.get('keys', []): key = {'key':key_string, 'key_type':'ssh'} key['key_id'] = self.driver.shell.AddPersonKey(person['person_id'], key) + if 'keys' not in person: + person['keys'] = [] person['keys'].append(key) # add the registry record @@ -525,7 +611,7 @@ class PlSlices: # peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \ # 'pointer': person['person_id']} # self.registry.register_peer_object(self.credential, peer_dict) - + for added_slice_user_id in added_slice_user_ids.union(added_user_ids): # add person to the slice self.driver.shell.AddPersonToSlice(added_slice_user_id, slice_record['name']) @@ -533,7 +619,7 @@ class PlSlices: # no need to return worry about it getting bound later return added_persons - + def verify_keys(self, persons, users, peer, options={}): # existing keys @@ -543,12 +629,12 @@ class PlSlices: keylist = self.driver.shell.GetKeys(key_ids, ['key_id', 'key']) keydict = {} for key in keylist: - keydict[key['key']] = key['key_id'] + keydict[key['key']] = key['key_id'] existing_keys = keydict.keys() persondict = {} for person in persons: - persondict[person['email']] = person - + persondict[person['email']] = person + # add new keys requested_keys = [] updated_persons = [] @@ -568,14 +654,14 @@ class PlSlices: key_index = user_keys.index(key['key']) remote_key_id = user['key_ids'][key_index] self.driver.shell.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id) - + finally: if peer: self.driver.shell.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id']) - + # remove old keys (only if we are not appending) append = options.get('append', True) - if append == False: + if append == False: removed_keys = set(existing_keys).difference(requested_keys) for existing_key_id in keydict: if keydict[existing_key_id] in removed_keys: @@ -584,7 +670,7 @@ class PlSlices: self.driver.shell.UnBindObjectFromPeer('key', existing_key_id, peer['shortname']) self.driver.shell.DeleteKey(existing_key_id) except: - pass + pass def verify_slice_attributes(self, slice, requested_slice_attributes, options={}, admin=False): append = options.get('append', True) @@ -600,7 +686,7 @@ class PlSlices: removed_slice_attributes = [] ignored_slice_attribute_names = [] existing_slice_attributes = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']}) - + # get attributes that should be removed for slice_tag in existing_slice_attributes: if slice_tag['tagname'] in ignored_slice_attribute_names: @@ -619,7 +705,7 @@ class PlSlices: if not attribute_found and not append: removed_slice_attributes.append(slice_tag) - + # get attributes that should be added: for requested_attribute in requested_slice_attributes: # if the requested attribute wasn't found we should add it