X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=sfa%2Fplanetlab%2Fplslices.py;h=2b59a01723a169b9bbdbc8d5ea5b09de64472a42;hb=30d9951e075d93127c3909dcb41be09b420b3525;hp=ed956874fb280082b51f15e4bbe8e76670b598e3;hpb=feae32b7b770f549392ed4802d48f7c66a09fda5;p=sfa.git diff --git a/sfa/planetlab/plslices.py b/sfa/planetlab/plslices.py index ed956874..2b59a017 100644 --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@ -1,5 +1,4 @@ import time -from types import StringTypes from collections import defaultdict from sfa.util.sfatime import utcparse, datetime_to_epoch @@ -15,7 +14,7 @@ MAXINT = 2L**31-1 class PlSlices: - rspec_to_slice_tag = {'max_rate':'net_max_rate'} + rspec_to_slice_tag = {'max_rate' : 'net_max_rate'} def __init__(self, driver): self.driver = driver @@ -40,7 +39,8 @@ class PlSlices: person_ids = list(person_ids) all_slice_tag_ids = list(all_slice_tag_ids) # Get user information - all_persons_list = self.driver.shell.GetPersons({'person_id':person_ids,'enabled':True}, ['person_id', 'enabled', 'key_ids']) + all_persons_list = self.driver.shell.GetPersons({'person_id':person_ids,'enabled':True}, + ['person_id', 'enabled', 'key_ids']) all_persons = {} for person in all_persons_list: all_persons[person['person_id']] = person @@ -111,7 +111,8 @@ class PlSlices: # XXX Sanity check; though technically this should be a system invariant # checked with an assertion - if slice['expires'] > MAXINT: slice['expires']= MAXINT + if slice['expires'] > MAXINT: + slice['expires'] = MAXINT slivers.append({ 'hrn': hrn, @@ -141,7 +142,8 @@ class PlSlices: def verify_slice_leases(self, slice, rspec_requested_leases): - leases = self.driver.shell.GetLeases({'name':slice['name'], 'clip':int(time.time())}, ['lease_id','name', 'hostname', 't_from', 't_until']) + leases = self.driver.shell.GetLeases({'name':slice['name'], 'clip':int(time.time())}, + ['lease_id','name', 'hostname', 't_from', 't_until']) grain = self.driver.shell.GetLeaseGranularity() requested_leases = [] @@ -240,7 +242,7 @@ class PlSlices: for node in resulting_nodes: client_id = slivers[node['hostname']]['client_id'] component_id = slivers[node['hostname']]['component_id'] - sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id']) + sliver_hrn = '{}.{}-{}'.format(self.driver.hrn, slice['slice_id'], node['node_id']) sliver_id = Xrn(sliver_hrn, type='sliver').urn record = SliverAllocation(sliver_id=sliver_id, client_id=client_id, component_id=component_id, @@ -318,8 +320,7 @@ class PlSlices: slice_tags.append({'name': 'vini_topo', 'value': 'manual', 'node_id': node_id}) #self.driver.shell.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) - self.verify_slice_attributes(slice, slice_tags, {'append': True}, admin=True) - + self.verify_slice_tags(slice, slice_tags, {'pltags':'append'}, admin=True) def verify_site(self, slice_xrn, slice_record=None, sfa_peer=None, options=None): @@ -332,39 +333,32 @@ class PlSlices: login_base = slice_hrn.split('.')[-2][:12] else: login_base = hash_loginbase(site_hrn) - #plxrn = PlXrn(xrn=slice_xrn) - #slice_hrn = plxrn.get_hrn() - #type = plxrn.get_type() - #site_hrn = plxrn.get_authority_hrn() - #authority_name = plxrn.pl_authname() - #slicename = plxrn.pl_slicename() - #login_base = plxrn.pl_login_base() - - sites = self.driver.shell.GetSites({'peer_id': None},['site_id','name','abbreviated_name','login_base','hrn']) # filter sites by hrn - site_exists = [site for site in sites if site['hrn'] == site_hrn] + sites = self.driver.shell.GetSites({'peer_id': None, 'hrn':site_hrn}, + ['site_id','name','abbreviated_name','login_base','hrn']) - if not site_exists: + # alredy exists + if sites: + site = sites[0] + else: # create new site record - site = {'name': 'sfa:%s' % site_hrn, + site = {'name': 'sfa:{}'.format(site_hrn), 'abbreviated_name': site_hrn, 'login_base': login_base, 'max_slices': 100, 'max_slivers': 1000, 'enabled': True, - 'peer_site_id': None} - - site['site_id'] = self.driver.shell.AddSite(site) - # Set site HRN - self.driver.shell.SetSiteHrn(int(site['site_id']), site_hrn) - # Tag this as created through SFA - self.driver.shell.SetSiteSfaCreated(int(site['site_id']), 'True') + 'peer_site_id': None, + 'hrn':site_hrn, + 'sfa_created': 'True', + } + site_id = self.driver.shell.AddSite(site) + # plcapi tends to mess with the incoming hrn so let's make sure + self.driver.shell.SetSiteHrn (site_id, site_hrn) + site['site_id'] = site_id # exempt federated sites from monitor policies - self.driver.shell.AddSiteTag(int(site['site_id']), 'exempt_site_until', "20200101") - - else: - site = site_exists[0] + self.driver.shell.AddSiteTag(site_id, 'exempt_site_until', "20200101") return site @@ -379,19 +373,19 @@ class PlSlices: else: login_base = hash_loginbase(site_hrn) slice_name = '_'.join([login_base, slice_part]) - #plxrn = PlXrn(xrn=slice_hrn) - #slice_hrn = plxrn.get_hrn() - #type = plxrn.get_type() - #site_hrn = plxrn.get_authority_hrn() - #authority_name = plxrn.pl_authname() - #slicename = plxrn.pl_slicename() - #login_base = plxrn.pl_login_base() - - slices = self.driver.shell.GetSlices({'peer_id': None},['slice_id','name','hrn']) - # Filter slices by HRN - slice_exists = [slice for slice in slices if slice['hrn'] == slice_hrn] + expires = int(datetime_to_epoch(utcparse(expiration))) - if not slice_exists: + # Filter slices by HRN + slices = self.driver.shell.GetSlices({'peer_id': None, 'hrn':slice_hrn}, + ['slice_id','name','hrn','expires']) + + if slices: + slice = slices[0] + slice_id = slice['slice_id'] + #Update expiration if necessary + if slice.get('expires', None) != expires: + self.driver.shell.UpdateSlice( slice_id, {'expires' : expires}) + else: if slice_record: url = slice_record.get('url', slice_hrn) description = slice_record.get('description', slice_hrn) @@ -400,23 +394,20 @@ class PlSlices: description = slice_hrn slice = {'name': slice_name, 'url': url, - 'description': description} - # add the slice - slice['slice_id'] = self.driver.shell.AddSlice(slice) - # set the slice HRN - self.driver.shell.SetSliceHrn(int(slice['slice_id']), slice_hrn) - # Tag this as created through SFA - self.driver.shell.SetSliceSfaCreated(int(slice['slice_id']), 'True') + 'description': description, + 'hrn': slice_hrn, + 'sfa_created': 'True', + #'expires': expires, + } + # add the slice + slice_id = self.driver.shell.AddSlice(slice) + # plcapi tends to mess with the incoming hrn so let's make sure + self.driver.shell.SetSliceHrn (slice_id, slice_hrn) + # cannot be set with AddSlice # set the expiration - self.driver.shell.UpdateSlice(int(slice['slice_id']), {'expires': expires}) + self.driver.shell.UpdateSlice(slice_id, {'expires': expires}) - else: - slice = slice_exists[0] - #Update expiration if necessary - if slice.get('expires', None) != expires: - self.driver.shell.UpdateSlice( int(slice['slice_id']), {'expires' : expires}) - - return self.driver.shell.GetSlices(int(slice['slice_id']))[0] + return self.driver.shell.GetSlices(slice_id)[0] # in the following code, we use @@ -425,27 +416,45 @@ class PlSlices: # 'slice_record': it seems like the first of these 'users' also contains a 'slice_record' # key that holds stuff like 'hrn', 'slice_id', 'authority',... # - def create_person (self, user, site_id): + def create_person_from_user (self, user, site_id): user_hrn = user['hrn'] - # the value to use if 'user' has no 'email' attached - xxx should be configurable - default_email = "%s@geni.net"%user_hrn.split('.')[-1] - # PLCAPI requires at least these to be set + # the value to use if 'user' has no 'email' attached - or if the attached email already exists + # typically + ( auth_hrn, _ , leaf ) = user_hrn.rpartition('.') + # somehow this has backslashes, get rid of them + auth_hrn = auth_hrn.replace('\\','') + default_email = "{}@{}.stub".format(leaf, auth_hrn) person_record = { + # required 'first_name': user.get('first_name',user_hrn), 'last_name': user.get('last_name',user_hrn), 'email': user.get('email', default_email), + # our additions + 'enabled': True, + 'sfa_created': 'True', + 'hrn': user_hrn, } - # make it enabled - person_record.update({'enabled': True}) - # mark it sfa_created; a string is required here, sfa_created is a tag - person_record.update({'sfa_created':'True'}) - # set hrn - person_record.update({'hrn':user_hrn}) - - person_id = int (self.driver.shell.AddPerson(person_record)) + + logger.debug ("about to attempt to AddPerson with {}".format(person_record)) + try: + # the thing is, the PLE db has a limitation on re-using the same e-mail + # in the case where people have an account on ple.upmc and then then come + # again from onelab.upmc, they will most likely have the same e-mail, and so kaboom.. + # so we first try with the accurate email + person_id = int (self.driver.shell.AddPerson(person_record)) + except: + logger.log_exc("caught during first attempt at AddPerson") + # and if that fails we start again with the email based on the hrn, which this time is unique.. + person_record['email'] = default_email + logger.debug ("second chance with email={}".format(person_record['email'])) + person_id = int (self.driver.shell.AddPerson(person_record)) self.driver.shell.AddRoleToPerson('user', person_id) - self.driver.shell.AddPersonToSite(person_id, site['site_id']) + self.driver.shell.AddPersonToSite(person_id, site_id) + # plcapi tends to mess with the incoming hrn so let's make sure + self.driver.shell.SetPersonHrn (person_id, user_hrn) + # also 'enabled':True does not seem to pass through with AddPerson + self.driver.shell.UpdatePerson (person_id, {'enabled': True}) return person_id @@ -458,6 +467,8 @@ class PlSlices: # this is for retrieving users from a hrn users_by_hrn = { user['hrn'] : user for user in users } + for user in users: logger.debug("incoming user {}".format(user)) + # compute the hrn's for the authority and site top_auth_hrn = top_auth(slice_hrn) site_hrn = '.'.join(slice_hrn.split('.')[:-1]) @@ -490,7 +501,7 @@ class PlSlices: existing_hrns = [ person['hrn'] for person in target_existing_persons ] tocreate_hrns = set (target_hrns) - set (existing_hrns) # create these - target_created_person_ids = [ self.create_person (users_by_hrn[hrn], site_id) for hrn in tocreate_hrns ] + target_created_person_ids = [ self.create_person_from_user (users_by_hrn[hrn], site_id) for hrn in tocreate_hrns ] # we can partition the persons of interest into one of these 3 classes add_person_ids = set(target_created_person_ids) | set(target_existing_person_ids) - set(slice_person_ids) @@ -503,8 +514,12 @@ class PlSlices: # about the last 2 sets, for managing keys, we need to trace back person_id -> user # and for this we need all the Person objects; we already have the target_existing ones - target_created_persons = self.driver.shell.GetPersons ({'peer_id':None, 'person_id':target_created_person_ids},person_fields) - persons_by_person_id = { person['person_id'] : person for person in target_existing_persons + target_created_persons } + # also we avoid issuing a call if possible + target_created_persons = [] if not target_created_person_ids \ + else self.driver.shell.GetPersons \ + ({'peer_id':None, 'person_id':target_created_person_ids}, person_fields) + persons_by_person_id = { person['person_id'] : person \ + for person in target_existing_persons + target_created_persons } def user_by_person_id (person_id): person = persons_by_person_id [person_id] @@ -540,70 +555,99 @@ class PlSlices: self.driver.shell.AddPersonKey(int(person_id), key) - def verify_slice_attributes(self, slice, requested_slice_attributes, options=None, admin=False): + def verify_slice_tags(self, slice, requested_slice_attributes, options=None, admin=False): + """ + This function deals with slice tags, and supports 3 modes described + in the 'pltags' option that can be either + (*) 'ignore' (default) - do nothing + (*) 'append' - only add incoming tags, that do not match an existing tag + (*) 'sync' - tries to do the plain wholesale thing, + i.e. to leave the db in sync with incoming tags + """ if options is None: options={} - append = options.get('append', True) - # get list of attributes users ar able to manage + + # lookup 'pltags' in options to find out which mode is requested here + pltags = options.get('pltags', 'ignore') + # make sure the default is 'ignore' + if pltags not in ('ignore', 'append', 'sync'): + pltags = 'ignore' + + if pltags == 'ignore': + logger.info('verify_slice_tags in ignore mode - leaving slice tags as-is') + return + + # incoming data (attributes) have a (name, value) pair + # while PLC data (tags) have a (tagname, value) pair + # we must be careful not to mix these up + + # get list of tags users are able to manage - based on category filter = {'category': '*slice*'} if not admin: filter['|roles'] = ['user'] - slice_attributes = self.driver.shell.GetTagTypes(filter) - valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes] + valid_tag_types = self.driver.shell.GetTagTypes(filter) + valid_tag_names = [ tag_type['tagname'] for tag_type in valid_tag_types ] + logger.debug("verify_slice_attributes: valid names={}".format(valid_tag_names)) - # get sliver attributes - added_slice_attributes = [] - removed_slice_attributes = [] + # get slice tags + slice_attributes_to_add = [] + slice_tags_to_remove = [] # we need to keep the slice hrn anyway - ignored_slice_attribute_names = ['hrn'] - existing_slice_attributes = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']}) + ignored_slice_tag_names = ['hrn'] + existing_slice_tags = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']}) - # get attributes that should be removed - for slice_tag in existing_slice_attributes: - if slice_tag['tagname'] in ignored_slice_attribute_names: + # get tags that should be removed + for slice_tag in existing_slice_tags: + if slice_tag['tagname'] in ignored_slice_tag_names: # If a slice already has a admin only role it was probably given to them by an # admin, so we should ignore it. - ignored_slice_attribute_names.append(slice_tag['tagname']) - attribute_found=True + ignored_slice_tag_names.append(slice_tag['tagname']) + tag_found = True else: - # If an existing slice attribute was not found in the request it should + # If an existing slice tag was not found in the request it should # be removed - attribute_found=False + tag_found = False for requested_attribute in requested_slice_attributes: if requested_attribute['name'] == slice_tag['tagname'] and \ requested_attribute['value'] == slice_tag['value']: - attribute_found=True + tag_found = True break + # remove tags only if not in append mode + if not tag_found and pltags != 'append': + slice_tags_to_remove.append(slice_tag) - if not attribute_found and not append: - removed_slice_attributes.append(slice_tag) - - # get attributes that should be added: + # get tags that should be added: for requested_attribute in requested_slice_attributes: # if the requested attribute wasn't found we should add it - if requested_attribute['name'] in valid_slice_attribute_names: - attribute_found = False - for existing_attribute in existing_slice_attributes: + if requested_attribute['name'] in valid_tag_names: + tag_found = False + for existing_attribute in existing_slice_tags: if requested_attribute['name'] == existing_attribute['tagname'] and \ requested_attribute['value'] == existing_attribute['value']: - attribute_found=True + tag_found = True break - if not attribute_found: - added_slice_attributes.append(requested_attribute) - - - # remove stale attributes - for attribute in removed_slice_attributes: + if not tag_found: + slice_attributes_to_add.append(requested_attribute) + + def friendly_message (tag_or_att): + name = tag_or_att['tagname'] if 'tagname' in tag_or_att else tag_or_att['name'] + return "SliceTag slice={}, tagname={} value={}, node_id={}"\ + .format(slice['name'], tag_or_att['name'], tag_or_att['value'], tag_or_att.get('node_id')) + + # remove stale tags + for tag in slice_tags_to_remove: try: - self.driver.shell.DeleteSliceTag(attribute['slice_tag_id']) - except Exception, e: - logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ - % (slice['name'], attribute['value'], attribute.get('node_id'), str(e))) - - # add requested_attributes - for attribute in added_slice_attributes: + logger.info("Removing Slice Tag {}".format(friendly_message(tag))) + self.driver.shell.DeleteSliceTag(tag['slice_tag_id']) + except Exception as e: + logger.warn("Failed to remove slice tag {}\nCause:{}"\ + .format(friendly_message(tag), e)) + + # add requested_tags + for attribute in slice_attributes_to_add: try: - self.driver.shell.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None)) - except Exception, e: - logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ - % (slice['name'], attribute['value'], attribute.get('node_id'), str(e))) - + logger.info("Adding Slice Tag {}".format(friendly_message(attribute))) + self.driver.shell.AddSliceTag(slice['name'], attribute['name'], + attribute['value'], attribute.get('node_id', None)) + except Exception as e: + logger.warn("Failed to add slice tag {}\nCause:{}"\ + .format(friendly_message(attribute), e))