X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=sfa%2Fiotlab%2Fiotlabslices.py;h=f44575c9c3684753672c0ef4b22586467c3e3ec1;hb=a18ad5b1dbc2a1dd346784105c89a0714086ebc4;hp=e449b213671de336f5dcd5e6082dbc9377ded4a0;hpb=efa9ed860628dba98c63b7722f472d922fbe8a87;p=sfa.git diff --git a/sfa/iotlab/iotlabslices.py b/sfa/iotlab/iotlabslices.py index e449b213..f44575c9 100644 --- a/sfa/iotlab/iotlabslices.py +++ b/sfa/iotlab/iotlabslices.py @@ -1,13 +1,22 @@ +""" +This file defines the IotlabSlices class by which all the slice checkings +upon lease creation are done. +""" from sfa.util.xrn import get_authority, urn_to_hrn from sfa.util.sfalogging import logger +MAXINT = 2L**31-1 -MAXINT = 2L**31-1 class IotlabSlices: - - rspec_to_slice_tag = {'max_rate':'net_max_rate'} - + """ + This class is responsible for checking the slice when creating a + lease or a sliver. Those checks include verifying that the user is valid, + that the slice is known from the testbed or from our peers, that the list + of nodes involved has not changed (in this case the lease is modified + accordingly). + """ + rspec_to_slice_tag = {'max_rate': 'net_max_rate'} def __init__(self, driver): """ @@ -15,13 +24,18 @@ class IotlabSlices: """ self.driver = driver - def get_peer(self, xrn): """ - Find the authority of a resources based on its xrn. + Finds the authority of a resource based on its xrn. If the authority is Iotlab (local) return None, Otherwise, look up in the DB if Iotlab is federated with this site - authority and returns its DB record if it is the case, + authority and returns its DB record if it is the case. + + :param xrn: resource's xrn + :type xrn: string + :returns: peer record + :rtype: dict + """ hrn, hrn_type = urn_to_hrn(xrn) #Does this slice belong to a local site or a peer iotlab site? @@ -31,29 +45,36 @@ class IotlabSlices: slice_authority = get_authority(hrn) #Iotlab stuff #This slice belongs to the current site - if slice_authority == self.driver.iotlab_api.root_auth: + if slice_authority == self.driver.iotlab_api.root_auth: site_authority = slice_authority return None site_authority = get_authority(slice_authority).lower() # get this site's authority (sfa root authority or sub authority) - logger.debug("IOTLABSLICES \ get_peer slice_authority %s \ - site_authority %s hrn %s" %(slice_authority, \ - site_authority, hrn)) - + logger.debug("IOTLABSLICES \t get_peer slice_authority %s \ + site_authority %s hrn %s" + % (slice_authority, site_authority, hrn)) # check if we are already peered with this site_authority #if so find the peer record - peers = self.driver.iotlab_api.GetPeers(peer_filter = site_authority) + peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority) for peer_record in peers: - if site_authority == peer_record.hrn: peer = peer_record - logger.debug(" IOTLABSLICES \tget_peer peer %s " %(peer)) + logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer)) return peer def get_sfa_peer(self, xrn): + """Returns the authority name for the xrn or None if the local site + is the authority. + + :param xrn: the xrn of the resource we are looking the authority for. + :type xrn: string + :returns: the resources's authority name. + :rtype: string + + """ hrn, hrn_type = urn_to_hrn(xrn) # return the authority for this hrn or None if we are the authority @@ -66,7 +87,6 @@ class IotlabSlices: return sfa_peer - def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer): """ Compare requested leases with the leases already scheduled/ @@ -75,36 +95,35 @@ class IotlabSlices: :param sfa_slice: sfa slice record :param requested_jobs_dict: dictionary of requested leases - :param peer: sfa peer + :param peer: sfa peer record :type sfa_slice: dict :type requested_jobs_dict: dict - :type peer: - :return: leases list of dictionary + :type peer: dict + :returns: leases list of dictionary :rtype: list """ - logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s \ - "%( sfa_slice)) + logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s " + % (sfa_slice)) #First get the list of current leases from OAR - leases = self.driver.iotlab_api.GetLeases({'name':sfa_slice['hrn']}) + leases = self.driver.iotlab_api.GetLeases({'name': sfa_slice['hrn']}) logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \ - leases %s "%(requested_jobs_dict, leases )) + leases %s " % (requested_jobs_dict, leases)) current_nodes_reserved_by_start_time = {} requested_nodes_by_start_time = {} leases_by_start_time = {} reschedule_jobs_dict = {} - #Create reduced dictionary with key start_time and value # the list of nodes #-for the leases already registered by OAR first # then for the new leases requested by the user #Leases already scheduled/running in OAR - for lease in leases : + for lease in leases: current_nodes_reserved_by_start_time[lease['t_from']] = \ lease['reserved_nodes'] leases_by_start_time[lease['t_from']] = lease @@ -116,8 +135,8 @@ class IotlabSlices: #Requested jobs for start_time in requested_jobs_dict: - requested_nodes_by_start_time[int(start_time)] = \ - requested_jobs_dict[start_time]['hostname'] + requested_nodes_by_start_time[int(start_time)] = \ + requested_jobs_dict[start_time]['hostname'] #Check if there is any difference between the leases already #registered in OAR and the requested jobs. #Difference could be: @@ -127,7 +146,7 @@ class IotlabSlices: logger.debug("IOTLABSLICES verify_slice_leases \ requested_nodes_by_start_time %s \ - "%(requested_nodes_by_start_time )) + "% (requested_nodes_by_start_time)) #Find all deleted leases start_time_list = \ list(set(leases_by_start_time.keys()).\ @@ -136,7 +155,6 @@ class IotlabSlices: for start_time in start_time_list] - #Find added or removed nodes in exisiting leases for start_time in requested_nodes_by_start_time: logger.debug("IOTLABSLICES verify_slice_leases start_time %s \ @@ -184,91 +202,112 @@ class IotlabSlices: job = requested_jobs_dict[str(start_time)] logger.debug("IOTLABSLICES \ - NEWLEASE slice %s job %s"\ - %(sfa_slice, job)) - self.driver.iotlab_api.AddLeases(job['hostname'], \ - sfa_slice, int(job['start_time']), \ - int(job['duration'])) + NEWLEASE slice %s job %s" + % (sfa_slice, job)) + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) #Deleted leases are the ones with lease id not declared in the Rspec if deleted_leases: - self.driver.iotlab_api.DeleteLeases(deleted_leases, sfa_slice['hrn']) + self.driver.iotlab_api.DeleteLeases(deleted_leases, + sfa_slice['user']['uid']) logger.debug("IOTLABSLICES \ - verify_slice_leases slice %s deleted_leases %s"\ - %(sfa_slice, deleted_leases)) - + verify_slice_leases slice %s deleted_leases %s" + % (sfa_slice, deleted_leases)) - if reschedule_jobs_dict : - for start_time in reschedule_jobs_dict: + if reschedule_jobs_dict: + for start_time in reschedule_jobs_dict: job = reschedule_jobs_dict[start_time] - self.driver.iotlab_api.AddLeases(job['hostname'], \ - sfa_slice, int(job['start_time']), \ + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), int(job['duration'])) return leases def verify_slice_nodes(self, sfa_slice, requested_slivers, peer): + """Check for wanted and unwanted nodes in the slice. + + Removes nodes and associated leases that the user does not want anymore + by deleteing the associated job in OAR (DeleteSliceFromNodes). + Returns the nodes' hostnames that are going to be in the slice. + + :param sfa_slice: slice record. Must contain node_ids and list_node_ids. + + :param requested_slivers: list of requested nodes' hostnames. + :param peer: unused so far. + + :type sfa_slice: dict + :type requested_slivers: list + :type peer: string + + :returns: list requested nodes hostnames + :rtype: list + + .. warning:: UNUSED SQA 24/07/13 + .. seealso:: DeleteSliceFromNodes + .. todo:: check what to do with the peer? Can not remove peer nodes from + slice here. Anyway, in this case, the peer should have gotten the + remove request too. + + """ current_slivers = [] deleted_nodes = [] if 'node_ids' in sfa_slice: - nodes = self.driver.iotlab_api.GetNodes(sfa_slice['list_node_ids'], \ + nodes = self.driver.iotlab_api.GetNodes( + sfa_slice['list_node_ids'], ['hostname']) current_slivers = [node['hostname'] for node in nodes] # remove nodes not in rspec - deleted_nodes = list(set(current_slivers).\ - difference(requested_slivers)) - # add nodes from rspec - #added_nodes = list(set(requested_slivers).\ - #difference(current_slivers)) - + deleted_nodes = list(set(current_slivers). + difference(requested_slivers)) logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\ - \r\n \r\n deleted_nodes %s"\ - %(sfa_slice, deleted_nodes)) + \r\n \r\n deleted_nodes %s" + % (sfa_slice, deleted_nodes)) if deleted_nodes: #Delete the entire experience self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice) - #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \ - #deleted_nodes) return nodes + def verify_slice(self, slice_hrn, slice_record, sfa_peer): + """Ensures slice record exists. + The slice record must exist either in Iotlab or in the other + federated testbed (sfa_peer). If the slice does not belong to Iotlab, + check if the user already exists in LDAP. In this case, adds the slice + to the sfa DB and associates its LDAP user. - def free_egre_key(self): - used = set() - for tag in self.driver.iotlab_api.GetSliceTags({'tagname': 'egre_key'}): - used.add(int(tag['value'])) - - for i in range(1, 256): - if i not in used: - key = i - break - else: - raise KeyError("No more EGRE keys available") - - return str(key) - - + :param slice_hrn: slice's name + :param slice_record: sfa record of the slice + :param sfa_peer: name of the peer authority if any.(not Iotlab). + :type slice_hrn: string + :type slice_record: dictionary + :type sfa_peer: string + .. seealso:: AddSlice - def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer): + """ - #login_base = slice_hrn.split(".")[0] slicename = slice_hrn - slices_list = self.driver.iotlab_api.GetSlices(slice_filter = slicename, \ - slice_filter_type = 'slice_hrn') + # check if slice belongs to Iotlab + slices_list = self.driver.iotlab_api.GetSlices( + slice_filter=slicename, slice_filter_type='slice_hrn') + sfa_slice = None + if slices_list: for sl in slices_list: - logger.debug("SLABSLICE \tverify_slice slicename %s \ - slices_list %s sl %s \ slice_record %s"\ - %(slicename, slices_list,sl, \ - slice_record)) + logger.debug("SLABSLICE \t verify_slice slicename %s \ + slices_list %s sl %s \r slice_record %s" + % (slicename, slices_list, sl, slice_record)) sfa_slice = sl sfa_slice.update(slice_record) @@ -277,83 +316,67 @@ class IotlabSlices: ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\ slice_record['user']) logger.debug(" IOTLABSLICES \tverify_slice Oups \ - slice_record %s sfa_peer %s ldap_user %s"\ - %(slice_record, sfa_peer, ldap_user )) + slice_record %s sfa_peer %s ldap_user %s" + % (slice_record, sfa_peer, ldap_user)) #User already registered in ldap, meaning user should be in SFA db #and hrn = sfa_auth+ uid sfa_slice = {'hrn': slicename, - #'url': slice_record.get('url', slice_hrn), - #'description': slice_record.get('description', slice_hrn) - 'node_list' : [], - 'authority' : slice_record['authority'], - 'gid':slice_record['gid'], - #'record_id_user' : user.record_id, - 'slice_id' : slice_record['record_id'], - 'reg-researchers':slice_record['reg-researchers'], - #'record_id_slice': slice_record['record_id'], - 'peer_authority':str(sfa_peer) - - } - if ldap_user : - hrn = self.driver.iotlab_api.root_auth +'.'+ ldap_user['uid'] - + 'node_list': [], + 'authority': slice_record['authority'], + 'gid': slice_record['gid'], + 'slice_id': slice_record['record_id'], + 'reg-researchers': slice_record['reg-researchers'], + 'peer_authority': str(sfa_peer) + } + + if ldap_user: + hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid'] user = self.driver.get_user_record(hrn) - logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" \ - %(hrn, user)) - #sfa_slice = {'slice_hrn': slicename, - ##'url': slice_record.get('url', slice_hrn), - ##'description': slice_record.get('description', slice_hrn) - #'node_list' : [], - #'authority' : slice_record['authority'], - #'gid':slice_record['gid'], - ##'record_id_user' : user.record_id, - #'slice_id' : slice_record['record_id'], - #'reg-researchers':slice_record['reg-researchers'], - ##'record_id_slice': slice_record['record_id'], - #'peer_authority':str(peer.hrn) - - #} - # add the slice - if sfa_slice : - self.driver.iotlab_api.AddSlice(sfa_slice, user) + logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" + % (hrn, user)) - if peer: - sfa_slice['slice_id'] = slice_record['record_id'] + # add the external slice to the local SFA iotlab DB + if sfa_slice: + self.driver.iotlab_api.AddSlice(sfa_slice, user) - #slice['slice_id'] = self.driver.iotlab_api.AddSlice(slice) logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK") - #slice['node_ids']=[] - #slice['person_ids'] = [] - #if peer: - #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None) - # mark this slice as an sfa peer record - #if sfa_peer: - #peer_dict = {'type': 'slice', 'hrn': slice_hrn, - #'peer_authority': sfa_peer, 'pointer': \ - #slice['slice_id']} - #self.registry.register_peer_object(self.credential, peer_dict) + return sfa_slice + def verify_persons(self, slice_hrn, slice_record, users, options={}): + """Ensures the users in users list exist and are enabled in LDAP. Adds + person if needed. - return sfa_slice + Checking that a user exist is based on the user's email. If the user is + still not found in the LDAP, it means that the user comes from another + federated. In this case an account has to be created in LDAP + so as to enable the user to use the testbed, since we trust the testbed + he comes from. This is done by calling AddPerson. + + :param slice_hrn: slice name + :param slice_record: record of the slice_hrn + :param users: users is a record list. Records can either be + local records or users records from known and trusted federated + sites.If the user is from another site that iotlab doesn't trust yet, + then Resolve will raise an error before getting to create_sliver. + + :type slice_hrn: string + :type slice_record: string + :type users: list + + .. seealso:: AddPerson + .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13. - def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \ - options={}): - """ - users is a record list. Records can either be local records - or users records from known and trusted federated sites. - If the user is from another site that iotlab doesn't trust yet, - then Resolve will raise an error before getting to create_sliver. """ #TODO SA 21/08/12 verify_persons Needs review logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \ - \t slice_record %s\r\n users %s \t peer %s "\ - %( slice_hrn, slice_record, users, peer)) + \t slice_record %s\r\n users %s \t " + % (slice_hrn, slice_record, users)) users_by_id = {} - #users_by_hrn = {} + users_by_email = {} #users_dict : dict whose keys can either be the user's hrn or its id. #Values contains only id and hrn @@ -361,8 +384,7 @@ class IotlabSlices: #First create dicts by hrn and id for each user in the user record list: for info in users: - - if 'slice_record' in info : + if 'slice_record' in info: slice_rec = info['slice_record'] user = slice_rec['user'] @@ -370,44 +392,36 @@ class IotlabSlices: users_by_email[user['email']] = user users_dict[user['email']] = user - #if 'hrn' in user: - #users_by_hrn[user['hrn']] = user - #users_dict[user['hrn']] = user - - logger.debug( "SLABSLICE.PY \t verify_person \ + logger.debug("SLABSLICE.PY \t verify_person \ users_dict %s \r\n user_by_email %s \r\n \ - \tusers_by_id %s " \ - %(users_dict,users_by_email, users_by_id)) + \tusers_by_id %s " + % (users_dict, users_by_email, users_by_id)) existing_user_ids = [] - #existing_user_hrns = [] existing_user_emails = [] existing_users = [] # Check if user is in Iotlab LDAP using its hrn. # Assuming Iotlab is centralised : one LDAP for all sites, - # user'as record_id unknown from LDAP - # LDAP does not provide users id, therefore we rely on hrns containing - # the login of the user. - # If the hrn is not a iotlab hrn, the user may not be in LDAP. + # user's record_id unknown from LDAP + # LDAP does not provide users id, therefore we rely on email to find the + # user in LDAP - if users_by_email : + if users_by_email: #Construct the list of filters (list of dicts) for GetPersons - filter_user = [] - for email in users_by_email : - filter_user.append (users_by_email[email]) - #Check user's in LDAP with GetPersons + filter_user = [users_by_email[email] for email in users_by_email] + #Check user i in LDAP with GetPersons #Needed because what if the user has been deleted in LDAP but #is still in SFA? existing_users = self.driver.iotlab_api.GetPersons(filter_user) logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user \ - %s existing_users %s " \ - %(filter_user, existing_users)) - #User's in iotlab LDAP + %s existing_users %s " + % (filter_user, existing_users)) + #User is in iotlab LDAP if existing_users: - for user in existing_users : + for user in existing_users: users_dict[user['email']].update(user) - existing_user_emails.append(\ - users_dict[user['email']]['email']) + existing_user_emails.append( + users_dict[user['email']]['email']) # User from another known trusted federated site. Check @@ -415,52 +429,39 @@ class IotlabSlices: else: req = 'mail=' if isinstance(users, list): - req += users[0]['email'] else: req += users['email'] - ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req) if ldap_reslt: logger.debug(" SLABSLICE.PY \tverify_person users \ USER already in Iotlab \t ldap_reslt %s \ - "%( ldap_reslt)) + " % (ldap_reslt)) existing_users.append(ldap_reslt[1]) else: #User not existing in LDAP - #TODO SA 21/08/12 raise smthg to add user or add it auto ? - #new_record = {} - #new_record['pkey'] = users[0]['keys'][0] - #new_record['mail'] = users[0]['email'] - - logger.debug(" SLABSLICE.PY \tverify_person users \ + logger.debug("SLABSLICE.PY \tverify_person users \ not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \ - ldap_reslt %s " %(users, ldap_reslt)) + ldap_reslt %s " % (users, ldap_reslt)) requested_user_emails = users_by_email.keys() requested_user_hrns = \ - [users_by_email[user]['hrn'] for user in users_by_email] + [users_by_email[user]['hrn'] for user in users_by_email] logger.debug("SLABSLICE.PY \tverify_person \ - users_by_email %s " %( users_by_email)) - #logger.debug("SLABSLICE.PY \tverify_person \ - #user_by_hrn %s " %( users_by_hrn)) - + users_by_email %s " % (users_by_email)) #Check that the user of the slice in the slice record #matches one of the existing users try: if slice_record['PI'][0] in requested_user_hrns: - #if slice_record['record_id_user'] in requested_user_ids and \ - #slice_record['PI'][0] in requested_user_hrns: logger.debug(" SLABSLICE \tverify_person ['PI']\ - slice_record %s" %(slice_record)) + slice_record %s" % (slice_record)) except KeyError: pass - # users to be added, removed or updated #One user in one iotlab slice : there should be no need #to remove/ add any user from/to a slice. @@ -475,23 +476,19 @@ class IotlabSlices: added_persons = [] # add new users - #requested_user_email is in existing_user_emails if len(added_user_emails) == 0: - slice_record['login'] = users_dict[requested_user_emails[0]]['uid'] - logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" \ - %(slice_record)) - + logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" + % (slice_record)) for added_user_email in added_user_emails: - #hrn, type = urn_to_hrn(added_user['urn']) added_user = users_dict[added_user_email] logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person \ - added_user %s" %(added_user)) + added_user %s" % (added_user)) person = {} - person['peer_person_id'] = None - k_list = ['first_name', 'last_name','person_id'] + person['peer_person_id'] = None + k_list = ['first_name', 'last_name', 'person_id'] for k in k_list: if k in added_user: person[k] = added_user[k] @@ -499,26 +496,25 @@ class IotlabSlices: person['pkey'] = added_user['keys'][0] person['mail'] = added_user['email'] person['email'] = added_user['email'] - person['key_ids'] = added_user.get('key_ids', []) - #person['urn'] = added_user['urn'] + person['key_ids'] = added_user.get('key_ids', []) - #person['person_id'] = self.driver.iotlab_api.AddPerson(person) ret = self.driver.iotlab_api.AddPerson(person) - if type(ret) == int : + if type(ret) == int: person['uid'] = ret logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person\ - personne %s" %(person)) + person %s" % (person)) #Update slice_Record with the id now known to LDAP slice_record['login'] = person['uid'] added_persons.append(person) - - return added_persons - #Unused + def verify_keys(self, persons, users, peer, options={}): + """ + .. warning:: unused + """ # existing keys key_ids = [] for person in persons: @@ -549,10 +545,11 @@ class IotlabSlices: #try: ##if peer: #person = persondict[user['email']] - #self.driver.iotlab_api.UnBindObjectFromPeer('person', - #person['person_id'], peer['shortname']) - ret = self.driver.iotlab_api.AddPersonKey(\ - user['email'], key) + #self.driver.iotlab_api.UnBindObjectFromPeer( + # 'person',person['person_id'], + # peer['shortname']) + ret = self.driver.iotlab_api.AddPersonKey( + user['email'], key) #if peer: #key_index = user_keys.index(key['key']) #remote_key_id = user['key_ids'][key_index] @@ -568,7 +565,7 @@ class IotlabSlices: # remove old keys (only if we are not appending) append = options.get('append', True) - if append == False: + if append is False: removed_keys = set(existing_keys).difference(requested_keys) for key in removed_keys: #if peer: