1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
8 rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
11 def __init__(self, driver):
13 Get the reference to the driver here.
17 def get_peer(self, xrn):
19 Finds the authority of a resource based on its xrn.
20 If the authority is Iotlab (local) return None,
21 Otherwise, look up in the DB if Iotlab is federated with this site
22 authority and returns its DB record if it is the case.
24 :param xrn: resource's xrn
30 hrn, hrn_type = urn_to_hrn(xrn)
31 #Does this slice belong to a local site or a peer iotlab site?
34 # get this slice's authority (site)
35 slice_authority = get_authority(hrn)
37 #This slice belongs to the current site
38 if slice_authority == self.driver.iotlab_api.root_auth:
39 site_authority = slice_authority
42 site_authority = get_authority(slice_authority).lower()
43 # get this site's authority (sfa root authority or sub authority)
45 logger.debug("IOTLABSLICES \ get_peer slice_authority %s \
46 site_authority %s hrn %s"
47 % (slice_authority, site_authority, hrn))
49 # check if we are already peered with this site_authority
50 #if so find the peer record
51 peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority)
52 for peer_record in peers:
54 if site_authority == peer_record.hrn:
56 logger.debug(" IOTLABSLICES \tget_peer peer %s " %(peer))
59 def get_sfa_peer(self, xrn):
60 """Returns the authority name for the xrn or None if the local site
63 :param xrn: the xrn of the resource we are looking the authority for.
65 :returns: the resources's authority name.
69 hrn, hrn_type = urn_to_hrn(xrn)
71 # return the authority for this hrn or None if we are the authority
73 slice_authority = get_authority(hrn)
74 site_authority = get_authority(slice_authority)
76 if site_authority != self.driver.hrn:
77 sfa_peer = site_authority
82 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
84 Compare requested leases with the leases already scheduled/
85 running in OAR. If necessary, delete and recreate modified leases,
86 and delete no longer requested ones.
88 :param sfa_slice: sfa slice record
89 :param requested_jobs_dict: dictionary of requested leases
93 :type requested_jobs_dict: dict
95 :returns: leases list of dictionary
100 logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
102 #First get the list of current leases from OAR
103 leases = self.driver.iotlab_api.GetLeases({'name': sfa_slice['hrn']})
104 logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
105 leases %s " % (requested_jobs_dict, leases))
107 current_nodes_reserved_by_start_time = {}
108 requested_nodes_by_start_time = {}
109 leases_by_start_time = {}
110 reschedule_jobs_dict = {}
112 #Create reduced dictionary with key start_time and value
114 #-for the leases already registered by OAR first
115 # then for the new leases requested by the user
117 #Leases already scheduled/running in OAR
119 current_nodes_reserved_by_start_time[lease['t_from']] = \
120 lease['reserved_nodes']
121 leases_by_start_time[lease['t_from']] = lease
123 #First remove job whose duration is too short
124 for job in requested_jobs_dict.values():
125 if job['duration'] < self.driver.iotlab_api.GetLeaseGranularity():
126 del requested_jobs_dict[job['start_time']]
129 for start_time in requested_jobs_dict:
130 requested_nodes_by_start_time[int(start_time)] = \
131 requested_jobs_dict[start_time]['hostname']
132 #Check if there is any difference between the leases already
133 #registered in OAR and the requested jobs.
134 #Difference could be:
135 #-Lease deleted in the requested jobs
136 #-Added/removed nodes
139 logger.debug("IOTLABSLICES verify_slice_leases \
140 requested_nodes_by_start_time %s \
141 "% (requested_nodes_by_start_time))
142 #Find all deleted leases
144 list(set(leases_by_start_time.keys()).\
145 difference(requested_nodes_by_start_time.keys()))
146 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
147 for start_time in start_time_list]
150 #Find added or removed nodes in exisiting leases
151 for start_time in requested_nodes_by_start_time:
152 logger.debug("IOTLABSLICES verify_slice_leases start_time %s \
154 if start_time in current_nodes_reserved_by_start_time:
156 if requested_nodes_by_start_time[start_time] == \
157 current_nodes_reserved_by_start_time[start_time]:
162 set(requested_nodes_by_start_time[start_time])
164 update_node_set.difference(\
165 current_nodes_reserved_by_start_time[start_time])
167 update_node_set.intersection(\
168 current_nodes_reserved_by_start_time[start_time])
171 current_nodes_reserved_by_start_time[start_time])
173 old_nodes_set.difference(\
174 requested_nodes_by_start_time[start_time])
175 logger.debug("IOTLABSLICES verify_slice_leases \
176 shared_nodes %s added_nodes %s removed_nodes %s"\
177 %(shared_nodes, added_nodes,removed_nodes ))
178 #If the lease is modified, delete it before
180 #Add the deleted lease job id in the list
181 #WARNING :rescheduling does not work if there is already
182 # 2 running/scheduled jobs because deleting a job
183 #takes time SA 18/10/2012
184 if added_nodes or removed_nodes:
185 deleted_leases.append(\
186 leases_by_start_time[start_time]['lease_id'])
188 if added_nodes or shared_nodes:
189 reschedule_jobs_dict[str(start_time)] = \
190 requested_jobs_dict[str(start_time)]
195 job = requested_jobs_dict[str(start_time)]
196 logger.debug("IOTLABSLICES \
197 NEWLEASE slice %s job %s"
199 self.driver.iotlab_api.AddLeases(
201 sfa_slice, int(job['start_time']),
202 int(job['duration']))
204 #Deleted leases are the ones with lease id not declared in the Rspec
206 self.driver.iotlab_api.DeleteLeases(deleted_leases,
207 sfa_slice['user']['uid'])
208 logger.debug("IOTLABSLICES \
209 verify_slice_leases slice %s deleted_leases %s"
210 % (sfa_slice, deleted_leases))
212 if reschedule_jobs_dict:
213 for start_time in reschedule_jobs_dict:
214 job = reschedule_jobs_dict[start_time]
215 self.driver.iotlab_api.AddLeases(
217 sfa_slice, int(job['start_time']),
218 int(job['duration']))
221 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
222 """Check for wanted and unwanted nodes in the slice.
224 Removes nodes and associated leases that the user does not want anymore
225 by deleteing the associated job in OAR (DeleteSliceFromNodes).
226 Returns the nodes' hostnames that are going to be in the slice.
228 :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
230 :param requested_slivers: list of requested nodes' hostnames.
231 :param peer: unused so far.
233 :type sfa_slice: dict
234 :type requested_slivers: list
237 :returns: list requested nodes hostnames
240 .. seealso:: DeleteSliceFromNodes
241 .. todo:: check what to do with the peer? Can not remove peer nodes from
242 slice here. Anyway, in this case, the peer should have gotten the
249 if 'node_ids' in sfa_slice:
250 nodes = self.driver.iotlab_api.GetNodes(
251 sfa_slice['list_node_ids'],
253 current_slivers = [node['hostname'] for node in nodes]
255 # remove nodes not in rspec
256 deleted_nodes = list(set(current_slivers).
257 difference(requested_slivers))
259 logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
260 \r\n \r\n deleted_nodes %s"
261 % (sfa_slice, deleted_nodes))
264 #Delete the entire experience
265 self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice)
268 def verify_slice(self, slice_hrn, slice_record, sfa_peer):
269 """Ensures slice record exists.
271 The slice record must exist either in Iotlab or in the other
272 federated testbed (sfa_peer). If the slice does not belong to Iotlab,
273 check if the user already exists in LDAP. In this case, adds the slice
274 to the sfa DB and associates its LDAP user.
276 :param slice_hrn: slice's name
277 :param slice_record: sfa record of the slice
278 :param sfa_peer: name of the peer authority if any.(not Iotlab).
280 :type slice_hrn: string
281 :type slice_record: dictionary
282 :type sfa_peer: string
284 .. seealso:: AddSlice
289 slicename = slice_hrn
290 # check if slice belongs to Iotlab
291 slices_list = self.driver.iotlab_api.GetSlices(
292 slice_filter=slicename, slice_filter_type='slice_hrn')
297 for sl in slices_list:
299 logger.debug("SLABSLICE \t verify_slice slicename %s \
300 slices_list %s sl %s \r slice_record %s"
301 % (slicename, slices_list, sl, slice_record))
303 sfa_slice.update(slice_record)
306 #Search for user in ldap based on email SA 14/11/12
307 ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\
308 slice_record['user'])
309 logger.debug(" IOTLABSLICES \tverify_slice Oups \
310 slice_record %s sfa_peer %s ldap_user %s"
311 % (slice_record, sfa_peer, ldap_user))
312 #User already registered in ldap, meaning user should be in SFA db
313 #and hrn = sfa_auth+ uid
314 sfa_slice = {'hrn': slicename,
316 'authority': slice_record['authority'],
317 'gid': slice_record['gid'],
318 'slice_id': slice_record['record_id'],
319 'reg-researchers': slice_record['reg-researchers'],
320 'peer_authority': str(sfa_peer)
324 hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid']
325 user = self.driver.get_user_record(hrn)
327 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
330 # add the external slice to the local SFA iotlab DB
332 self.driver.iotlab_api.AddSlice(sfa_slice, user)
334 logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
338 def verify_persons(self, slice_hrn, slice_record, users, options={}):
339 """Ensures the users in users list exist and are enabled in LDAP. Adds
342 Checking that a user exist is based on the user's email. If the user is
343 still not found in the LDAP, it means that the user comes from another
344 federated. In this case an account has to be created in LDAP
345 so as to enable the user to use the testbed, since we trust the testbed
346 he comes from. This is done by calling AddPerson.
348 :param slice_hrn: slice name
349 :param slice_record: record of the slice_hrn
350 :param users: users is a record list. Records can either be
351 local records or users records from known and trusted federated
352 sites.If the user is from another site that iotlab doesn't trust yet,
353 then Resolve will raise an error before getting to create_sliver.
355 :type slice_hrn: string
356 :type slice_record: string
359 .. seealso:: AddPerson
360 .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
364 #TODO SA 21/08/12 verify_persons Needs review
366 logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \
367 \t slice_record %s\r\n users %s \t "
368 % (slice_hrn, slice_record, users))
372 #users_dict : dict whose keys can either be the user's hrn or its id.
373 #Values contains only id and hrn
376 #First create dicts by hrn and id for each user in the user record list:
378 if 'slice_record' in info:
379 slice_rec = info['slice_record']
380 user = slice_rec['user']
383 users_by_email[user['email']] = user
384 users_dict[user['email']] = user
386 logger.debug("SLABSLICE.PY \t verify_person \
387 users_dict %s \r\n user_by_email %s \r\n \
389 % (users_dict, users_by_email, users_by_id))
391 existing_user_ids = []
392 existing_user_emails = []
394 # Check if user is in Iotlab LDAP using its hrn.
395 # Assuming Iotlab is centralised : one LDAP for all sites,
396 # user's record_id unknown from LDAP
397 # LDAP does not provide users id, therefore we rely on email to find the
401 #Construct the list of filters (list of dicts) for GetPersons
402 filter_user = [users_by_email[email] for email in users_by_email]
403 #Check user i in LDAP with GetPersons
404 #Needed because what if the user has been deleted in LDAP but
406 existing_users = self.driver.iotlab_api.GetPersons(filter_user)
407 logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user \
408 %s existing_users %s "
409 % (filter_user, existing_users))
410 #User is in iotlab LDAP
412 for user in existing_users:
413 users_dict[user['email']].update(user)
414 existing_user_emails.append(
415 users_dict[user['email']]['email'])
418 # User from another known trusted federated site. Check
419 # if a iotlab account matching the email has already been created.
422 if isinstance(users, list):
423 req += users[0]['email']
425 req += users['email']
426 ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req)
429 logger.debug(" SLABSLICE.PY \tverify_person users \
430 USER already in Iotlab \t ldap_reslt %s \
432 existing_users.append(ldap_reslt[1])
435 #User not existing in LDAP
436 logger.debug("SLABSLICE.PY \tverify_person users \
437 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
438 ldap_reslt %s " % (users, ldap_reslt))
440 requested_user_emails = users_by_email.keys()
441 requested_user_hrns = \
442 [users_by_email[user]['hrn'] for user in users_by_email]
443 logger.debug("SLABSLICE.PY \tverify_person \
444 users_by_email %s " % (users_by_email))
446 #Check that the user of the slice in the slice record
447 #matches one of the existing users
449 if slice_record['PI'][0] in requested_user_hrns:
450 logger.debug(" SLABSLICE \tverify_person ['PI']\
451 slice_record %s" % (slice_record))
456 # users to be added, removed or updated
457 #One user in one iotlab slice : there should be no need
458 #to remove/ add any user from/to a slice.
459 #However a user from SFA which is not registered in Iotlab yet
460 #should be added to the LDAP.
461 added_user_emails = set(requested_user_emails).\
462 difference(set(existing_user_emails))
465 #self.verify_keys(existing_slice_users, updated_users_list, \
470 #requested_user_email is in existing_user_emails
471 if len(added_user_emails) == 0:
472 slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
473 logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s"
476 for added_user_email in added_user_emails:
477 added_user = users_dict[added_user_email]
478 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person \
479 added_user %s" % (added_user))
481 person['peer_person_id'] = None
482 k_list = ['first_name', 'last_name', 'person_id']
485 person[k] = added_user[k]
487 person['pkey'] = added_user['keys'][0]
488 person['mail'] = added_user['email']
489 person['email'] = added_user['email']
490 person['key_ids'] = added_user.get('key_ids', [])
492 ret = self.driver.iotlab_api.AddPerson(person)
496 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person\
497 person %s" % (person))
498 #Update slice_Record with the id now known to LDAP
499 slice_record['login'] = person['uid']
501 added_persons.append(person)
505 def verify_keys(self, persons, users, peer, options={}):
511 for person in persons:
512 key_ids.extend(person['key_ids'])
513 keylist = self.driver.iotlab_api.GetKeys(key_ids, ['key_id', 'key'])
517 keydict[key['key']] = key['key_id']
518 existing_keys = keydict.keys()
521 for person in persons:
522 persondict[person['email']] = person
527 users_by_key_string = {}
529 user_keys = user.get('keys', [])
530 updated_persons.append(user)
531 for key_string in user_keys:
532 users_by_key_string[key_string] = user
533 requested_keys.append(key_string)
534 if key_string not in existing_keys:
535 key = {'key': key_string, 'key_type': 'ssh'}
538 #person = persondict[user['email']]
539 #self.driver.iotlab_api.UnBindObjectFromPeer('person',
540 #person['person_id'], peer['shortname'])
541 ret = self.driver.iotlab_api.AddPersonKey(\
544 #key_index = user_keys.index(key['key'])
545 #remote_key_id = user['key_ids'][key_index]
546 #self.driver.iotlab_api.BindObjectToPeer('key', \
547 #key['key_id'], peer['shortname'], \
552 #self.driver.iotlab_api.BindObjectToPeer('person', \
553 #person['person_id'], peer['shortname'], \
556 # remove old keys (only if we are not appending)
557 append = options.get('append', True)
559 removed_keys = set(existing_keys).difference(requested_keys)
560 for key in removed_keys:
562 #self.driver.iotlab_api.UnBindObjectFromPeer('key', \
563 #key, peer['shortname'])
565 user = users_by_key_string[key]
566 self.driver.iotlab_api.DeleteKey(user, key)