2 This file defines the IotlabSlices class by which all the slice checkings
3 upon lease creation are done.
5 from sfa.util.xrn import get_authority, urn_to_hrn, hrn_to_urn
6 from sfa.util.sfalogging import logger
13 This class is responsible for checking the slice when creating a
14 lease or a sliver. Those checks include verifying that the user is valid,
15 that the slice is known from the testbed or from our peers, that the list
16 of nodes involved has not changed (in this case the lease is modified
19 rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
21 def __init__(self, driver):
23 Get the reference to the driver here.
27 def get_peer(self, xrn):
29 Finds the authority of a resource based on its xrn.
30 If the authority is Iotlab (local) return None,
31 Otherwise, look up in the DB if Iotlab is federated with this site
32 authority and returns its DB record if it is the case.
34 :param xrn: resource's xrn
40 hrn, hrn_type = urn_to_hrn(xrn)
41 #Does this slice belong to a local site or a peer iotlab site?
44 # get this slice's authority (site)
45 slice_authority = get_authority(hrn)
47 #This slice belongs to the current site
48 if slice_authority == self.driver.testbed_shell.root_auth:
49 site_authority = slice_authority
52 site_authority = get_authority(slice_authority).lower()
53 # get this site's authority (sfa root authority or sub authority)
55 logger.debug("IOTLABSLICES \t get_peer slice_authority %s \
56 site_authority %s hrn %s"
57 % (slice_authority, site_authority, hrn))
59 # check if we are already peered with this site_authority
60 #if so find the peer record
61 peers = self.driver.GetPeers(peer_filter=site_authority)
62 for peer_record in peers:
63 if site_authority == peer_record.hrn:
65 logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer))
68 def get_sfa_peer(self, xrn):
69 """Returns the authority name for the xrn or None if the local site
72 :param xrn: the xrn of the resource we are looking the authority for.
74 :returns: the resources's authority name.
78 hrn, hrn_type = urn_to_hrn(xrn)
80 # return the authority for this hrn or None if we are the authority
82 slice_authority = get_authority(hrn)
83 site_authority = get_authority(slice_authority)
85 if site_authority != self.driver.hrn:
86 sfa_peer = site_authority
90 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
92 Compare requested leases with the leases already scheduled/
93 running in OAR. If necessary, delete and recreate modified leases,
94 and delete no longer requested ones.
96 :param sfa_slice: sfa slice record
97 :param requested_jobs_dict: dictionary of requested leases
98 :param peer: sfa peer record
100 :type sfa_slice: dict
101 :type requested_jobs_dict: dict
103 :returns: leases list of dictionary
108 logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
110 #First get the list of current leases from OAR
111 leases = self.driver.GetLeases({'slice_hrn': sfa_slice['hrn']})
112 logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
113 leases %s " % (requested_jobs_dict, leases))
115 current_nodes_reserved_by_start_time = {}
116 requested_nodes_by_start_time = {}
117 leases_by_start_time = {}
118 reschedule_jobs_dict = {}
120 #Create reduced dictionary with key start_time and value
122 #-for the leases already registered by OAR first
123 # then for the new leases requested by the user
125 #Leases already scheduled/running in OAR
127 current_nodes_reserved_by_start_time[lease['t_from']] = \
128 lease['reserved_nodes']
129 leases_by_start_time[lease['t_from']] = lease
131 #First remove job whose duration is too short
132 for job in requested_jobs_dict.values():
134 str(int(job['duration']) \
135 * self.driver.testbed_shell.GetLeaseGranularity())
136 if job['duration'] < \
137 self.driver.testbed_shell.GetLeaseGranularity():
138 logger.debug("JORDAN removed too short lease !!!! %r" % (requested_jobs_dict[job['start_time']],))
139 del requested_jobs_dict[job['start_time']]
142 for start_time in requested_jobs_dict:
143 requested_nodes_by_start_time[int(start_time)] = \
144 requested_jobs_dict[start_time]['hostname']
145 #Check if there is any difference between the leases already
146 #registered in OAR and the requested jobs.
147 #Difference could be:
148 #-Lease deleted in the requested jobs
149 #-Added/removed nodes
152 logger.debug("IOTLABSLICES verify_slice_leases \
153 requested_nodes_by_start_time %s \
154 "% (requested_nodes_by_start_time))
155 #Find all deleted leases
157 list(set(leases_by_start_time.keys()).\
158 difference(requested_nodes_by_start_time.keys()))
159 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
160 for start_time in start_time_list]
163 #Find added or removed nodes in exisiting leases
164 for start_time in requested_nodes_by_start_time:
165 logger.debug("IOTLABSLICES verify_slice_leases start_time %s \
167 if start_time in current_nodes_reserved_by_start_time:
169 # JORDAN : if we request the same nodes: do nothing
170 if requested_nodes_by_start_time[start_time] == \
171 current_nodes_reserved_by_start_time[start_time]:
176 set(requested_nodes_by_start_time[start_time])
178 update_node_set.difference(\
179 current_nodes_reserved_by_start_time[start_time])
181 update_node_set.intersection(\
182 current_nodes_reserved_by_start_time[start_time])
185 current_nodes_reserved_by_start_time[start_time])
187 old_nodes_set.difference(\
188 requested_nodes_by_start_time[start_time])
189 logger.debug("IOTLABSLICES verify_slice_leases \
190 shared_nodes %s added_nodes %s removed_nodes %s"\
191 %(shared_nodes, added_nodes,removed_nodes ))
192 #If the lease is modified, delete it before
194 #Add the deleted lease job id in the list
195 #WARNING :rescheduling does not work if there is already
196 # 2 running/scheduled jobs because deleting a job
197 #takes time SA 18/10/2012
198 if added_nodes or removed_nodes:
199 deleted_leases.append(\
200 leases_by_start_time[start_time]['lease_id'])
202 if added_nodes or shared_nodes:
203 reschedule_jobs_dict[str(start_time)] = \
204 requested_jobs_dict[str(start_time)]
209 job = requested_jobs_dict[str(start_time)]
210 logger.debug("IOTLABSLICES \
211 NEWLEASE slice %s job %s"
213 job_id = self.driver.AddLeases(
215 sfa_slice, int(job['start_time']),
216 int(job['duration']))
218 #if job_id is not None:
219 # new_leases = self.driver.GetLeases(login=
220 # sfa_slice['login'])
221 # for new_lease in new_leases:
222 # leases.append(new_lease)
224 #Deleted leases are the ones with lease id not declared in the Rspec
226 self.driver.testbed_shell.DeleteLeases(deleted_leases,
228 #self.driver.testbed_shell.DeleteLeases(deleted_leases,
229 # sfa_slice['user']['uid'])
230 logger.debug("IOTLABSLICES \
231 verify_slice_leases slice %s deleted_leases %s"
232 % (sfa_slice, deleted_leases))
234 if reschedule_jobs_dict:
235 logger.debug("JORDAN re-schedule jobs: %r" % (reschedule_jobs_dict,))
236 for start_time in reschedule_jobs_dict:
237 job = reschedule_jobs_dict[start_time]
238 self.driver.AddLeases(
240 sfa_slice, int(job['start_time']),
241 int(job['duration']))
243 # Added by Jordan: until we find a better solution, always update the list of leases
244 return self.driver.GetLeases(login= sfa_slice['login'])
247 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
248 """Check for wanted and unwanted nodes in the slice.
250 Removes nodes and associated leases that the user does not want anymore
251 by deleteing the associated job in OAR (DeleteSliceFromNodes).
252 Returns the nodes' hostnames that are going to be in the slice.
254 :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
256 :param requested_slivers: list of requested nodes' hostnames.
257 :param peer: unused so far.
259 :type sfa_slice: dict
260 :type requested_slivers: list
263 :returns: list requested nodes hostnames
266 .. warning:: UNUSED SQA 24/07/13
267 .. seealso:: DeleteSliceFromNodes
268 .. todo:: check what to do with the peer? Can not remove peer nodes from
269 slice here. Anyway, in this case, the peer should have gotten the
276 if 'node_ids' in sfa_slice:
277 nodes = self.driver.testbed_shell.GetNodes(
278 sfa_slice['list_node_ids'],
280 current_slivers = [node['hostname'] for node in nodes]
282 # remove nodes not in rspec
283 deleted_nodes = list(set(current_slivers).
284 difference(requested_slivers))
286 logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
287 \r\n \r\n deleted_nodes %s"
288 % (sfa_slice, deleted_nodes))
291 #Delete the entire experience
292 self.driver.testbed_shell.DeleteSliceFromNodes(sfa_slice)
295 def verify_slice(self, slice_hrn, slice_record, sfa_peer):
296 """Ensures slice record exists.
298 The slice record must exist either in Iotlab or in the other
299 federated testbed (sfa_peer). If the slice does not belong to Iotlab,
300 check if the user already exists in LDAP. In this case, adds the slice
301 to the sfa DB and associates its LDAP user.
303 :param slice_hrn: slice's name
304 :param slice_record: sfa record of the slice
305 :param sfa_peer: name of the peer authority if any.(not Iotlab).
307 :type slice_hrn: string
308 :type slice_record: dictionary
309 :type sfa_peer: string
311 .. seealso:: AddSlice
316 slicename = slice_hrn
319 # check if slice belongs to Iotlab
320 if slicename.startswith("iotlab"):
321 slices_list = self.driver.GetSlices(slice_filter=slicename,
322 slice_filter_type='slice_hrn')
326 for sl in slices_list:
328 logger.debug("IOTLABSLICES \t verify_slice slicename %s \
329 slices_list %s sl %s \r slice_record %s"
330 % (slicename, slices_list, sl, slice_record))
332 sfa_slice.update(slice_record)
335 #Search for user in ldap based on email SA 14/11/12
336 ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
337 slice_record['user'])
338 logger.debug(" IOTLABSLICES \tverify_slice Oups \
339 slice_record %s sfa_peer %s ldap_user %s"
340 % (slice_record, sfa_peer, ldap_user))
341 #User already registered in ldap, meaning user should be in SFA db
342 #and hrn = sfa_auth+ uid
343 sfa_slice = {'hrn': slicename,
345 'authority': slice_record['authority'],
346 'gid': slice_record['gid'],
347 'slice_id': slice_record['record_id'],
348 'urn': hrn_to_urn(slicename,'slice'),
349 'reg-researchers': slice_record['reg-researchers'],
350 #'peer_authority': str(sfa_peer)
354 hrn = self.driver.testbed_shell.root_auth + '.' \
356 # Registry returns a user record from get_user_record
357 user = self.driver.get_user_record(hrn)
359 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
362 # add the external slice to the local SFA iotlab DB
364 self.driver.AddSlice(sfa_slice, user)
366 logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
368 sfa_slice = {'hrn': slicename,
370 'authority': slice_record['authority'],
371 'gid': slice_record['gid'],
372 'urn': hrn_to_urn(slicename,'slice'),
373 #'slice_id': slice_record['record_id'],
374 'reg-researchers': slice_record['reg-researchers'],
375 #'peer_authority': str(sfa_peer)
380 logger.debug("JORDAN ADDSLICE")
381 logger.debug("ADDSLICE user hrn = %s" % slice_record['user']['hrn'])
382 # XXX LOIC !!! searching in IOTLAB DB because has been added in verify_persons
383 user = self.driver.get_user_record(slice_record['user']['hrn'])
385 logger.debug("LOIC ADDSLICE Search in IOTLAB DB for user = %s" % user)
386 # XXX LOIC !!! not searching in LDAP because this has been done in verify_persons
387 #ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
388 # slice_record['user'])
390 # hrn = self.driver.testbed_shell.root_auth + '.' \
392 # user = self.driver.get_user_record(hrn)
393 # logger.debug(" IOTLAB SLICES JORDAN user: %r %r " % (user, hrn))
395 # logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
398 # add the external slice to the local SFA iotlab DB
400 self.driver.AddSlice(sfa_slice, user)
401 logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
406 def verify_persons(self, slice_hrn, slice_record, users, options=None):
407 """Ensures the users in users list exist and are enabled in LDAP. Adds
408 person if needed (AddPerson).
410 Checking that a user exist is based on the user's email. If the user is
411 still not found in the LDAP, it means that the user comes from another
412 federated testbed. In this case an account has to be created in LDAP
413 so as to enable the user to use the testbed, since we trust the testbed
414 he comes from. This is done by calling AddPerson.
416 :param slice_hrn: slice name
417 :param slice_record: record of the slice_hrn
418 :param users: users is a record list. Records can either be
419 local records or users records from known and trusted federated
420 sites.If the user is from another site that iotlab doesn't trust
421 yet, then Resolve will raise an error before getting to allocate.
423 :type slice_hrn: string
424 :type slice_record: string
427 .. seealso:: AddPerson
428 .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
432 if options is None: options={}
433 logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \
434 \t slice_record %s\r\n users %s \t "
435 % (slice_hrn, slice_record, users))
438 #users_dict : dict whose keys can either be the user's hrn or its id.
439 #Values contains only id and hrn
442 #First create dicts by hrn and id for each user in the user record list:
444 # if 'slice_record' in info:
445 # slice_rec = info['slice_record']
446 # if 'user' in slice_rec :
447 # user = slice_rec['user']
450 users_by_email[info['email']] = info
451 users_dict[info['email']] = info
453 logger.debug("IOTLABSLICES.PY \t verify_person \
454 users_dict %s \r\n user_by_email %s \r\n "
455 % (users_dict, users_by_email))
457 existing_user_ids = []
458 existing_user_emails = []
460 # Check if user is in Iotlab LDAP using its hrn.
461 # Assuming Iotlab is centralised : one LDAP for all sites,
462 # user's record_id unknown from LDAP
463 # LDAP does not provide users id, therefore we rely on email to find the
467 #Construct the list of filters (list of dicts) for GetPersons
468 filter_user = [users_by_email[email] for email in users_by_email]
469 #Check user i in LDAP with GetPersons
470 #Needed because what if the user has been deleted in LDAP but
472 existing_users = self.driver.testbed_shell.GetPersons(filter_user)
473 logger.debug(" \r\n IOTLABSLICES.PY \tverify_person filter_user %s\
475 % (filter_user, existing_users))
476 # User is in iotlab LDAP
477 # XXX LOIC !!! user is existing in iotlab LDAP but coming from OneLab portal
478 if existing_users and slice_hrn.startswith("iotlab"):
479 for user in existing_users:
480 user['login'] = user['uid']
481 users_dict[user['email']].update(user)
482 existing_user_emails.append(
483 users_dict[user['email']]['email'])
485 # User from another known trusted federated site. Check
486 # if a iotlab account matching the email has already been created.
489 if isinstance(users, list):
490 req += users[0]['email']
493 req += users['email']
495 ldap_reslt = self.driver.testbed_shell.ldap.LdapSearch(req)
498 logger.debug(" IOTLABSLICES.PY \tverify_person users \
499 USER already in Iotlab \t ldap_reslt %s \
502 #existing_users.append(ldap_reslt[1])
503 # XXX LOIC !!! Not sure why we use to take the element 1
504 if len(ldap_reslt)>0:
505 ldap_reslt = ldap_reslt[0]
506 logger.debug(ldap_reslt)
507 if len(ldap_reslt)>1:
508 ldap_reslt = ldap_reslt[1]
509 logger.debug("LOIC - iotlabslices.py - ldap_reslt = %s" % (ldap_reslt))
510 existing_users.append(ldap_reslt)
511 existing_user_emails.append(ldap_reslt['mail'][0])
513 # XXX LOIC !!! This login is required
514 # sfa/iotlab/iotlabdriver.py", line 523, in AddLeases
515 if 'uid' in ldap_reslt:
516 # meaning that the Person was found in LDAP
517 slice_record['login'] = ldap_reslt['uid'][0]
519 # XXX LOIC !!! Add the user to IOTLAB DB Registry???
521 # user['pkey'] = user['keys'][0]
523 #ret = self.driver.AddPerson(user,add_to_ldap=False)
524 #logger.debug("LOIC verify_persons AddPerson ret = %s" % ret)
526 #User not existing in LDAP
527 logger.debug("IOTLABSLICES.PY \tverify_person users \
528 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
529 ldap_reslt %s " % (users, ldap_reslt))
530 requested_user_emails = users_by_email.keys()
531 # requested_user_hrns = \
532 # [users_by_email[user]['hrn'] for user in users_by_email]
533 # logger.debug("IOTLABSLICES.PY \tverify_person \
534 # users_by_email %s " % (users_by_email))
536 # #Check that the user of the slice in the slice record
537 # #matches one of the existing users
539 # if slice_record['reg-researchers'][0] in requested_user_hrns:
540 # logger.debug(" IOTLABSLICES \tverify_person ['PI']\
541 # slice_record %s" % (slice_record))
546 # users to be added, removed or updated
547 #One user in one iotlab slice : there should be no need
548 #to remove/ add any user from/to a slice.
549 #However a user from SFA which is not registered in Iotlab yet
550 #should be added to the LDAP.
551 logger.debug("LOIC - iotlabslice.py - requested_user_emails = %r" % requested_user_emails)
552 logger.debug("LOIC - iotlabslice.py - existing_user_emails = %r" % existing_user_emails)
554 added_user_emails = set(requested_user_emails).\
555 difference(set(existing_user_emails))
556 logger.debug("LOIC - iotlabslice.py - added_user_emails = %r" % added_user_emails)
557 logger.debug("LOIC - iotlabslice.py - existing_user_emails = %r" % existing_user_emails)
558 #self.verify_keys(existing_slice_users, updated_users_list, \
561 # XXX JORDAN the uid of the user is put in slice_record['login']
565 #requested_user_email is in existing_user_emails
566 if len(added_user_emails) == 0:
567 # slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
568 logger.debug(" IOTLABSLICES \tverify_person QUICK DIRTY %s"
571 # XXX JORDAN uid == 'register'
572 # new_hrn = slice_record['user']['hrn']
573 # new_user = self.driver.get_user_record(new_hrn)
575 # # XXX HERE WE SHOULD CREATE A SFA USER !!!!!!
576 # added_user = users_dict[requested_user_emails[0]]
578 # person['peer_person_id'] = None
579 # k_list = ['first_name', 'last_name', 'person_id']
581 # if k in added_user:
582 # person[k] = added_user[k]
583 # # bug user without key
584 # if added_user['keys']:
585 # person['pkey'] = added_user['keys'][0]
586 # person['mail'] = added_user['email']
587 # person['email'] = added_user['email']
588 # person['key_ids'] = added_user.get('key_ids', [])
589 # # LOIC !!! il faudrait transformer onelab.upmc.XXX en iotlab.XXX
590 # if new_hrn.startswith("iotlab"):
591 # person['hrn'] = new_hrn
593 # hrn_hierarchy = new_hrn.split(".")
594 # person['hrn'] = "iotlab." + hrn_hierarchy[-1]
596 # ret = self.driver.AddPerson(person, add_to_ldap=False)
597 # logger.debug("AddPerson return = %r type = %s" % (ret,type(ret)))
598 # # LOIC !!! XXX Dans un cas ça retourne un dict du LDAP dans l'autre cas pas de LDAP donc ça ne retourne rien
600 # person['uid'] = slice_record['login']
603 # # meaning bool is True and the AddPerson was successful
604 # person['uid'] = ret['uid']
605 # slice_record['login'] = person['uid']
607 # XXX JORDAN i have no added_user_emails
608 for added_user_email in added_user_emails:
609 added_user = users_dict[added_user_email]
610 logger.debug(" IOTLABSLICES \r\n \r\n \t verify_person \
611 added_user %s" % (added_user))
613 person['peer_person_id'] = None
614 k_list = ['first_name', 'last_name', 'person_id']
617 person[k] = added_user[k]
618 # bug user without key
619 if added_user['keys']:
620 person['pkey'] = added_user['keys'][0]
621 person['mail'] = added_user['email']
622 person['email'] = added_user['email']
623 person['key_ids'] = added_user.get('key_ids', [])
625 ret = self.driver.AddPerson(person)
627 # meaning bool is True and the AddPerson was successful
628 person['uid'] = ret['uid']
629 slice_record['login'] = person['uid']
631 # error message in ret
632 logger.debug(" IOTLABSLICES ret message %s" %(ret))
634 logger.debug(" IOTLABSLICES \r\n \r\n \t THE SECOND verify_person\
635 person %s" % (person))
636 #Update slice_Record with the id now known to LDAP
639 added_persons.append(person)
643 def verify_keys(self, persons, users, peer, options=None):
647 if options is None: options={}
650 for person in persons:
651 key_ids.extend(person['key_ids'])
652 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
656 keydict[key['key']] = key['key_id']
657 existing_keys = keydict.keys()
660 for person in persons:
661 persondict[person['email']] = person
666 users_by_key_string = {}
668 user_keys = user.get('keys', [])
669 updated_persons.append(user)
670 for key_string in user_keys:
671 users_by_key_string[key_string] = user
672 requested_keys.append(key_string)
673 if key_string not in existing_keys:
674 key = {'key': key_string, 'key_type': 'ssh'}
677 #person = persondict[user['email']]
678 #self.driver.testbed_shell.UnBindObjectFromPeer(
679 # 'person',person['person_id'],
681 ret = self.driver.testbed_shell.AddPersonKey(
684 #key_index = user_keys.index(key['key'])
685 #remote_key_id = user['key_ids'][key_index]
686 #self.driver.testbed_shell.BindObjectToPeer('key', \
687 #key['key_id'], peer['shortname'], \
690 # remove old keys (only if we are not appending)
691 append = options.get('append', True)
693 removed_keys = set(existing_keys).difference(requested_keys)
694 for key in removed_keys:
696 #self.driver.testbed_shell.UnBindObjectFromPeer('key', \
697 #key, peer['shortname'])
699 user = users_by_key_string[key]
700 self.driver.testbed_shell.DeleteKey(user, key)