2 This file defines the IotlabSlices class by which all the slice checkings
3 upon lease creation are done.
5 from sfa.util.xrn import get_authority, urn_to_hrn, hrn_to_urn
6 from sfa.util.sfalogging import logger
13 This class is responsible for checking the slice when creating a
14 lease or a sliver. Those checks include verifying that the user is valid,
15 that the slice is known from the testbed or from our peers, that the list
16 of nodes involved has not changed (in this case the lease is modified
19 rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
21 def __init__(self, driver):
23 Get the reference to the driver here.
27 def get_peer(self, xrn):
29 Finds the authority of a resource based on its xrn.
30 If the authority is Iotlab (local) return None,
31 Otherwise, look up in the DB if Iotlab is federated with this site
32 authority and returns its DB record if it is the case.
34 :param xrn: resource's xrn
40 hrn, hrn_type = urn_to_hrn(xrn)
41 #Does this slice belong to a local site or a peer iotlab site?
44 # get this slice's authority (site)
45 slice_authority = get_authority(hrn)
47 #This slice belongs to the current site
48 if slice_authority == self.driver.testbed_shell.root_auth:
49 site_authority = slice_authority
52 site_authority = get_authority(slice_authority).lower()
53 # get this site's authority (sfa root authority or sub authority)
55 logger.debug("IOTLABSLICES \t get_peer slice_authority %s \
56 site_authority %s hrn %s"
57 % (slice_authority, site_authority, hrn))
59 # check if we are already peered with this site_authority
60 #if so find the peer record
61 peers = self.driver.GetPeers(peer_filter=site_authority)
62 for peer_record in peers:
63 if site_authority == peer_record.hrn:
65 logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer))
68 def get_sfa_peer(self, xrn):
69 """Returns the authority name for the xrn or None if the local site
72 :param xrn: the xrn of the resource we are looking the authority for.
74 :returns: the resources's authority name.
78 hrn, hrn_type = urn_to_hrn(xrn)
80 # return the authority for this hrn or None if we are the authority
82 slice_authority = get_authority(hrn)
83 site_authority = get_authority(slice_authority)
85 if site_authority != self.driver.hrn:
86 sfa_peer = site_authority
90 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
92 Compare requested leases with the leases already scheduled/
93 running in OAR. If necessary, delete and recreate modified leases,
94 and delete no longer requested ones.
96 :param sfa_slice: sfa slice record
97 :param requested_jobs_dict: dictionary of requested leases
98 :param peer: sfa peer record
100 :type sfa_slice: dict
101 :type requested_jobs_dict: dict
103 :returns: leases list of dictionary
108 logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
110 #First get the list of current leases from OAR
111 leases = self.driver.GetLeases({'slice_hrn': sfa_slice['hrn']})
112 logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
113 leases %s " % (requested_jobs_dict, leases))
115 current_nodes_reserved_by_start_time = {}
116 requested_nodes_by_start_time = {}
117 leases_by_start_time = {}
118 reschedule_jobs_dict = {}
120 #Create reduced dictionary with key start_time and value
122 #-for the leases already registered by OAR first
123 # then for the new leases requested by the user
125 #Leases already scheduled/running in OAR
127 current_nodes_reserved_by_start_time[lease['t_from']] = \
128 lease['reserved_nodes']
129 leases_by_start_time[lease['t_from']] = lease
131 #First remove job whose duration is too short
132 for job in requested_jobs_dict.values():
134 str(int(job['duration']) \
135 * self.driver.testbed_shell.GetLeaseGranularity())
136 if job['duration'] < \
137 self.driver.testbed_shell.GetLeaseGranularity():
138 del requested_jobs_dict[job['start_time']]
141 for start_time in requested_jobs_dict:
142 requested_nodes_by_start_time[int(start_time)] = \
143 requested_jobs_dict[start_time]['hostname']
144 #Check if there is any difference between the leases already
145 #registered in OAR and the requested jobs.
146 #Difference could be:
147 #-Lease deleted in the requested jobs
148 #-Added/removed nodes
151 logger.debug("IOTLABSLICES verify_slice_leases \
152 requested_nodes_by_start_time %s \
153 "% (requested_nodes_by_start_time))
154 #Find all deleted leases
156 list(set(leases_by_start_time.keys()).\
157 difference(requested_nodes_by_start_time.keys()))
158 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
159 for start_time in start_time_list]
162 #Find added or removed nodes in exisiting leases
163 for start_time in requested_nodes_by_start_time:
164 logger.debug("IOTLABSLICES verify_slice_leases start_time %s \
166 if start_time in current_nodes_reserved_by_start_time:
168 if requested_nodes_by_start_time[start_time] == \
169 current_nodes_reserved_by_start_time[start_time]:
174 set(requested_nodes_by_start_time[start_time])
176 update_node_set.difference(\
177 current_nodes_reserved_by_start_time[start_time])
179 update_node_set.intersection(\
180 current_nodes_reserved_by_start_time[start_time])
183 current_nodes_reserved_by_start_time[start_time])
185 old_nodes_set.difference(\
186 requested_nodes_by_start_time[start_time])
187 logger.debug("IOTLABSLICES verify_slice_leases \
188 shared_nodes %s added_nodes %s removed_nodes %s"\
189 %(shared_nodes, added_nodes,removed_nodes ))
190 #If the lease is modified, delete it before
192 #Add the deleted lease job id in the list
193 #WARNING :rescheduling does not work if there is already
194 # 2 running/scheduled jobs because deleting a job
195 #takes time SA 18/10/2012
196 if added_nodes or removed_nodes:
197 deleted_leases.append(\
198 leases_by_start_time[start_time]['lease_id'])
200 if added_nodes or shared_nodes:
201 reschedule_jobs_dict[str(start_time)] = \
202 requested_jobs_dict[str(start_time)]
207 job = requested_jobs_dict[str(start_time)]
208 logger.debug("IOTLABSLICES \
209 NEWLEASE slice %s job %s"
211 job_id = self.driver.AddLeases(
213 sfa_slice, int(job['start_time']),
214 int(job['duration']))
217 #if job_id is not None:
218 # new_leases = self.driver.GetLeases(login=
219 # sfa_slice['login'])
220 # for new_lease in new_leases:
221 # leases.append(new_lease)
223 #Deleted leases are the ones with lease id not declared in the Rspec
225 self.driver.testbed_shell.DeleteLeases(deleted_leases,
227 #self.driver.testbed_shell.DeleteLeases(deleted_leases,
228 # sfa_slice['user']['uid'])
229 logger.debug("IOTLABSLICES \
230 verify_slice_leases slice %s deleted_leases %s"
231 % (sfa_slice, deleted_leases))
233 if reschedule_jobs_dict:
234 for start_time in reschedule_jobs_dict:
235 job = reschedule_jobs_dict[start_time]
236 self.driver.AddLeases(
238 sfa_slice, int(job['start_time']),
239 int(job['duration']))
241 # Added by Jordan: until we find a better solution, always update the list of leases
242 return self.driver.GetLeases(login= sfa_slice['login'])
245 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
246 """Check for wanted and unwanted nodes in the slice.
248 Removes nodes and associated leases that the user does not want anymore
249 by deleteing the associated job in OAR (DeleteSliceFromNodes).
250 Returns the nodes' hostnames that are going to be in the slice.
252 :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
254 :param requested_slivers: list of requested nodes' hostnames.
255 :param peer: unused so far.
257 :type sfa_slice: dict
258 :type requested_slivers: list
261 :returns: list requested nodes hostnames
264 .. warning:: UNUSED SQA 24/07/13
265 .. seealso:: DeleteSliceFromNodes
266 .. todo:: check what to do with the peer? Can not remove peer nodes from
267 slice here. Anyway, in this case, the peer should have gotten the
274 if 'node_ids' in sfa_slice:
275 nodes = self.driver.testbed_shell.GetNodes(
276 sfa_slice['list_node_ids'],
278 current_slivers = [node['hostname'] for node in nodes]
280 # remove nodes not in rspec
281 deleted_nodes = list(set(current_slivers).
282 difference(requested_slivers))
284 logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
285 \r\n \r\n deleted_nodes %s"
286 % (sfa_slice, deleted_nodes))
289 #Delete the entire experience
290 self.driver.testbed_shell.DeleteSliceFromNodes(sfa_slice)
293 def verify_slice(self, slice_hrn, slice_record, sfa_peer):
294 """Ensures slice record exists.
296 The slice record must exist either in Iotlab or in the other
297 federated testbed (sfa_peer). If the slice does not belong to Iotlab,
298 check if the user already exists in LDAP. In this case, adds the slice
299 to the sfa DB and associates its LDAP user.
301 :param slice_hrn: slice's name
302 :param slice_record: sfa record of the slice
303 :param sfa_peer: name of the peer authority if any.(not Iotlab).
305 :type slice_hrn: string
306 :type slice_record: dictionary
307 :type sfa_peer: string
309 .. seealso:: AddSlice
314 slicename = slice_hrn
315 # check if slice belongs to Iotlab
316 slices_list = self.driver.GetSlices(slice_filter=slicename,
317 slice_filter_type='slice_hrn')
322 for sl in slices_list:
324 logger.debug("IOTLABSLICES \t verify_slice slicename %s \
325 slices_list %s sl %s \r slice_record %s"
326 % (slicename, slices_list, sl, slice_record))
328 sfa_slice.update(slice_record)
331 #Search for user in ldap based on email SA 14/11/12
332 ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
333 slice_record['user'])
334 logger.debug(" IOTLABSLICES \tverify_slice Oups \
335 slice_record %s sfa_peer %s ldap_user %s"
336 % (slice_record, sfa_peer, ldap_user))
337 #User already registered in ldap, meaning user should be in SFA db
338 #and hrn = sfa_auth+ uid
339 sfa_slice = {'hrn': slicename,
341 'authority': slice_record['authority'],
342 'gid': slice_record['gid'],
343 #'slice_id': slice_record['record_id'],
344 'reg-researchers': slice_record['reg-researchers'],
345 'urn': hrn_to_urn(slicename,'slice'),
346 #'peer_authority': str(sfa_peer)
350 hrn = self.driver.testbed_shell.root_auth + '.' \
352 user = self.driver.get_user_record(hrn)
354 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
357 # add the external slice to the local SFA iotlab DB
359 self.driver.AddSlice(sfa_slice, user)
361 logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
365 def verify_persons(self, slice_hrn, slice_record, users, options=None):
366 """Ensures the users in users list exist and are enabled in LDAP. Adds
367 person if needed (AddPerson).
369 Checking that a user exist is based on the user's email. If the user is
370 still not found in the LDAP, it means that the user comes from another
371 federated testbed. In this case an account has to be created in LDAP
372 so as to enable the user to use the testbed, since we trust the testbed
373 he comes from. This is done by calling AddPerson.
375 :param slice_hrn: slice name
376 :param slice_record: record of the slice_hrn
377 :param users: users is a record list. Records can either be
378 local records or users records from known and trusted federated
379 sites.If the user is from another site that iotlab doesn't trust
380 yet, then Resolve will raise an error before getting to allocate.
382 :type slice_hrn: string
383 :type slice_record: string
386 .. seealso:: AddPerson
387 .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
391 slice_user = slice_record['user']
393 if options is None: options={}
394 logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \
395 \t slice_record %s\r\n users %s \t "
396 % (slice_hrn, slice_record, users))
399 #users_dict : dict whose keys can either be the user's hrn or its id.
400 #Values contains only id and hrn
403 #First create dicts by hrn and id for each user in the user record list:
405 # if 'slice_record' in info:
406 # slice_rec = info['slice_record']
407 # if 'user' in slice_rec :
408 # user = slice_rec['user']
411 users_by_email[info['email']] = info
412 users_dict[info['email']] = info
414 logger.debug("IOTLABSLICES.PY \t verify_person \
415 users_dict %s \r\n user_by_email %s \r\n "
416 % (users_dict, users_by_email))
418 existing_user_ids = []
419 existing_user_emails = []
421 # Check if user is in Iotlab LDAP using its hrn.
422 # Assuming Iotlab is centralised : one LDAP for all sites,
423 # user's record_id unknown from LDAP
424 # LDAP does not provide users id, therefore we rely on email to find the
428 #Construct the list of filters (list of dicts) for GetPersons
429 filter_user = [users_by_email[email] for email in users_by_email]
430 #Check user i in LDAP with GetPersons
431 #Needed because what if the user has been deleted in LDAP but
433 # GetPersons -> LdapFindUser -> _process_ldap_info_for_one_user
434 existing_users = self.driver.testbed_shell.GetPersons(filter_user)
435 logger.debug(" \r\n IOTLABSLICES.PY \tverify_person filter_user %s\
437 % (filter_user, existing_users))
438 #User is in iotlab LDAP
440 for user in existing_users:
441 user['login'] = user['uid']
442 users_dict[user['email']].update(user)
443 existing_user_emails.append(
444 users_dict[user['email']]['email'])
445 logger.debug("User is in iotlab LDAP slice_record[user] = %s" % slice_user)
447 # User from another known trusted federated site. Check
448 # if a iotlab account matching the email has already been created.
451 if isinstance(users, list):
452 req += users[0]['email']
454 req += users['email']
455 ldap_reslt = self.driver.testbed_shell.ldap.LdapSearch(req)
456 logger.debug("LdapSearch slice_record[user] = %s" % slice_user)
458 logger.debug(" IOTLABSLICES.PY \tverify_person users \
459 USER already in Iotlab \t ldap_reslt %s \
461 existing_users.append(ldap_reslt[1])
462 logger.debug("ldap_reslt slice_record[user] = %s" % slice_user)
464 #User not existing in LDAP
465 logger.debug("IOTLABSLICES.PY \tverify_person users \
466 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
467 ldap_reslt %s " % (users, ldap_reslt))
469 requested_user_emails = users_by_email.keys()
470 # requested_user_hrns = \
471 # [users_by_email[user]['hrn'] for user in users_by_email]
472 # logger.debug("IOTLABSLICES.PY \tverify_person \
473 # users_by_email %s " % (users_by_email))
475 # #Check that the user of the slice in the slice record
476 # #matches one of the existing users
478 # if slice_record['reg-researchers'][0] in requested_user_hrns:
479 # logger.debug(" IOTLABSLICES \tverify_person ['PI']\
480 # slice_record %s" % (slice_record))
485 # users to be added, removed or updated
486 #One user in one iotlab slice : there should be no need
487 #to remove/ add any user from/to a slice.
488 #However a user from SFA which is not registered in Iotlab yet
489 #should be added to the LDAP.
490 added_user_emails = set(requested_user_emails).\
491 difference(set(existing_user_emails))
494 #self.verify_keys(existing_slice_users, updated_users_list, \
499 #requested_user_email is in existing_user_emails
500 if len(added_user_emails) == 0:
501 slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
502 logger.debug(" IOTLABSLICES \tverify_person QUICK DIRTY %s"
505 for added_user_email in added_user_emails:
506 added_user = users_dict[added_user_email]
507 logger.debug(" IOTLABSLICES \r\n \r\n \t verify_person \
508 added_user %s" % (added_user))
510 person['peer_person_id'] = None
511 k_list = ['first_name', 'last_name', 'person_id']
514 person[k] = added_user[k]
515 # bug user without key
516 if added_user['keys']:
517 person['pkey'] = added_user['keys'][0]
518 person['mail'] = added_user['email']
519 person['email'] = added_user['email']
520 person['key_ids'] = added_user.get('key_ids', [])
522 ret = self.driver.AddPerson(person)
524 # meaning bool is True and the AddPerson was successful
525 person['uid'] = ret['uid']
526 slice_record['login'] = person['uid']
528 # error message in ret
529 logger.debug(" IOTLABSLICES ret message %s" %(ret))
531 logger.debug(" IOTLABSLICES \r\n \r\n \t THE SECOND verify_person\
532 person %s" % (person))
533 #Update slice_Record with the id now known to LDAP
536 added_persons.append(person)
540 def verify_keys(self, persons, users, peer, options=None):
544 if options is None: options={}
547 for person in persons:
548 key_ids.extend(person['key_ids'])
549 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
553 keydict[key['key']] = key['key_id']
554 existing_keys = keydict.keys()
557 for person in persons:
558 persondict[person['email']] = person
563 users_by_key_string = {}
565 user_keys = user.get('keys', [])
566 updated_persons.append(user)
567 for key_string in user_keys:
568 users_by_key_string[key_string] = user
569 requested_keys.append(key_string)
570 if key_string not in existing_keys:
571 key = {'key': key_string, 'key_type': 'ssh'}
574 #person = persondict[user['email']]
575 #self.driver.testbed_shell.UnBindObjectFromPeer(
576 # 'person',person['person_id'],
578 ret = self.driver.testbed_shell.AddPersonKey(
581 #key_index = user_keys.index(key['key'])
582 #remote_key_id = user['key_ids'][key_index]
583 #self.driver.testbed_shell.BindObjectToPeer('key', \
584 #key['key_id'], peer['shortname'], \
587 # remove old keys (only if we are not appending)
588 append = options.get('append', True)
590 removed_keys = set(existing_keys).difference(requested_keys)
591 for key in removed_keys:
593 #self.driver.testbed_shell.UnBindObjectFromPeer('key', \
594 #key, peer['shortname'])
596 user = users_by_key_string[key]
597 self.driver.testbed_shell.DeleteKey(user, key)