2 This file defines the IotlabSlices class by which all the slice checkings
3 upon lease creation are done.
5 from sfa.util.xrn import get_authority, urn_to_hrn
6 from sfa.util.sfalogging import logger
11 class CortexlabSlices:
13 This class is responsible for checking the slice when creating a
14 lease or a sliver. Those checks include verifying that the user is valid,
15 that the slice is known from the testbed or from our peers, that the list
16 of nodes involved has not changed (in this case the lease is modified
19 rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
21 def __init__(self, driver):
23 Get the reference to the driver here.
27 def get_peer(self, xrn):
29 Finds the authority of a resource based on its xrn.
30 If the authority is Iotlab (local) return None,
31 Otherwise, look up in the DB if Iotlab is federated with this site
32 authority and returns its DB record if it is the case.
34 :param xrn: resource's xrn
40 hrn, hrn_type = urn_to_hrn(xrn)
41 #Does this slice belong to a local site or a peer cortexlab site?
44 # get this slice's authority (site)
45 slice_authority = get_authority(hrn)
47 #This slice belongs to the current site
48 if slice_authority == self.driver.testbed_shell.root_auth:
49 site_authority = slice_authority
52 site_authority = get_authority(slice_authority).lower()
53 # get this site's authority (sfa root authority or sub authority)
55 logger.debug("CortexlabSlices \t get_peer slice_authority %s \
56 site_authority %s hrn %s"
57 % (slice_authority, site_authority, hrn))
59 # check if we are already peered with this site_authority
60 #if so find the peer record
61 peers = self.driver.GetPeers(peer_filter=site_authority)
62 for peer_record in peers:
63 if site_authority == peer_record.hrn:
65 logger.debug(" CortexlabSlices \tget_peer peer %s " % (peer))
68 def get_sfa_peer(self, xrn):
69 """Returns the authority name for the xrn or None if the local site
72 :param xrn: the xrn of the resource we are looking the authority for.
74 :returns: the resources's authority name.
78 hrn, hrn_type = urn_to_hrn(xrn)
80 # return the authority for this hrn or None if we are the authority
82 slice_authority = get_authority(hrn)
83 site_authority = get_authority(slice_authority)
85 if site_authority != self.driver.hrn:
86 sfa_peer = site_authority
90 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
92 Compare requested leases with the leases already scheduled/
93 running in OAR. If necessary, delete and recreate modified leases,
94 and delete no longer requested ones.
96 :param sfa_slice: sfa slice record
97 :param requested_jobs_dict: dictionary of requested leases
98 :param peer: sfa peer record
100 :type sfa_slice: dict
101 :type requested_jobs_dict: dict
103 :returns: leases list of dictionary
108 logger.debug("CortexlabSlices verify_slice_leases sfa_slice %s "
110 #First get the list of current leases from OAR
111 leases = self.driver.GetLeases({'slice_hrn': sfa_slice['hrn']})
112 logger.debug("CortexlabSlices verify_slice_leases requested_jobs_dict %s \
113 leases %s " % (requested_jobs_dict, leases))
115 current_nodes_reserved_by_start_time = {}
116 requested_nodes_by_start_time = {}
117 leases_by_start_time = {}
118 reschedule_jobs_dict = {}
120 #Create reduced dictionary with key start_time and value
122 #-for the leases already registered by OAR first
123 # then for the new leases requested by the user
125 #Leases already scheduled/running in OAR
127 current_nodes_reserved_by_start_time[lease['t_from']] = \
128 lease['reserved_nodes']
129 leases_by_start_time[lease['t_from']] = lease
131 #First remove job whose duration is too short
132 for job in requested_jobs_dict.values():
134 str(int(job['duration']) \
135 * self.driver.testbed_shell.GetLeaseGranularity())
136 if job['duration'] < self.driver.testbed_shell.GetLeaseGranularity():
137 del requested_jobs_dict[job['start_time']]
140 for start_time in requested_jobs_dict:
141 requested_nodes_by_start_time[int(start_time)] = \
142 requested_jobs_dict[start_time]['hostname']
143 #Check if there is any difference between the leases already
144 #registered in OAR and the requested jobs.
145 #Difference could be:
146 #-Lease deleted in the requested jobs
147 #-Added/removed nodes
150 logger.debug("CortexlabSlices verify_slice_leases \
151 requested_nodes_by_start_time %s \
152 "% (requested_nodes_by_start_time))
153 #Find all deleted leases
155 list(set(leases_by_start_time.keys()).\
156 difference(requested_nodes_by_start_time.keys()))
157 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
158 for start_time in start_time_list]
161 #Find added or removed nodes in exisiting leases
162 for start_time in requested_nodes_by_start_time:
163 logger.debug("CortexlabSlices verify_slice_leases start_time %s \
165 if start_time in current_nodes_reserved_by_start_time:
167 if requested_nodes_by_start_time[start_time] == \
168 current_nodes_reserved_by_start_time[start_time]:
173 set(requested_nodes_by_start_time[start_time])
175 update_node_set.difference(\
176 current_nodes_reserved_by_start_time[start_time])
178 update_node_set.intersection(\
179 current_nodes_reserved_by_start_time[start_time])
182 current_nodes_reserved_by_start_time[start_time])
184 old_nodes_set.difference(\
185 requested_nodes_by_start_time[start_time])
186 logger.debug("CortexlabSlices verify_slice_leases \
187 shared_nodes %s added_nodes %s removed_nodes %s"\
188 %(shared_nodes, added_nodes,removed_nodes ))
189 #If the lease is modified, delete it before
191 #Add the deleted lease job id in the list
192 #WARNING :rescheduling does not work if there is already
193 # 2 running/scheduled jobs because deleting a job
194 #takes time SA 18/10/2012
195 if added_nodes or removed_nodes:
196 deleted_leases.append(\
197 leases_by_start_time[start_time]['lease_id'])
199 if added_nodes or shared_nodes:
200 reschedule_jobs_dict[str(start_time)] = \
201 requested_jobs_dict[str(start_time)]
206 job = requested_jobs_dict[str(start_time)]
207 logger.debug("CortexlabSlices \
208 NEWLEASE slice %s job %s"
210 job_id = self.driver.AddLeases(job['hostname'],
211 sfa_slice, int(job['start_time']),
212 int(job['duration']))
213 if job_id is not None:
214 new_leases = self.driver.GetLeases(login=
216 for new_lease in new_leases:
217 leases.append(new_lease)
219 #Deleted leases are the ones with lease id not declared in the Rspec
221 self.driver.testbed_shell.DeleteLeases(deleted_leases,
222 sfa_slice['user']['uid'])
223 logger.debug("CortexlabSlices \
224 verify_slice_leases slice %s deleted_leases %s"
225 % (sfa_slice, deleted_leases))
227 if reschedule_jobs_dict:
228 for start_time in reschedule_jobs_dict:
229 job = reschedule_jobs_dict[start_time]
230 self.driver.AddLeases(job['hostname'],
231 sfa_slice, int(job['start_time']),
232 int(job['duration']))
235 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
236 """Check for wanted and unwanted nodes in the slice.
238 Removes nodes and associated leases that the user does not want anymore
239 by deleteing the associated job in OAR (DeleteSliceFromNodes).
240 Returns the nodes' hostnames that are going to be in the slice.
242 :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
244 :param requested_slivers: list of requested nodes' hostnames.
245 :param peer: unused so far.
247 :type sfa_slice: dict
248 :type requested_slivers: list
251 :returns: list requested nodes hostnames
254 .. warning:: UNUSED SQA 24/07/13
255 .. seealso:: DeleteSliceFromNodes
256 .. todo:: check what to do with the peer? Can not remove peer nodes from
257 slice here. Anyway, in this case, the peer should have gotten the
264 if 'node_ids' in sfa_slice:
265 nodes = self.driver.testbed_shell.GetNodes(
266 sfa_slice['list_node_ids'],
268 current_slivers = [node['hostname'] for node in nodes]
270 # remove nodes not in rspec
271 deleted_nodes = list(set(current_slivers).
272 difference(requested_slivers))
274 logger.debug("CortexlabSlices \tverify_slice_nodes slice %s\
275 \r\n \r\n deleted_nodes %s"
276 % (sfa_slice, deleted_nodes))
279 #Delete the entire experience
280 self.driver.testbed_shell.DeleteSliceFromNodes(sfa_slice)
283 def verify_slice(self, slice_hrn, slice_record, sfa_peer):
284 """Ensures slice record exists.
286 The slice record must exist either in Iotlab or in the other
287 federated testbed (sfa_peer). If the slice does not belong to Iotlab,
288 check if the user already exists in LDAP. In this case, adds the slice
289 to the sfa DB and associates its LDAP user.
291 :param slice_hrn: slice's name
292 :param slice_record: sfa record of the slice
293 :param sfa_peer: name of the peer authority if any.(not Iotlab).
295 :type slice_hrn: string
296 :type slice_record: dictionary
297 :type sfa_peer: string
299 .. seealso:: AddSlice
304 slicename = slice_hrn
305 # check if slice belongs to Iotlab
306 slices_list = self.driver.GetSlices(
307 slice_filter=slicename, slice_filter_type='slice_hrn')
312 for sl in slices_list:
314 logger.debug("CortexlabSlices \t verify_slice slicename %s \
315 slices_list %s sl %s \r slice_record %s"
316 % (slicename, slices_list, sl, slice_record))
318 sfa_slice.update(slice_record)
321 #Search for user in ldap based on email SA 14/11/12
322 ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
323 slice_record['user'])
324 logger.debug(" CortexlabSlices \tverify_slice Oups \
325 slice_record %s sfa_peer %s ldap_user %s"
326 % (slice_record, sfa_peer, ldap_user))
327 #User already registered in ldap, meaning user should be in SFA db
328 #and hrn = sfa_auth+ uid
329 sfa_slice = {'hrn': slicename,
331 'authority': slice_record['authority'],
332 'gid': slice_record['gid'],
333 'slice_id': slice_record['record_id'],
334 'reg-researchers': slice_record['reg-researchers'],
335 'peer_authority': str(sfa_peer)
339 hrn = self.driver.testbed_shell.root_auth + '.' \
341 user = self.driver.get_user_record(hrn)
343 logger.debug(" CortexlabSlices \tverify_slice hrn %s USER %s"
346 # add the external slice to the local SFA DB
348 self.driver.AddSlice(sfa_slice, user)
350 logger.debug("CortexlabSlices \tverify_slice ADDSLICE OK")
354 def verify_persons(self, slice_hrn, slice_record, users, options=None):
355 """Ensures the users in users list exist and are enabled in LDAP. Adds
356 person if needed(AddPerson).
358 Checking that a user exist is based on the user's email. If the user is
359 still not found in the LDAP, it means that the user comes from another
360 federated testbed. In this case an account has to be created in LDAP
361 so as to enable the user to use the testbed, since we trust the testbed
362 he comes from. This is done by calling AddPerson.
364 :param slice_hrn: slice name
365 :param slice_record: record of the slice_hrn
366 :param users: users is a record list. Records can either be
367 local records or users records from known and trusted federated
368 sites.If the user is from another site that cortex;ab doesn't trust
369 yet, then Resolve will raise an error before getting to allocate.
371 :type slice_hrn: string
372 :type slice_record: string
375 .. seealso:: AddPerson
376 .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
381 if options is None: options={}
383 logger.debug("CortexlabSlices \tverify_persons \tslice_hrn %s \
384 \t slice_record %s\r\n users %s \t "
385 % (slice_hrn, slice_record, users))
389 #users_dict : dict whose keys can either be the user's hrn or its id.
390 #Values contains only id and hrn
393 #First create dicts by hrn and id for each user in the user record list:
395 # if 'slice_record' in info:
396 # slice_rec = info['slice_record']
397 # if 'user' in slice_rec :
398 # user = slice_rec['user']
401 users_by_email[info['email']] = info
402 users_dict[info['email']] = info
405 logger.debug("CortexlabSlices.PY \t verify_person \
406 users_dict %s \r\n user_by_email %s \r\n "
407 %(users_dict, users_by_email))
409 existing_user_ids = []
410 existing_user_emails = []
412 # Check if user is in Iotlab LDAP using its hrn.
413 # Assuming Iotlab is centralised : one LDAP for all sites,
414 # user's record_id unknown from LDAP
415 # LDAP does not provide users id, therefore we rely on email to find the
419 #Construct the list of filters (list of dicts) for GetPersons
420 filter_user = [users_by_email[email] for email in users_by_email]
421 #Check user i in LDAP with GetPersons
422 #Needed because what if the user has been deleted in LDAP but
424 existing_users = self.driver.testbed_shell.GetPersons(filter_user)
425 logger.debug(" \r\n CortexlabSlices.PY \tverify_person filter_user \
426 %s existing_users %s "
427 % (filter_user, existing_users))
430 for user in existing_users:
431 user['login'] = user['uid']
432 users_dict[user['email']].update(user)
433 existing_user_emails.append(
434 users_dict[user['email']]['email'])
437 # User from another known trusted federated site. Check
438 # if a cortexlab account matching the email has already been created.
441 if isinstance(users, list):
442 req += users[0]['email']
444 req += users['email']
445 ldap_reslt = self.driver.testbed_shell.ldap.LdapSearch(req)
448 logger.debug(" CortexlabSlices.PY \tverify_person users \
449 USER already in Iotlab \t ldap_reslt %s \
451 existing_users.append(ldap_reslt[1])
454 #User not existing in LDAP
455 logger.debug("CortexlabSlices.PY \tverify_person users \
456 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
457 ldap_reslt %s " % (users, ldap_reslt))
459 requested_user_emails = users_by_email.keys()
460 requested_user_hrns = \
461 [users_by_email[user]['hrn'] for user in users_by_email]
462 logger.debug("CortexlabSlices.PY \tverify_person \
463 users_by_email %s " % (users_by_email))
465 #Check that the user of the slice in the slice record
466 #matches one of the existing users
468 if slice_record['reg-researchers'][0] in requested_user_hrns:
469 logger.debug(" CortexlabSlices \tverify_person ['PI']\
470 slice_record %s" % (slice_record))
475 # users to be added, removed or updated
476 #One user in one cortexlab slice : there should be no need
477 #to remove/ add any user from/to a slice.
478 #However a user from SFA which is not registered in Iotlab yet
479 #should be added to the LDAP.
480 added_user_emails = set(requested_user_emails).\
481 difference(set(existing_user_emails))
484 #self.verify_keys(existing_slice_users, updated_users_list, \
489 #requested_user_email is in existing_user_emails
490 if len(added_user_emails) == 0:
491 slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
492 logger.debug(" CortexlabSlices \tverify_person QUICK DIRTY %s"
495 for added_user_email in added_user_emails:
496 added_user = users_dict[added_user_email]
497 logger.debug(" CortexlabSlices \r\n \r\n \t verify_person \
498 added_user %s" % (added_user))
500 person['peer_person_id'] = None
501 k_list = ['first_name', 'last_name', 'person_id']
504 person[k] = added_user[k]
506 person['pkey'] = added_user['keys'][0]
507 person['mail'] = added_user['email']
508 person['email'] = added_user['email']
509 person['key_ids'] = added_user.get('key_ids', [])
511 ret = self.driver.testbed_shell.AddPerson(person)
513 # meaning bool is True and the AddPerson was successful
514 person['uid'] = ret['uid']
515 slice_record['login'] = person['uid']
517 # error message in ret
518 logger.debug(" CortexlabSlices ret message %s" %(ret))
520 logger.debug(" CortexlabSlices \r\n \r\n \t THE SECOND verify_person\
521 person %s" % (person))
522 #Update slice_Record with the id now known to LDAP
525 added_persons.append(person)
529 def verify_keys(self, persons, users, peer, options=None):
533 if options is None: options={}
536 for person in persons:
537 key_ids.extend(person['key_ids'])
538 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
542 keydict[key['key']] = key['key_id']
543 existing_keys = keydict.keys()
546 for person in persons:
547 persondict[person['email']] = person
552 users_by_key_string = {}
554 user_keys = user.get('keys', [])
555 updated_persons.append(user)
556 for key_string in user_keys:
557 users_by_key_string[key_string] = user
558 requested_keys.append(key_string)
559 if key_string not in existing_keys:
560 key = {'key': key_string, 'key_type': 'ssh'}
563 #person = persondict[user['email']]
564 #self.driver.testbed_shell.UnBindObjectFromPeer(
565 # 'person',person['person_id'],
567 ret = self.driver.testbed_shell.AddPersonKey(
570 #key_index = user_keys.index(key['key'])
571 #remote_key_id = user['key_ids'][key_index]
572 #self.driver.testbed_shell.BindObjectToPeer('key', \
573 #key['key_id'], peer['shortname'], \
578 # remove old keys (only if we are not appending)
579 append = options.get('append', True)
581 removed_keys = set(existing_keys).difference(requested_keys)
582 for key in removed_keys:
584 #self.driver.testbed_shell.UnBindObjectFromPeer('key', \
585 #key, peer['shortname'])
587 user = users_by_key_string[key]
588 self.driver.testbed_shell.DeleteKey(user, key)