iotlab driver: applying Jordan's fixes to the driver
[sfa.git] / sfa / iotlab / iotlabslices.py
1 """
2 This file defines the IotlabSlices class by which all the slice checkings
3 upon lease creation are done.
4 """
5 from sfa.util.xrn import get_authority, urn_to_hrn, hrn_to_urn
6 from sfa.util.sfalogging import logger
7
8 MAXINT = 2L**31-1
9
10
11 class IotlabSlices:
12     """
13     This class is responsible for checking the slice when creating a
14     lease or a sliver. Those checks include verifying that the user is valid,
15     that the slice is known from the testbed or from our peers, that the list
16     of nodes involved has not changed (in this case the lease is modified
17     accordingly).
18     """
19     rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
20
21     def __init__(self, driver):
22         """
23         Get the reference to the driver here.
24         """
25         self.driver = driver
26
27     def get_peer(self, xrn):
28         """
29         Finds the authority of a resource based on its xrn.
30         If the authority is Iotlab (local) return None,
31         Otherwise, look up in the DB if Iotlab is federated with this site
32         authority and returns its DB record if it is the case.
33
34         :param xrn: resource's xrn
35         :type xrn: string
36         :returns: peer record
37         :rtype: dict
38
39         """
40         hrn, hrn_type = urn_to_hrn(xrn)
41         #Does this slice belong to a local site or a peer iotlab site?
42         peer = None
43
44         # get this slice's authority (site)
45         slice_authority = get_authority(hrn)
46         #Iotlab stuff
47         #This slice belongs to the current site
48         if slice_authority == self.driver.testbed_shell.root_auth:
49             site_authority = slice_authority
50             return None
51
52         site_authority = get_authority(slice_authority).lower()
53         # get this site's authority (sfa root authority or sub authority)
54
55         logger.debug("IOTLABSLICES \t get_peer slice_authority  %s \
56                     site_authority %s hrn %s"
57                      % (slice_authority, site_authority, hrn))
58
59         # check if we are already peered with this site_authority
60         #if so find the peer record
61         peers = self.driver.GetPeers(peer_filter=site_authority)
62         for peer_record in peers:
63             if site_authority == peer_record.hrn:
64                 peer = peer_record
65         logger.debug(" IOTLABSLICES \tget_peer peer  %s " % (peer))
66         return peer
67
68     def get_sfa_peer(self, xrn):
69         """Returns the authority name for the xrn or None if the local site
70         is the authority.
71
72         :param xrn: the xrn of the resource we are looking the authority for.
73         :type xrn: string
74         :returns: the resources's authority name.
75         :rtype: string
76
77         """
78         hrn, hrn_type = urn_to_hrn(xrn)
79
80         # return the authority for this hrn or None if we are the authority
81         sfa_peer = None
82         slice_authority = get_authority(hrn)
83         site_authority = get_authority(slice_authority)
84
85         if site_authority != self.driver.hrn:
86             sfa_peer = site_authority
87
88         return sfa_peer
89
90     def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
91         """
92         Compare requested leases with the leases already scheduled/
93         running in OAR. If necessary, delete and recreate modified leases,
94         and delete no longer requested ones.
95
96         :param sfa_slice: sfa slice record
97         :param requested_jobs_dict: dictionary of requested leases
98         :param peer: sfa peer record
99
100         :type sfa_slice: dict
101         :type requested_jobs_dict: dict
102         :type peer: dict
103         :returns: leases list of dictionary
104         :rtype: list
105
106         """
107
108         logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
109                      % (sfa_slice))
110         #First get the list of current leases from OAR
111         leases = self.driver.GetLeases({'slice_hrn': sfa_slice['hrn']})
112         logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
113                         leases %s " % (requested_jobs_dict, leases))
114
115         current_nodes_reserved_by_start_time = {}
116         requested_nodes_by_start_time = {}
117         leases_by_start_time = {}
118         reschedule_jobs_dict = {}
119
120         #Create reduced dictionary with key start_time and value
121         # the list of nodes
122         #-for the leases already registered by OAR first
123         # then for the new leases requested by the user
124
125         #Leases already scheduled/running in OAR
126         for lease in leases:
127             current_nodes_reserved_by_start_time[lease['t_from']] = \
128                     lease['reserved_nodes']
129             leases_by_start_time[lease['t_from']] = lease
130
131         #First remove job whose duration is too short
132         for job in requested_jobs_dict.values():
133             job['duration'] = \
134                 str(int(job['duration']) \
135                 * self.driver.testbed_shell.GetLeaseGranularity())
136             if job['duration'] < \
137                     self.driver.testbed_shell.GetLeaseGranularity():
138                 del requested_jobs_dict[job['start_time']]
139
140         #Requested jobs
141         for start_time in requested_jobs_dict:
142             requested_nodes_by_start_time[int(start_time)] = \
143                 requested_jobs_dict[start_time]['hostname']
144         #Check if there is any difference between the leases already
145         #registered in OAR and the requested jobs.
146         #Difference could be:
147         #-Lease deleted in the requested jobs
148         #-Added/removed nodes
149         #-Newly added lease
150
151         logger.debug("IOTLABSLICES verify_slice_leases \
152                         requested_nodes_by_start_time %s \
153                         "% (requested_nodes_by_start_time))
154         #Find all deleted leases
155         start_time_list = \
156             list(set(leases_by_start_time.keys()).\
157             difference(requested_nodes_by_start_time.keys()))
158         deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
159                             for start_time in start_time_list]
160
161
162         #Find added or removed nodes in exisiting leases
163         for start_time in requested_nodes_by_start_time:
164             logger.debug("IOTLABSLICES verify_slice_leases  start_time %s \
165                          "%( start_time))
166             if start_time in current_nodes_reserved_by_start_time:
167
168                 if requested_nodes_by_start_time[start_time] == \
169                     current_nodes_reserved_by_start_time[start_time]:
170                     continue
171
172                 else:
173                     update_node_set = \
174                             set(requested_nodes_by_start_time[start_time])
175                     added_nodes = \
176                         update_node_set.difference(\
177                         current_nodes_reserved_by_start_time[start_time])
178                     shared_nodes = \
179                         update_node_set.intersection(\
180                         current_nodes_reserved_by_start_time[start_time])
181                     old_nodes_set = \
182                         set(\
183                         current_nodes_reserved_by_start_time[start_time])
184                     removed_nodes = \
185                         old_nodes_set.difference(\
186                         requested_nodes_by_start_time[start_time])
187                     logger.debug("IOTLABSLICES verify_slice_leases \
188                         shared_nodes %s  added_nodes %s removed_nodes %s"\
189                         %(shared_nodes, added_nodes,removed_nodes ))
190                     #If the lease is modified, delete it before
191                     #creating it again.
192                     #Add the deleted lease job id in the list
193                     #WARNING :rescheduling does not work if there is already
194                     # 2 running/scheduled jobs because deleting a job
195                     #takes time SA 18/10/2012
196                     if added_nodes or removed_nodes:
197                         deleted_leases.append(\
198                             leases_by_start_time[start_time]['lease_id'])
199                         #Reschedule the job
200                         if added_nodes or shared_nodes:
201                             reschedule_jobs_dict[str(start_time)] = \
202                                         requested_jobs_dict[str(start_time)]
203
204             else:
205                     #New lease
206
207                 job = requested_jobs_dict[str(start_time)]
208                 logger.debug("IOTLABSLICES \
209                               NEWLEASE slice %s  job %s"
210                              % (sfa_slice, job))
211                 job_id = self.driver.AddLeases(
212                     job['hostname'],
213                     sfa_slice, int(job['start_time']),
214                     int(job['duration']))
215
216                 # Removed by jordan
217                 #if job_id is not None:
218                 #    new_leases = self.driver.GetLeases(login=
219                 #        sfa_slice['login'])
220                 #    for new_lease in new_leases:
221                 #        leases.append(new_lease)
222
223         #Deleted leases are the ones with lease id not declared in the Rspec
224         if deleted_leases:
225             self.driver.testbed_shell.DeleteLeases(deleted_leases,
226                                                 sfa_slice['login'])
227             #self.driver.testbed_shell.DeleteLeases(deleted_leases,
228             #                                    sfa_slice['user']['uid'])
229             logger.debug("IOTLABSLICES \
230                           verify_slice_leases slice %s deleted_leases %s"
231                          % (sfa_slice, deleted_leases))
232
233         if reschedule_jobs_dict:
234             for start_time in reschedule_jobs_dict:
235                 job = reschedule_jobs_dict[start_time]
236                 self.driver.AddLeases(
237                     job['hostname'],
238                     sfa_slice, int(job['start_time']),
239                     int(job['duration']))
240
241         # Added by Jordan: until we find a better solution, always update the list of leases
242         return self.driver.GetLeases(login= sfa_slice['login'])
243         #return leases
244
245     def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
246         """Check for wanted and unwanted nodes in the slice.
247
248         Removes nodes and associated leases that the user does not want anymore
249         by deleteing the associated job in OAR (DeleteSliceFromNodes).
250         Returns the nodes' hostnames that are going to be in the slice.
251
252         :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
253
254         :param requested_slivers: list of requested nodes' hostnames.
255         :param peer: unused so far.
256
257         :type sfa_slice: dict
258         :type requested_slivers: list
259         :type peer: string
260
261         :returns: list requested nodes hostnames
262         :rtype: list
263
264         .. warning:: UNUSED SQA 24/07/13
265         .. seealso:: DeleteSliceFromNodes
266         .. todo:: check what to do with the peer? Can not remove peer nodes from
267             slice here. Anyway, in this case, the peer should have gotten the
268             remove request too.
269
270         """
271         current_slivers = []
272         deleted_nodes = []
273
274         if 'node_ids' in sfa_slice:
275             nodes = self.driver.testbed_shell.GetNodes(
276                 sfa_slice['list_node_ids'],
277                 ['hostname'])
278             current_slivers = [node['hostname'] for node in nodes]
279
280             # remove nodes not in rspec
281             deleted_nodes = list(set(current_slivers).
282                                  difference(requested_slivers))
283
284             logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
285                                          \r\n \r\n deleted_nodes %s"
286                          % (sfa_slice, deleted_nodes))
287
288             if deleted_nodes:
289                 #Delete the entire experience
290                 self.driver.testbed_shell.DeleteSliceFromNodes(sfa_slice)
291             return nodes
292
293     def verify_slice(self, slice_hrn, slice_record, sfa_peer):
294         """Ensures slice record exists.
295
296         The slice record must exist either in Iotlab or in the other
297         federated testbed (sfa_peer). If the slice does not belong to Iotlab,
298         check if the user already exists in LDAP. In this case, adds the slice
299         to the sfa DB and associates its LDAP user.
300
301         :param slice_hrn: slice's name
302         :param slice_record: sfa record of the slice
303         :param sfa_peer: name of the peer authority if any.(not Iotlab).
304
305         :type slice_hrn: string
306         :type slice_record: dictionary
307         :type sfa_peer: string
308
309         .. seealso:: AddSlice
310
311
312         """
313
314         slicename = slice_hrn
315         # check if slice belongs to Iotlab
316         slices_list = self.driver.GetSlices(slice_filter=slicename,
317                                             slice_filter_type='slice_hrn')
318
319         sfa_slice = None
320
321         if slices_list:
322             for sl in slices_list:
323
324                 logger.debug("IOTLABSLICES \t verify_slice slicename %s \
325                                 slices_list %s sl %s \r slice_record %s"
326                              % (slicename, slices_list, sl, slice_record))
327                 sfa_slice = sl
328                 sfa_slice.update(slice_record)
329
330         else:
331             #Search for user in ldap based on email SA 14/11/12
332             ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
333                                                     slice_record['user'])
334             logger.debug(" IOTLABSLICES \tverify_slice Oups \
335                         slice_record %s sfa_peer %s ldap_user %s"
336                         % (slice_record, sfa_peer, ldap_user))
337             #User already registered in ldap, meaning user should be in SFA db
338             #and hrn = sfa_auth+ uid
339             sfa_slice = {'hrn': slicename,
340                          'node_list': [],
341                          'authority': slice_record['authority'],
342                          'gid': slice_record['gid'],
343                          #'slice_id': slice_record['record_id'],
344                          'reg-researchers': slice_record['reg-researchers'],
345                          'urn': hrn_to_urn(slicename,'slice'),
346                          #'peer_authority': str(sfa_peer)
347                          }
348
349             if ldap_user:
350                 hrn = self.driver.testbed_shell.root_auth + '.' \
351                                                 + ldap_user['uid']
352                 user = self.driver.get_user_record(hrn)
353
354                 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
355                              % (hrn, user))
356
357                  # add the external slice to the local SFA iotlab DB
358                 if sfa_slice:
359                     self.driver.AddSlice(sfa_slice, user)
360
361             logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
362         return sfa_slice
363
364
365     def verify_persons(self, slice_hrn, slice_record, users, options=None):
366         """Ensures the users in users list exist and are enabled in LDAP. Adds
367         person if needed (AddPerson).
368
369         Checking that a user exist is based on the user's email. If the user is
370         still not found in the LDAP, it means that the user comes from another
371         federated testbed. In this case an account has to be created in LDAP
372         so as to enable the user to use the testbed, since we trust the testbed
373         he comes from. This is done by calling AddPerson.
374
375         :param slice_hrn: slice name
376         :param slice_record: record of the slice_hrn
377         :param users: users is a record list. Records can either be
378             local records or users records from known and trusted federated
379             sites.If the user is from another site that iotlab doesn't trust
380             yet, then Resolve will raise an error before getting to allocate.
381
382         :type slice_hrn: string
383         :type slice_record: string
384         :type users: list
385
386         .. seealso:: AddPerson
387         .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
388
389
390         """
391         slice_user = slice_record['user']
392
393         if options is None: options={}
394         logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn  %s  \
395                     \t slice_record %s\r\n users %s \t  "
396                      % (slice_hrn, slice_record, users))
397
398         users_by_email = {}
399         #users_dict : dict whose keys can either be the user's hrn or its id.
400         #Values contains only id and hrn
401         users_dict = {}
402
403         #First create dicts by hrn and id for each user in the user record list:
404         for info in users:
405             # if 'slice_record' in info:
406             #     slice_rec = info['slice_record']
407                 # if 'user' in slice_rec :
408                 #     user = slice_rec['user']
409
410             if 'email' in info:
411                 users_by_email[info['email']] = info
412                 users_dict[info['email']] = info
413
414         logger.debug("IOTLABSLICES.PY \t verify_person  \
415                         users_dict %s \r\n user_by_email %s \r\n  "
416                      % (users_dict, users_by_email))
417
418         existing_user_ids = []
419         existing_user_emails = []
420         existing_users = []
421         # Check if user is in Iotlab LDAP using its hrn.
422         # Assuming Iotlab is centralised :  one LDAP for all sites,
423         # user's record_id unknown from LDAP
424         # LDAP does not provide users id, therefore we rely on email to find the
425         # user in LDAP
426
427         if users_by_email:
428             #Construct the list of filters (list of dicts) for GetPersons
429             filter_user = [users_by_email[email] for email in users_by_email]
430             #Check user i in LDAP with GetPersons
431             #Needed because what if the user has been deleted in LDAP but
432             #is still in SFA?
433             # GetPersons -> LdapFindUser -> _process_ldap_info_for_one_user
434             existing_users = self.driver.testbed_shell.GetPersons(filter_user)
435             logger.debug(" \r\n IOTLABSLICES.PY \tverify_person  filter_user %s\
436                        existing_users %s  "
437                         % (filter_user, existing_users))
438             #User is in iotlab LDAP
439             if existing_users:
440                 for user in existing_users:
441                     user['login'] = user['uid']
442                     users_dict[user['email']].update(user)
443                     existing_user_emails.append(
444                         users_dict[user['email']]['email'])
445                 logger.debug("User is in iotlab LDAP slice_record[user] = %s" % slice_user)
446
447             # User from another known trusted federated site. Check
448             # if a iotlab account matching the email has already been created.
449             else:
450                 req = 'mail='
451                 if isinstance(users, list):
452                     req += users[0]['email']
453                 else:
454                     req += users['email']
455                 ldap_reslt = self.driver.testbed_shell.ldap.LdapSearch(req)
456                 logger.debug("LdapSearch slice_record[user] = %s" % slice_user)
457                 if ldap_reslt:
458                     logger.debug(" IOTLABSLICES.PY \tverify_person users \
459                                 USER already in Iotlab \t ldap_reslt %s \
460                                 " % (ldap_reslt))
461                     existing_users.append(ldap_reslt[1])
462                     logger.debug("ldap_reslt slice_record[user] = %s" % slice_user)
463                 else:
464                     #User not existing in LDAP
465                     logger.debug("IOTLABSLICES.PY \tverify_person users \
466                                 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
467                                 ldap_reslt %s " % (users, ldap_reslt))
468
469         requested_user_emails = users_by_email.keys()
470         # requested_user_hrns = \
471         #     [users_by_email[user]['hrn'] for user in users_by_email]
472         # logger.debug("IOTLABSLICES.PY \tverify_person  \
473         #                users_by_email  %s " % (users_by_email))
474
475         # #Check that the user of the slice in the slice record
476         # #matches one of the existing users
477         # try:
478         #     if slice_record['reg-researchers'][0] in requested_user_hrns:
479         #         logger.debug(" IOTLABSLICES  \tverify_person ['PI']\
480         #                         slice_record %s" % (slice_record))
481
482         # except KeyError:
483         #     pass
484
485         # users to be added, removed or updated
486         #One user in one iotlab slice : there should be no need
487         #to remove/ add any user from/to a slice.
488         #However a user from SFA which is not registered in Iotlab yet
489         #should be added to the LDAP.
490         added_user_emails = set(requested_user_emails).\
491                                         difference(set(existing_user_emails))
492
493
494         #self.verify_keys(existing_slice_users, updated_users_list, \
495                                                             #peer, append)
496
497         added_persons = []
498         # add new users
499         #requested_user_email is in existing_user_emails
500         if len(added_user_emails) == 0:
501             slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
502             logger.debug(" IOTLABSLICES  \tverify_person QUICK DIRTY %s"
503                          % (slice_record))
504
505         for added_user_email in added_user_emails:
506             added_user = users_dict[added_user_email]
507             logger.debug(" IOTLABSLICES \r\n \r\n  \t  verify_person \
508                          added_user %s" % (added_user))
509             person = {}
510             person['peer_person_id'] = None
511             k_list = ['first_name', 'last_name', 'person_id']
512             for k in k_list:
513                 if k in added_user:
514                     person[k] = added_user[k]
515             # bug user without key
516             if added_user['keys']:
517                 person['pkey'] = added_user['keys'][0]
518             person['mail'] = added_user['email']
519             person['email'] = added_user['email']
520             person['key_ids'] = added_user.get('key_ids', [])
521
522             ret = self.driver.AddPerson(person)
523             if 'uid' in ret:
524                 # meaning bool is True and the AddPerson was successful
525                 person['uid'] = ret['uid']
526                 slice_record['login'] = person['uid']
527             else:
528                 # error message in ret
529                 logger.debug(" IOTLABSLICES ret message %s" %(ret))
530
531             logger.debug(" IOTLABSLICES \r\n \r\n  \t THE SECOND verify_person\
532                            person %s" % (person))
533             #Update slice_Record with the id now known to LDAP
534
535
536             added_persons.append(person)
537         return added_persons
538
539
540     def verify_keys(self, persons, users, peer, options=None):
541         """
542         .. warning:: unused
543         """
544         if options is None: options={}
545         # existing keys
546         key_ids = []
547         for person in persons:
548             key_ids.extend(person['key_ids'])
549         keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
550
551         keydict = {}
552         for key in keylist:
553             keydict[key['key']] = key['key_id']
554         existing_keys = keydict.keys()
555
556         persondict = {}
557         for person in persons:
558             persondict[person['email']] = person
559
560         # add new keys
561         requested_keys = []
562         updated_persons = []
563         users_by_key_string = {}
564         for user in users:
565             user_keys = user.get('keys', [])
566             updated_persons.append(user)
567             for key_string in user_keys:
568                 users_by_key_string[key_string] = user
569                 requested_keys.append(key_string)
570                 if key_string not in existing_keys:
571                     key = {'key': key_string, 'key_type': 'ssh'}
572                     #try:
573                         ##if peer:
574                             #person = persondict[user['email']]
575                             #self.driver.testbed_shell.UnBindObjectFromPeer(
576                                 # 'person',person['person_id'],
577                                 # peer['shortname'])
578                     ret = self.driver.testbed_shell.AddPersonKey(
579                         user['email'], key)
580                         #if peer:
581                             #key_index = user_keys.index(key['key'])
582                             #remote_key_id = user['key_ids'][key_index]
583                             #self.driver.testbed_shell.BindObjectToPeer('key', \
584                                             #key['key_id'], peer['shortname'], \
585                                             #remote_key_id)
586
587         # remove old keys (only if we are not appending)
588         append = options.get('append', True)
589         if append is False:
590             removed_keys = set(existing_keys).difference(requested_keys)
591             for key in removed_keys:
592                     #if peer:
593                         #self.driver.testbed_shell.UnBindObjectFromPeer('key', \
594                                         #key, peer['shortname'])
595
596                 user = users_by_key_string[key]
597                 self.driver.testbed_shell.DeleteKey(user, key)
598
599         return