sfa/iotlab/iotlabslices.py

   1 """
   2 This file defines the IotlabSlices class by which all the slice checkings
   3 upon lease creation are done.
   4 """
   5 from sfa.util.xrn import get_authority, urn_to_hrn
   6 from sfa.util.sfalogging import logger
   7
   8 MAXINT = 2L**31-1
   9
  10
  11 class IotlabSlices:
  12     """
  13     This class is responsible for checking the slice when creating a
  14     lease or a sliver. Those checks include verifying that the user is valid,
  15     that the slice is known from the testbed or from our peers, that the list
  16     of nodes involved has not changed (in this case the lease is modified
  17     accordingly).
  18     """
  19     rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
  20
  21     def __init__(self, driver):
  22         """
  23         Get the reference to the driver here.
  24         """
  25         self.driver = driver
  26
  27     def get_peer(self, xrn):
  28         """
  29         Finds the authority of a resource based on its xrn.
  30         If the authority is Iotlab (local) return None,
  31         Otherwise, look up in the DB if Iotlab is federated with this site
  32         authority and returns its DB record if it is the case.
  33
  34         :param xrn: resource's xrn
  35         :type xrn: string
  36         :returns: peer record
  37         :rtype: dict
  38
  39         """
  40         hrn, hrn_type = urn_to_hrn(xrn)
  41         #Does this slice belong to a local site or a peer iotlab site?
  42         peer = None
  43
  44         # get this slice's authority (site)
  45         slice_authority = get_authority(hrn)
  46         #Iotlab stuff
  47         #This slice belongs to the current site
  48         if slice_authority == self.driver.testbed_shell.root_auth:
  49             site_authority = slice_authority
  50             return None
  51
  52         site_authority = get_authority(slice_authority).lower()
  53         # get this site's authority (sfa root authority or sub authority)
  54
  55         logger.debug("IOTLABSLICES \t get_peer slice_authority  %s \
  56                     site_authority %s hrn %s"
  57                      % (slice_authority, site_authority, hrn))
  58
  59         # check if we are already peered with this site_authority
  60         #if so find the peer record
  61         peers = self.driver.testbed_shell.GetPeers(peer_filter=site_authority)
  62         for peer_record in peers:
  63             if site_authority == peer_record.hrn:
  64                 peer = peer_record
  65         logger.debug(" IOTLABSLICES \tget_peer peer  %s " % (peer))
  66         return peer
  67
  68     def get_sfa_peer(self, xrn):
  69         """Returns the authority name for the xrn or None if the local site
  70         is the authority.
  71
  72         :param xrn: the xrn of the resource we are looking the authority for.
  73         :type xrn: string
  74         :returns: the resources's authority name.
  75         :rtype: string
  76
  77         """
  78         hrn, hrn_type = urn_to_hrn(xrn)
  79
  80         # return the authority for this hrn or None if we are the authority
  81         sfa_peer = None
  82         slice_authority = get_authority(hrn)
  83         site_authority = get_authority(slice_authority)
  84
  85         if site_authority != self.driver.hrn:
  86             sfa_peer = site_authority
  87
  88         return sfa_peer
  89
  90     def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
  91         """
  92         Compare requested leases with the leases already scheduled/
  93         running in OAR. If necessary, delete and recreate modified leases,
  94         and delete no longer requested ones.
  95
  96         :param sfa_slice: sfa slice record
  97         :param requested_jobs_dict: dictionary of requested leases
  98         :param peer: sfa peer record
  99
 100         :type sfa_slice: dict
 101         :type requested_jobs_dict: dict
 102         :type peer: dict
 103         :returns: leases list of dictionary
 104         :rtype: list
 105
 106         """
 107
 108         logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
 109                      % (sfa_slice))
 110         #First get the list of current leases from OAR
 111         leases = self.driver.testbed_shell.GetLeases({'slice_hrn': sfa_slice['hrn']})
 112         logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
 113                         leases %s " % (requested_jobs_dict, leases))
 114
 115         current_nodes_reserved_by_start_time = {}
 116         requested_nodes_by_start_time = {}
 117         leases_by_start_time = {}
 118         reschedule_jobs_dict = {}
 119
 120         #Create reduced dictionary with key start_time and value
 121         # the list of nodes
 122         #-for the leases already registered by OAR first
 123         # then for the new leases requested by the user
 124
 125         #Leases already scheduled/running in OAR
 126         for lease in leases:
 127             current_nodes_reserved_by_start_time[lease['t_from']] = \
 128                     lease['reserved_nodes']
 129             leases_by_start_time[lease['t_from']] = lease
 130
 131         #First remove job whose duration is too short
 132         for job in requested_jobs_dict.values():
 133             job['duration'] = \
 134                 str(int(job['duration']) \
 135                 * self.driver.testbed_shell.GetLeaseGranularity())
 136             if job['duration'] < self.driver.testbed_shell.GetLeaseGranularity():
 137                 del requested_jobs_dict[job['start_time']]
 138
 139         #Requested jobs
 140         for start_time in requested_jobs_dict:
 141             requested_nodes_by_start_time[int(start_time)] = \
 142                 requested_jobs_dict[start_time]['hostname']
 143         #Check if there is any difference between the leases already
 144         #registered in OAR and the requested jobs.
 145         #Difference could be:
 146         #-Lease deleted in the requested jobs
 147         #-Added/removed nodes
 148         #-Newly added lease
 149
 150         logger.debug("IOTLABSLICES verify_slice_leases \
 151                         requested_nodes_by_start_time %s \
 152                         "% (requested_nodes_by_start_time))
 153         #Find all deleted leases
 154         start_time_list = \
 155             list(set(leases_by_start_time.keys()).\
 156             difference(requested_nodes_by_start_time.keys()))
 157         deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
 158                             for start_time in start_time_list]
 159
 160
 161         #Find added or removed nodes in exisiting leases
 162         for start_time in requested_nodes_by_start_time:
 163             logger.debug("IOTLABSLICES verify_slice_leases  start_time %s \
 164                          "%( start_time))
 165             if start_time in current_nodes_reserved_by_start_time:
 166
 167                 if requested_nodes_by_start_time[start_time] == \
 168                     current_nodes_reserved_by_start_time[start_time]:
 169                     continue
 170
 171                 else:
 172                     update_node_set = \
 173                             set(requested_nodes_by_start_time[start_time])
 174                     added_nodes = \
 175                         update_node_set.difference(\
 176                         current_nodes_reserved_by_start_time[start_time])
 177                     shared_nodes = \
 178                         update_node_set.intersection(\
 179                         current_nodes_reserved_by_start_time[start_time])
 180                     old_nodes_set = \
 181                         set(\
 182                         current_nodes_reserved_by_start_time[start_time])
 183                     removed_nodes = \
 184                         old_nodes_set.difference(\
 185                         requested_nodes_by_start_time[start_time])
 186                     logger.debug("IOTLABSLICES verify_slice_leases \
 187                         shared_nodes %s  added_nodes %s removed_nodes %s"\
 188                         %(shared_nodes, added_nodes,removed_nodes ))
 189                     #If the lease is modified, delete it before
 190                     #creating it again.
 191                     #Add the deleted lease job id in the list
 192                     #WARNING :rescheduling does not work if there is already
 193                     # 2 running/scheduled jobs because deleting a job
 194                     #takes time SA 18/10/2012
 195                     if added_nodes or removed_nodes:
 196                         deleted_leases.append(\
 197                             leases_by_start_time[start_time]['lease_id'])
 198                         #Reschedule the job
 199                         if added_nodes or shared_nodes:
 200                             reschedule_jobs_dict[str(start_time)] = \
 201                                         requested_jobs_dict[str(start_time)]
 202
 203             else:
 204                     #New lease
 205
 206                 job = requested_jobs_dict[str(start_time)]
 207                 logger.debug("IOTLABSLICES \
 208                               NEWLEASE slice %s  job %s"
 209                              % (sfa_slice, job))
 210                 job_id = self.driver.testbed_shell.AddLeases(
 211                     job['hostname'],
 212                     sfa_slice, int(job['start_time']),
 213                     int(job['duration']))
 214                 if job_id is not None:
 215                     new_leases = self.driver.testbed_shell.GetLeases(login=
 216                         sfa_slice['login'])
 217                     for new_lease in new_leases:
 218                         leases.append(new_lease)
 219
 220         #Deleted leases are the ones with lease id not declared in the Rspec
 221         if deleted_leases:
 222             self.driver.testbed_shell.DeleteLeases(deleted_leases,
 223                                                 sfa_slice['user']['uid'])
 224             logger.debug("IOTLABSLICES \
 225                           verify_slice_leases slice %s deleted_leases %s"
 226                          % (sfa_slice, deleted_leases))
 227
 228         if reschedule_jobs_dict:
 229             for start_time in reschedule_jobs_dict:
 230                 job = reschedule_jobs_dict[start_time]
 231                 self.driver.testbed_shell.AddLeases(
 232                     job['hostname'],
 233                     sfa_slice, int(job['start_time']),
 234                     int(job['duration']))
 235         return leases
 236
 237     def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
 238         """Check for wanted and unwanted nodes in the slice.
 239
 240         Removes nodes and associated leases that the user does not want anymore
 241         by deleteing the associated job in OAR (DeleteSliceFromNodes).
 242         Returns the nodes' hostnames that are going to be in the slice.
 243
 244         :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
 245
 246         :param requested_slivers: list of requested nodes' hostnames.
 247         :param peer: unused so far.
 248
 249         :type sfa_slice: dict
 250         :type requested_slivers: list
 251         :type peer: string
 252
 253         :returns: list requested nodes hostnames
 254         :rtype: list
 255
 256         .. warning:: UNUSED SQA 24/07/13
 257         .. seealso:: DeleteSliceFromNodes
 258         .. todo:: check what to do with the peer? Can not remove peer nodes from
 259             slice here. Anyway, in this case, the peer should have gotten the
 260             remove request too.
 261
 262         """
 263         current_slivers = []
 264         deleted_nodes = []
 265
 266         if 'node_ids' in sfa_slice:
 267             nodes = self.driver.testbed_shell.GetNodes(
 268                 sfa_slice['list_node_ids'],
 269                 ['hostname'])
 270             current_slivers = [node['hostname'] for node in nodes]
 271
 272             # remove nodes not in rspec
 273             deleted_nodes = list(set(current_slivers).
 274                                  difference(requested_slivers))
 275
 276             logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
 277                                          \r\n \r\n deleted_nodes %s"
 278                          % (sfa_slice, deleted_nodes))
 279
 280             if deleted_nodes:
 281                 #Delete the entire experience
 282                 self.driver.testbed_shell.DeleteSliceFromNodes(sfa_slice)
 283             return nodes
 284
 285     def verify_slice(self, slice_hrn, slice_record, sfa_peer):
 286         """Ensures slice record exists.
 287
 288         The slice record must exist either in Iotlab or in the other
 289         federated testbed (sfa_peer). If the slice does not belong to Iotlab,
 290         check if the user already exists in LDAP. In this case, adds the slice
 291         to the sfa DB and associates its LDAP user.
 292
 293         :param slice_hrn: slice's name
 294         :param slice_record: sfa record of the slice
 295         :param sfa_peer: name of the peer authority if any.(not Iotlab).
 296
 297         :type slice_hrn: string
 298         :type slice_record: dictionary
 299         :type sfa_peer: string
 300
 301         .. seealso:: AddSlice
 302
 303
 304         """
 305
 306         slicename = slice_hrn
 307         # check if slice belongs to Iotlab
 308         slices_list = self.driver.testbed_shell.GetSlices(
 309             slice_filter=slicename, slice_filter_type='slice_hrn')
 310
 311         sfa_slice = None
 312
 313         if slices_list:
 314             for sl in slices_list:
 315
 316                 logger.debug("SLABSLICE \t verify_slice slicename %s \
 317                                 slices_list %s sl %s \r slice_record %s"
 318                              % (slicename, slices_list, sl, slice_record))
 319                 sfa_slice = sl
 320                 sfa_slice.update(slice_record)
 321
 322         else:
 323             #Search for user in ldap based on email SA 14/11/12
 324             ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
 325                                                     slice_record['user'])
 326             logger.debug(" IOTLABSLICES \tverify_slice Oups \
 327                         slice_record %s sfa_peer %s ldap_user %s"
 328                         % (slice_record, sfa_peer, ldap_user))
 329             #User already registered in ldap, meaning user should be in SFA db
 330             #and hrn = sfa_auth+ uid
 331             sfa_slice = {'hrn': slicename,
 332                          'node_list': [],
 333                          'authority': slice_record['authority'],
 334                          'gid': slice_record['gid'],
 335                          'slice_id': slice_record['record_id'],
 336                          'reg-researchers': slice_record['reg-researchers'],
 337                          'peer_authority': str(sfa_peer)
 338                          }
 339
 340             if ldap_user:
 341                 hrn = self.driver.testbed_shell.root_auth + '.' + ldap_user['uid']
 342                 user = self.driver.get_user_record(hrn)
 343
 344                 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
 345                              % (hrn, user))
 346
 347                  # add the external slice to the local SFA iotlab DB
 348                 if sfa_slice:
 349                     self.driver.testbed_shell.AddSlice(sfa_slice, user)
 350
 351             logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
 352         return sfa_slice
 353
 354
 355     def verify_persons(self, slice_hrn, slice_record, users, options={}):
 356         """Ensures the users in users list exist and are enabled in LDAP. Adds
 357         person if needed.
 358
 359         Checking that a user exist is based on the user's email. If the user is
 360         still not found in the LDAP, it means that the user comes from another
 361         federated testbed. In this case an account has to be created in LDAP
 362         so as to enable the user to use the testbed, since we trust the testbed
 363         he comes from. This is done by calling AddPerson.
 364
 365         :param slice_hrn: slice name
 366         :param slice_record: record of the slice_hrn
 367         :param users: users is a record list. Records can either be
 368             local records or users records from known and trusted federated
 369             sites.If the user is from another site that iotlab doesn't trust yet,
 370             then Resolve will raise an error before getting to create_sliver.
 371
 372         :type slice_hrn: string
 373         :type slice_record: string
 374         :type users: list
 375
 376         .. seealso:: AddPerson
 377         .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
 378
 379
 380         """
 381         #TODO SA 21/08/12 verify_persons Needs review
 382
 383         logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn  %s  \
 384                     \t slice_record %s\r\n users %s \t  "
 385                      % (slice_hrn, slice_record, users))
 386
 387         users_by_email = {}
 388         #users_dict : dict whose keys can either be the user's hrn or its id.
 389         #Values contains only id and hrn
 390         users_dict = {}
 391
 392         #First create dicts by hrn and id for each user in the user record list:
 393         for info in users:
 394             # if 'slice_record' in info:
 395             #     slice_rec = info['slice_record']
 396                 # if 'user' in slice_rec :
 397                 #     user = slice_rec['user']
 398
 399             if 'email' in info:
 400                 users_by_email[info['email']] = info
 401                 users_dict[info['email']] = info
 402
 403         logger.debug("SLABSLICE.PY \t verify_person  \
 404                         users_dict %s \r\n user_by_email %s \r\n  "
 405                      % (users_dict, users_by_email))
 406
 407         existing_user_ids = []
 408         existing_user_emails = []
 409         existing_users = []
 410         # Check if user is in Iotlab LDAP using its hrn.
 411         # Assuming Iotlab is centralised :  one LDAP for all sites,
 412         # user's record_id unknown from LDAP
 413         # LDAP does not provide users id, therefore we rely on email to find the
 414         # user in LDAP
 415
 416         if users_by_email:
 417             #Construct the list of filters (list of dicts) for GetPersons
 418             filter_user = [users_by_email[email] for email in users_by_email]
 419             #Check user i in LDAP with GetPersons
 420             #Needed because what if the user has been deleted in LDAP but
 421             #is still in SFA?
 422             existing_users = self.driver.testbed_shell.GetPersons(filter_user)
 423             logger.debug(" \r\n IOTLABSLICES.PY \tverify_person  filter_user %s\
 424                        existing_users %s  "
 425                         % (filter_user, existing_users))
 426             #User is in iotlab LDAP
 427             if existing_users:
 428                 for user in existing_users:
 429                     user['login'] = user['uid']
 430                     users_dict[user['email']].update(user)
 431                     existing_user_emails.append(
 432                         users_dict[user['email']]['email'])
 433
 434
 435             # User from another known trusted federated site. Check
 436             # if a iotlab account matching the email has already been created.
 437             else:
 438                 req = 'mail='
 439                 if isinstance(users, list):
 440                     req += users[0]['email']
 441                 else:
 442                     req += users['email']
 443                 ldap_reslt = self.driver.testbed_shell.ldap.LdapSearch(req)
 444
 445                 if ldap_reslt:
 446                     logger.debug(" SLABSLICE.PY \tverify_person users \
 447                                 USER already in Iotlab \t ldap_reslt %s \
 448                                 " % (ldap_reslt))
 449                     existing_users.append(ldap_reslt[1])
 450
 451                 else:
 452                     #User not existing in LDAP
 453                     logger.debug("SLABSLICE.PY \tverify_person users \
 454                                 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
 455                                 ldap_reslt %s " % (users, ldap_reslt))
 456
 457         requested_user_emails = users_by_email.keys()
 458         requested_user_hrns = \
 459             [users_by_email[user]['hrn'] for user in users_by_email]
 460         logger.debug("SLABSLICE.PY \tverify_person  \
 461                        users_by_email  %s " % (users_by_email))
 462
 463         #Check that the user of the slice in the slice record
 464         #matches one of the existing users
 465         try:
 466             if slice_record['reg-researchers'][0] in requested_user_hrns:
 467                 logger.debug(" SLABSLICE  \tverify_person ['PI']\
 468                                 slice_record %s" % (slice_record))
 469
 470         except KeyError:
 471             pass
 472
 473         # users to be added, removed or updated
 474         #One user in one iotlab slice : there should be no need
 475         #to remove/ add any user from/to a slice.
 476         #However a user from SFA which is not registered in Iotlab yet
 477         #should be added to the LDAP.
 478         added_user_emails = set(requested_user_emails).\
 479                                         difference(set(existing_user_emails))
 480
 481
 482         #self.verify_keys(existing_slice_users, updated_users_list, \
 483                                                             #peer, append)
 484
 485         added_persons = []
 486         # add new users
 487         #requested_user_email is in existing_user_emails
 488         if len(added_user_emails) == 0:
 489             slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
 490             logger.debug(" SLABSLICE  \tverify_person QUICK DIRTY %s"
 491                          % (slice_record))
 492
 493         for added_user_email in added_user_emails:
 494             added_user = users_dict[added_user_email]
 495             logger.debug(" IOTLABSLICES \r\n \r\n  \t  verify_person \
 496                          added_user %s" % (added_user))
 497             person = {}
 498             person['peer_person_id'] = None
 499             k_list = ['first_name', 'last_name', 'person_id']
 500             for k in k_list:
 501                 if k in added_user:
 502                     person[k] = added_user[k]
 503
 504             person['pkey'] = added_user['keys'][0]
 505             person['mail'] = added_user['email']
 506             person['email'] = added_user['email']
 507             person['key_ids'] = added_user.get('key_ids', [])
 508
 509             ret = self.driver.testbed_shell.AddPerson(person)
 510             if 'uid' in ret:
 511                 # meaning bool is True and the AddPerson was successful
 512                 person['uid'] = ret['uid']
 513                 slice_record['login'] = person['uid']
 514             else:
 515                 # error message in ret
 516                 logger.debug(" IOTLABSLICES ret message %s" %(ret))
 517
 518             logger.debug(" SLABSLICE \r\n \r\n  \t THE SECOND verify_person\
 519                            person %s" % (person))
 520             #Update slice_Record with the id now known to LDAP
 521
 522
 523             added_persons.append(person)
 524         return added_persons
 525
 526
 527     def verify_keys(self, persons, users, peer, options={}):
 528         """
 529         .. warning:: unused
 530         """
 531         # existing keys
 532         key_ids = []
 533         for person in persons:
 534             key_ids.extend(person['key_ids'])
 535         keylist = self.driver.testbed_shell.GetKeys(key_ids, ['key_id', 'key'])
 536
 537         keydict = {}
 538         for key in keylist:
 539             keydict[key['key']] = key['key_id']
 540         existing_keys = keydict.keys()
 541
 542         persondict = {}
 543         for person in persons:
 544             persondict[person['email']] = person
 545
 546         # add new keys
 547         requested_keys = []
 548         updated_persons = []
 549         users_by_key_string = {}
 550         for user in users:
 551             user_keys = user.get('keys', [])
 552             updated_persons.append(user)
 553             for key_string in user_keys:
 554                 users_by_key_string[key_string] = user
 555                 requested_keys.append(key_string)
 556                 if key_string not in existing_keys:
 557                     key = {'key': key_string, 'key_type': 'ssh'}
 558                     #try:
 559                         ##if peer:
 560                             #person = persondict[user['email']]
 561                             #self.driver.testbed_shell.UnBindObjectFromPeer(
 562                                 # 'person',person['person_id'],
 563                                 # peer['shortname'])
 564                     ret = self.driver.testbed_shell.AddPersonKey(
 565                         user['email'], key)
 566                         #if peer:
 567                             #key_index = user_keys.index(key['key'])
 568                             #remote_key_id = user['key_ids'][key_index]
 569                             #self.driver.testbed_shell.BindObjectToPeer('key', \
 570                                             #key['key_id'], peer['shortname'], \
 571                                             #remote_key_id)
 572
 573                     #finally:
 574                         #if peer:
 575                             #self.driver.testbed_shell.BindObjectToPeer('person', \
 576                                     #person['person_id'], peer['shortname'], \
 577                                     #user['person_id'])
 578
 579         # remove old keys (only if we are not appending)
 580         append = options.get('append', True)
 581         if append is False:
 582             removed_keys = set(existing_keys).difference(requested_keys)
 583             for key in removed_keys:
 584                     #if peer:
 585                         #self.driver.testbed_shell.UnBindObjectFromPeer('key', \
 586                                         #key, peer['shortname'])
 587
 588                 user = users_by_key_string[key]
 589                 self.driver.testbed_shell.DeleteKey(user, key)
 590
 591         return