sfa/iotlab/iotlabslices.py

   1 """
   2 This file defines the IotlabSlices class by which all the slice checkings
   3 upon lease creation are done.
   4 """
   5 from sfa.util.xrn import get_authority, urn_to_hrn
   6 from sfa.util.sfalogging import logger
   7
   8 MAXINT = 2L**31-1
   9
  10
  11 class IotlabSlices:
  12     """
  13     This class is responsible for checking the slice when creating a
  14     lease or a sliver. Those checks include verifying that the user is valid,
  15     that the slice is known from the testbed or from our peers, that the list
  16     of nodes involved has not changed (in this case the lease is modified
  17     accordingly).
  18     """
  19     rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
  20
  21     def __init__(self, driver):
  22         """
  23         Get the reference to the driver here.
  24         """
  25         self.driver = driver
  26
  27     def get_peer(self, xrn):
  28         """
  29         Finds the authority of a resource based on its xrn.
  30         If the authority is Iotlab (local) return None,
  31         Otherwise, look up in the DB if Iotlab is federated with this site
  32         authority and returns its DB record if it is the case.
  33
  34         :param xrn: resource's xrn
  35         :type xrn: string
  36         :returns: peer record
  37         :rtype: dict
  38
  39         """
  40         hrn, hrn_type = urn_to_hrn(xrn)
  41         #Does this slice belong to a local site or a peer iotlab site?
  42         peer = None
  43
  44         # get this slice's authority (site)
  45         slice_authority = get_authority(hrn)
  46         #Iotlab stuff
  47         #This slice belongs to the current site
  48         if slice_authority == self.driver.testbed_shell.root_auth:
  49             site_authority = slice_authority
  50             return None
  51
  52         site_authority = get_authority(slice_authority).lower()
  53         # get this site's authority (sfa root authority or sub authority)
  54
  55         logger.debug("IOTLABSLICES \t get_peer slice_authority  %s \
  56                     site_authority %s hrn %s"
  57                      % (slice_authority, site_authority, hrn))
  58
  59         # check if we are already peered with this site_authority
  60         #if so find the peer record
  61         peers = self.driver.GetPeers(peer_filter=site_authority)
  62         for peer_record in peers:
  63             if site_authority == peer_record.hrn:
  64                 peer = peer_record
  65         logger.debug(" IOTLABSLICES \tget_peer peer  %s " % (peer))
  66         return peer
  67
  68     def get_sfa_peer(self, xrn):
  69         """Returns the authority name for the xrn or None if the local site
  70         is the authority.
  71
  72         :param xrn: the xrn of the resource we are looking the authority for.
  73         :type xrn: string
  74         :returns: the resources's authority name.
  75         :rtype: string
  76
  77         """
  78         hrn, hrn_type = urn_to_hrn(xrn)
  79
  80         # return the authority for this hrn or None if we are the authority
  81         sfa_peer = None
  82         slice_authority = get_authority(hrn)
  83         site_authority = get_authority(slice_authority)
  84
  85         if site_authority != self.driver.hrn:
  86             sfa_peer = site_authority
  87
  88         return sfa_peer
  89
  90     def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
  91         """
  92         Compare requested leases with the leases already scheduled/
  93         running in OAR. If necessary, delete and recreate modified leases,
  94         and delete no longer requested ones.
  95
  96         :param sfa_slice: sfa slice record
  97         :param requested_jobs_dict: dictionary of requested leases
  98         :param peer: sfa peer record
  99
 100         :type sfa_slice: dict
 101         :type requested_jobs_dict: dict
 102         :type peer: dict
 103         :returns: leases list of dictionary
 104         :rtype: list
 105
 106         """
 107
 108         logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
 109                      % (sfa_slice))
 110         #First get the list of current leases from OAR
 111         leases = self.driver.GetLeases({'slice_hrn': sfa_slice['hrn']})
 112         logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
 113                         leases %s " % (requested_jobs_dict, leases))
 114
 115         current_nodes_reserved_by_start_time = {}
 116         requested_nodes_by_start_time = {}
 117         leases_by_start_time = {}
 118         reschedule_jobs_dict = {}
 119
 120         #Create reduced dictionary with key start_time and value
 121         # the list of nodes
 122         #-for the leases already registered by OAR first
 123         # then for the new leases requested by the user
 124
 125         #Leases already scheduled/running in OAR
 126         for lease in leases:
 127             current_nodes_reserved_by_start_time[lease['t_from']] = \
 128                     lease['reserved_nodes']
 129             leases_by_start_time[lease['t_from']] = lease
 130
 131         #First remove job whose duration is too short
 132         for job in requested_jobs_dict.values():
 133             job['duration'] = \
 134                 str(int(job['duration']) \
 135                 * self.driver.testbed_shell.GetLeaseGranularity())
 136             if job['duration'] < \
 137                     self.driver.testbed_shell.GetLeaseGranularity():
 138                 del requested_jobs_dict[job['start_time']]
 139
 140         #Requested jobs
 141         for start_time in requested_jobs_dict:
 142             requested_nodes_by_start_time[int(start_time)] = \
 143                 requested_jobs_dict[start_time]['hostname']
 144         #Check if there is any difference between the leases already
 145         #registered in OAR and the requested jobs.
 146         #Difference could be:
 147         #-Lease deleted in the requested jobs
 148         #-Added/removed nodes
 149         #-Newly added lease
 150
 151         logger.debug("IOTLABSLICES verify_slice_leases \
 152                         requested_nodes_by_start_time %s \
 153                         "% (requested_nodes_by_start_time))
 154         #Find all deleted leases
 155         start_time_list = \
 156             list(set(leases_by_start_time.keys()).\
 157             difference(requested_nodes_by_start_time.keys()))
 158         deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
 159                             for start_time in start_time_list]
 160
 161
 162         #Find added or removed nodes in exisiting leases
 163         for start_time in requested_nodes_by_start_time:
 164             logger.debug("IOTLABSLICES verify_slice_leases  start_time %s \
 165                          "%( start_time))
 166             if start_time in current_nodes_reserved_by_start_time:
 167
 168                 if requested_nodes_by_start_time[start_time] == \
 169                     current_nodes_reserved_by_start_time[start_time]:
 170                     continue
 171
 172                 else:
 173                     update_node_set = \
 174                             set(requested_nodes_by_start_time[start_time])
 175                     added_nodes = \
 176                         update_node_set.difference(\
 177                         current_nodes_reserved_by_start_time[start_time])
 178                     shared_nodes = \
 179                         update_node_set.intersection(\
 180                         current_nodes_reserved_by_start_time[start_time])
 181                     old_nodes_set = \
 182                         set(\
 183                         current_nodes_reserved_by_start_time[start_time])
 184                     removed_nodes = \
 185                         old_nodes_set.difference(\
 186                         requested_nodes_by_start_time[start_time])
 187                     logger.debug("IOTLABSLICES verify_slice_leases \
 188                         shared_nodes %s  added_nodes %s removed_nodes %s"\
 189                         %(shared_nodes, added_nodes,removed_nodes ))
 190                     #If the lease is modified, delete it before
 191                     #creating it again.
 192                     #Add the deleted lease job id in the list
 193                     #WARNING :rescheduling does not work if there is already
 194                     # 2 running/scheduled jobs because deleting a job
 195                     #takes time SA 18/10/2012
 196                     if added_nodes or removed_nodes:
 197                         deleted_leases.append(\
 198                             leases_by_start_time[start_time]['lease_id'])
 199                         #Reschedule the job
 200                         if added_nodes or shared_nodes:
 201                             reschedule_jobs_dict[str(start_time)] = \
 202                                         requested_jobs_dict[str(start_time)]
 203
 204             else:
 205                     #New lease
 206
 207                 job = requested_jobs_dict[str(start_time)]
 208                 logger.debug("IOTLABSLICES \
 209                               NEWLEASE slice %s  job %s"
 210                              % (sfa_slice, job))
 211                 job_id = self.driver.AddLeases(
 212                     job['hostname'],
 213                     sfa_slice, int(job['start_time']),
 214                     int(job['duration']))
 215                 if job_id is not None:
 216                     new_leases = self.driver.GetLeases(login=
 217                         sfa_slice['login'])
 218                     for new_lease in new_leases:
 219                         leases.append(new_lease)
 220
 221         #Deleted leases are the ones with lease id not declared in the Rspec
 222         if deleted_leases:
 223             self.driver.testbed_shell.DeleteLeases(deleted_leases,
 224                                                 sfa_slice['login'])
 225             #self.driver.testbed_shell.DeleteLeases(deleted_leases,
 226             #                                    sfa_slice['user']['uid'])
 227             logger.debug("IOTLABSLICES \
 228                           verify_slice_leases slice %s deleted_leases %s"
 229                          % (sfa_slice, deleted_leases))
 230
 231         if reschedule_jobs_dict:
 232             for start_time in reschedule_jobs_dict:
 233                 job = reschedule_jobs_dict[start_time]
 234                 self.driver.AddLeases(
 235                     job['hostname'],
 236                     sfa_slice, int(job['start_time']),
 237                     int(job['duration']))
 238         return leases
 239
 240     def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
 241         """Check for wanted and unwanted nodes in the slice.
 242
 243         Removes nodes and associated leases that the user does not want anymore
 244         by deleteing the associated job in OAR (DeleteSliceFromNodes).
 245         Returns the nodes' hostnames that are going to be in the slice.
 246
 247         :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
 248
 249         :param requested_slivers: list of requested nodes' hostnames.
 250         :param peer: unused so far.
 251
 252         :type sfa_slice: dict
 253         :type requested_slivers: list
 254         :type peer: string
 255
 256         :returns: list requested nodes hostnames
 257         :rtype: list
 258
 259         .. warning:: UNUSED SQA 24/07/13
 260         .. seealso:: DeleteSliceFromNodes
 261         .. todo:: check what to do with the peer? Can not remove peer nodes from
 262             slice here. Anyway, in this case, the peer should have gotten the
 263             remove request too.
 264
 265         """
 266         current_slivers = []
 267         deleted_nodes = []
 268
 269         if 'node_ids' in sfa_slice:
 270             nodes = self.driver.testbed_shell.GetNodes(
 271                 sfa_slice['list_node_ids'],
 272                 ['hostname'])
 273             current_slivers = [node['hostname'] for node in nodes]
 274
 275             # remove nodes not in rspec
 276             deleted_nodes = list(set(current_slivers).
 277                                  difference(requested_slivers))
 278
 279             logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
 280                                          \r\n \r\n deleted_nodes %s"
 281                          % (sfa_slice, deleted_nodes))
 282
 283             if deleted_nodes:
 284                 #Delete the entire experience
 285                 self.driver.testbed_shell.DeleteSliceFromNodes(sfa_slice)
 286             return nodes
 287
 288     def verify_slice(self, slice_hrn, slice_record, sfa_peer):
 289         """Ensures slice record exists.
 290
 291         The slice record must exist either in Iotlab or in the other
 292         federated testbed (sfa_peer). If the slice does not belong to Iotlab,
 293         check if the user already exists in LDAP. In this case, adds the slice
 294         to the sfa DB and associates its LDAP user.
 295
 296         :param slice_hrn: slice's name
 297         :param slice_record: sfa record of the slice
 298         :param sfa_peer: name of the peer authority if any.(not Iotlab).
 299
 300         :type slice_hrn: string
 301         :type slice_record: dictionary
 302         :type sfa_peer: string
 303
 304         .. seealso:: AddSlice
 305
 306
 307         """
 308
 309         slicename = slice_hrn
 310         # check if slice belongs to Iotlab
 311         slices_list = self.driver.GetSlices(slice_filter=slicename,
 312                                             slice_filter_type='slice_hrn')
 313
 314         sfa_slice = None
 315
 316         if slices_list:
 317             for sl in slices_list:
 318
 319                 logger.debug("IOTLABSLICES \t verify_slice slicename %s \
 320                                 slices_list %s sl %s \r slice_record %s"
 321                              % (slicename, slices_list, sl, slice_record))
 322                 sfa_slice = sl
 323                 sfa_slice.update(slice_record)
 324
 325         else:
 326             #Search for user in ldap based on email SA 14/11/12
 327             ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
 328                                                     slice_record['user'])
 329             logger.debug(" IOTLABSLICES \tverify_slice Oups \
 330                         slice_record %s sfa_peer %s ldap_user %s"
 331                         % (slice_record, sfa_peer, ldap_user))
 332             #User already registered in ldap, meaning user should be in SFA db
 333             #and hrn = sfa_auth+ uid
 334             sfa_slice = {'hrn': slicename,
 335                          'node_list': [],
 336                          'authority': slice_record['authority'],
 337                          'gid': slice_record['gid'],
 338                          #'slice_id': slice_record['record_id'],
 339                          'reg-researchers': slice_record['reg-researchers'],
 340                          'urn': hrn_to_urn(slicename,'slice'),
 341                          #'peer_authority': str(sfa_peer)
 342                          }
 343
 344             if ldap_user:
 345                 hrn = self.driver.testbed_shell.root_auth + '.' \
 346                                                 + ldap_user['uid']
 347                 user = self.driver.get_user_record(hrn)
 348
 349                 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
 350                              % (hrn, user))
 351
 352                  # add the external slice to the local SFA iotlab DB
 353                 if sfa_slice:
 354                     self.driver.AddSlice(sfa_slice, user)
 355
 356             logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
 357         return sfa_slice
 358
 359
 360     def verify_persons(self, slice_hrn, slice_record, users, options=None):
 361         """Ensures the users in users list exist and are enabled in LDAP. Adds
 362         person if needed (AddPerson).
 363
 364         Checking that a user exist is based on the user's email. If the user is
 365         still not found in the LDAP, it means that the user comes from another
 366         federated testbed. In this case an account has to be created in LDAP
 367         so as to enable the user to use the testbed, since we trust the testbed
 368         he comes from. This is done by calling AddPerson.
 369
 370         :param slice_hrn: slice name
 371         :param slice_record: record of the slice_hrn
 372         :param users: users is a record list. Records can either be
 373             local records or users records from known and trusted federated
 374             sites.If the user is from another site that iotlab doesn't trust
 375             yet, then Resolve will raise an error before getting to allocate.
 376
 377         :type slice_hrn: string
 378         :type slice_record: string
 379         :type users: list
 380
 381         .. seealso:: AddPerson
 382         .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
 383
 384
 385         """
 386         slice_user = slice_record['user']
 387
 388         if options is None: options={}
 389         logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn  %s  \
 390                     \t slice_record %s\r\n users %s \t  "
 391                      % (slice_hrn, slice_record, users))
 392
 393         users_by_email = {}
 394         #users_dict : dict whose keys can either be the user's hrn or its id.
 395         #Values contains only id and hrn
 396         users_dict = {}
 397
 398         #First create dicts by hrn and id for each user in the user record list:
 399         for info in users:
 400             # if 'slice_record' in info:
 401             #     slice_rec = info['slice_record']
 402                 # if 'user' in slice_rec :
 403                 #     user = slice_rec['user']
 404
 405             if 'email' in info:
 406                 users_by_email[info['email']] = info
 407                 users_dict[info['email']] = info
 408
 409         logger.debug("IOTLABSLICES.PY \t verify_person  \
 410                         users_dict %s \r\n user_by_email %s \r\n  "
 411                      % (users_dict, users_by_email))
 412
 413         existing_user_ids = []
 414         existing_user_emails = []
 415         existing_users = []
 416         # Check if user is in Iotlab LDAP using its hrn.
 417         # Assuming Iotlab is centralised :  one LDAP for all sites,
 418         # user's record_id unknown from LDAP
 419         # LDAP does not provide users id, therefore we rely on email to find the
 420         # user in LDAP
 421
 422         if users_by_email:
 423             #Construct the list of filters (list of dicts) for GetPersons
 424             filter_user = [users_by_email[email] for email in users_by_email]
 425             #Check user i in LDAP with GetPersons
 426             #Needed because what if the user has been deleted in LDAP but
 427             #is still in SFA?
 428             # GetPersons -> LdapFindUser -> _process_ldap_info_for_one_user
 429             existing_users = self.driver.testbed_shell.GetPersons(filter_user)
 430             logger.debug(" \r\n IOTLABSLICES.PY \tverify_person  filter_user %s\
 431                        existing_users %s  "
 432                         % (filter_user, existing_users))
 433             #User is in iotlab LDAP
 434             if existing_users:
 435                 for user in existing_users:
 436                     user['login'] = user['uid']
 437                     users_dict[user['email']].update(user)
 438                     existing_user_emails.append(
 439                         users_dict[user['email']]['email'])
 440                 logger.debug("User is in iotlab LDAP slice_record[user] = %s" % slice_user)
 441
 442             # User from another known trusted federated site. Check
 443             # if a iotlab account matching the email has already been created.
 444             else:
 445                 req = 'mail='
 446                 if isinstance(users, list):
 447                     req += users[0]['email']
 448                 else:
 449                     req += users['email']
 450                 ldap_reslt = self.driver.testbed_shell.ldap.LdapSearch(req)
 451                 logger.debug("LdapSearch slice_record[user] = %s" % slice_user)
 452                 if ldap_reslt:
 453                     logger.debug(" IOTLABSLICES.PY \tverify_person users \
 454                                 USER already in Iotlab \t ldap_reslt %s \
 455                                 " % (ldap_reslt))
 456                     existing_users.append(ldap_reslt[1])
 457                     logger.debug("ldap_reslt slice_record[user] = %s" % slice_user)
 458                 else:
 459                     #User not existing in LDAP
 460                     logger.debug("IOTLABSLICES.PY \tverify_person users \
 461                                 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
 462                                 ldap_reslt %s " % (users, ldap_reslt))
 463
 464         requested_user_emails = users_by_email.keys()
 465         # requested_user_hrns = \
 466         #     [users_by_email[user]['hrn'] for user in users_by_email]
 467         # logger.debug("IOTLABSLICES.PY \tverify_person  \
 468         #                users_by_email  %s " % (users_by_email))
 469
 470         # #Check that the user of the slice in the slice record
 471         # #matches one of the existing users
 472         # try:
 473         #     if slice_record['reg-researchers'][0] in requested_user_hrns:
 474         #         logger.debug(" IOTLABSLICES  \tverify_person ['PI']\
 475         #                         slice_record %s" % (slice_record))
 476
 477         # except KeyError:
 478         #     pass
 479
 480         # users to be added, removed or updated
 481         #One user in one iotlab slice : there should be no need
 482         #to remove/ add any user from/to a slice.
 483         #However a user from SFA which is not registered in Iotlab yet
 484         #should be added to the LDAP.
 485         added_user_emails = set(requested_user_emails).\
 486                                         difference(set(existing_user_emails))
 487
 488
 489         #self.verify_keys(existing_slice_users, updated_users_list, \
 490                                                             #peer, append)
 491
 492         added_persons = []
 493         # add new users
 494         #requested_user_email is in existing_user_emails
 495         if len(added_user_emails) == 0:
 496             slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
 497             logger.debug(" IOTLABSLICES  \tverify_person QUICK DIRTY %s"
 498                          % (slice_record))
 499
 500         for added_user_email in added_user_emails:
 501             added_user = users_dict[added_user_email]
 502             logger.debug(" IOTLABSLICES \r\n \r\n  \t  verify_person \
 503                          added_user %s" % (added_user))
 504             person = {}
 505             person['peer_person_id'] = None
 506             k_list = ['first_name', 'last_name', 'person_id']
 507             for k in k_list:
 508                 if k in added_user:
 509                     person[k] = added_user[k]
 510             # bug user without key
 511             if added_user['keys']:
 512                 person['pkey'] = added_user['keys'][0]
 513             person['mail'] = added_user['email']
 514             person['email'] = added_user['email']
 515             person['key_ids'] = added_user.get('key_ids', [])
 516
 517             ret = self.driver.AddPerson(person)
 518             if 'uid' in ret:
 519                 # meaning bool is True and the AddPerson was successful
 520                 person['uid'] = ret['uid']
 521                 slice_record['login'] = person['uid']
 522             else:
 523                 # error message in ret
 524                 logger.debug(" IOTLABSLICES ret message %s" %(ret))
 525
 526             logger.debug(" IOTLABSLICES \r\n \r\n  \t THE SECOND verify_person\
 527                            person %s" % (person))
 528             #Update slice_Record with the id now known to LDAP
 529
 530
 531             added_persons.append(person)
 532         return added_persons
 533
 534
 535     def verify_keys(self, persons, users, peer, options=None):
 536         """
 537         .. warning:: unused
 538         """
 539         if options is None: options={}
 540         # existing keys
 541         key_ids = []
 542         for person in persons:
 543             key_ids.extend(person['key_ids'])
 544         keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
 545
 546         keydict = {}
 547         for key in keylist:
 548             keydict[key['key']] = key['key_id']
 549         existing_keys = keydict.keys()
 550
 551         persondict = {}
 552         for person in persons:
 553             persondict[person['email']] = person
 554
 555         # add new keys
 556         requested_keys = []
 557         updated_persons = []
 558         users_by_key_string = {}
 559         for user in users:
 560             user_keys = user.get('keys', [])
 561             updated_persons.append(user)
 562             for key_string in user_keys:
 563                 users_by_key_string[key_string] = user
 564                 requested_keys.append(key_string)
 565                 if key_string not in existing_keys:
 566                     key = {'key': key_string, 'key_type': 'ssh'}
 567                     #try:
 568                         ##if peer:
 569                             #person = persondict[user['email']]
 570                             #self.driver.testbed_shell.UnBindObjectFromPeer(
 571                                 # 'person',person['person_id'],
 572                                 # peer['shortname'])
 573                     ret = self.driver.testbed_shell.AddPersonKey(
 574                         user['email'], key)
 575                         #if peer:
 576                             #key_index = user_keys.index(key['key'])
 577                             #remote_key_id = user['key_ids'][key_index]
 578                             #self.driver.testbed_shell.BindObjectToPeer('key', \
 579                                             #key['key_id'], peer['shortname'], \
 580                                             #remote_key_id)
 581
 582         # remove old keys (only if we are not appending)
 583         append = options.get('append', True)
 584         if append is False:
 585             removed_keys = set(existing_keys).difference(requested_keys)
 586             for key in removed_keys:
 587                     #if peer:
 588                         #self.driver.testbed_shell.UnBindObjectFromPeer('key', \
 589                                         #key, peer['shortname'])
 590
 591                 user = users_by_key_string[key]
 592                 self.driver.testbed_shell.DeleteKey(user, key)
 593
 594         return