sfa/iotlab/iotlabslices.py

   1 """
   2 This file defines the IotlabSlices class by which all the slice checkings
   3 upon lease creation are done.
   4 """
   5 from sfa.util.xrn import get_authority, urn_to_hrn, hrn_to_urn
   6 from sfa.util.sfalogging import logger
   7
   8 MAXINT = 2L**31-1
   9
  10
  11 class IotlabSlices:
  12     """
  13     This class is responsible for checking the slice when creating a
  14     lease or a sliver. Those checks include verifying that the user is valid,
  15     that the slice is known from the testbed or from our peers, that the list
  16     of nodes involved has not changed (in this case the lease is modified
  17     accordingly).
  18     """
  19     rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
  20
  21     def __init__(self, driver):
  22         """
  23         Get the reference to the driver here.
  24         """
  25         self.driver = driver
  26
  27     def get_peer(self, xrn):
  28         """
  29         Finds the authority of a resource based on its xrn.
  30         If the authority is Iotlab (local) return None,
  31         Otherwise, look up in the DB if Iotlab is federated with this site
  32         authority and returns its DB record if it is the case.
  33
  34         :param xrn: resource's xrn
  35         :type xrn: string
  36         :returns: peer record
  37         :rtype: dict
  38
  39         """
  40         hrn, hrn_type = urn_to_hrn(xrn)
  41         #Does this slice belong to a local site or a peer iotlab site?
  42         peer = None
  43
  44         # get this slice's authority (site)
  45         slice_authority = get_authority(hrn)
  46         #Iotlab stuff
  47         #This slice belongs to the current site
  48         if slice_authority == self.driver.testbed_shell.root_auth:
  49             site_authority = slice_authority
  50             return None
  51
  52         site_authority = get_authority(slice_authority).lower()
  53         # get this site's authority (sfa root authority or sub authority)
  54
  55         logger.debug("IOTLABSLICES \t get_peer slice_authority  %s \
  56                     site_authority %s hrn %s"
  57                      % (slice_authority, site_authority, hrn))
  58
  59         # check if we are already peered with this site_authority
  60         #if so find the peer record
  61         peers = self.driver.GetPeers(peer_filter=site_authority)
  62         for peer_record in peers:
  63             if site_authority == peer_record.hrn:
  64                 peer = peer_record
  65         logger.debug(" IOTLABSLICES \tget_peer peer  %s " % (peer))
  66         return peer
  67
  68     def get_sfa_peer(self, xrn):
  69         """Returns the authority name for the xrn or None if the local site
  70         is the authority.
  71
  72         :param xrn: the xrn of the resource we are looking the authority for.
  73         :type xrn: string
  74         :returns: the resources's authority name.
  75         :rtype: string
  76
  77         """
  78         hrn, hrn_type = urn_to_hrn(xrn)
  79
  80         # return the authority for this hrn or None if we are the authority
  81         sfa_peer = None
  82         slice_authority = get_authority(hrn)
  83         site_authority = get_authority(slice_authority)
  84
  85         if site_authority != self.driver.hrn:
  86             sfa_peer = site_authority
  87
  88         return sfa_peer
  89
  90     def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
  91         """
  92         Compare requested leases with the leases already scheduled/
  93         running in OAR. If necessary, delete and recreate modified leases,
  94         and delete no longer requested ones.
  95
  96         :param sfa_slice: sfa slice record
  97         :param requested_jobs_dict: dictionary of requested leases
  98         :param peer: sfa peer record
  99
 100         :type sfa_slice: dict
 101         :type requested_jobs_dict: dict
 102         :type peer: dict
 103         :returns: leases list of dictionary
 104         :rtype: list
 105
 106         """
 107
 108         logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
 109                      % (sfa_slice))
 110         #First get the list of current leases from OAR
 111         leases = self.driver.GetLeases({'slice_hrn': sfa_slice['hrn']})
 112         logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
 113                         leases %s " % (requested_jobs_dict, leases))
 114
 115         current_nodes_reserved_by_start_time = {}
 116         requested_nodes_by_start_time = {}
 117         leases_by_start_time = {}
 118         reschedule_jobs_dict = {}
 119
 120         #Create reduced dictionary with key start_time and value
 121         # the list of nodes
 122         #-for the leases already registered by OAR first
 123         # then for the new leases requested by the user
 124
 125         #Leases already scheduled/running in OAR
 126         for lease in leases:
 127             current_nodes_reserved_by_start_time[lease['t_from']] = \
 128                     lease['reserved_nodes']
 129             leases_by_start_time[lease['t_from']] = lease
 130
 131         #First remove job whose duration is too short
 132         for job in requested_jobs_dict.values():
 133             job['duration'] = \
 134                 str(int(job['duration']) \
 135                 * self.driver.testbed_shell.GetLeaseGranularity())
 136             if job['duration'] < \
 137                     self.driver.testbed_shell.GetLeaseGranularity():
 138                 del requested_jobs_dict[job['start_time']]
 139
 140         #Requested jobs
 141         for start_time in requested_jobs_dict:
 142             requested_nodes_by_start_time[int(start_time)] = \
 143                 requested_jobs_dict[start_time]['hostname']
 144         #Check if there is any difference between the leases already
 145         #registered in OAR and the requested jobs.
 146         #Difference could be:
 147         #-Lease deleted in the requested jobs
 148         #-Added/removed nodes
 149         #-Newly added lease
 150
 151         logger.debug("IOTLABSLICES verify_slice_leases \
 152                         requested_nodes_by_start_time %s \
 153                         "% (requested_nodes_by_start_time))
 154         #Find all deleted leases
 155         start_time_list = \
 156             list(set(leases_by_start_time.keys()).\
 157             difference(requested_nodes_by_start_time.keys()))
 158         deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
 159                             for start_time in start_time_list]
 160
 161
 162         #Find added or removed nodes in exisiting leases
 163         for start_time in requested_nodes_by_start_time:
 164             logger.debug("IOTLABSLICES verify_slice_leases  start_time %s \
 165                          "%( start_time))
 166             if start_time in current_nodes_reserved_by_start_time:
 167
 168                 # JORDAN : if we request the same nodes: do nothing
 169                 if requested_nodes_by_start_time[start_time] == \
 170                     current_nodes_reserved_by_start_time[start_time]:
 171                     continue
 172
 173                 else:
 174                     update_node_set = \
 175                             set(requested_nodes_by_start_time[start_time])
 176                     added_nodes = \
 177                         update_node_set.difference(\
 178                         current_nodes_reserved_by_start_time[start_time])
 179                     shared_nodes = \
 180                         update_node_set.intersection(\
 181                         current_nodes_reserved_by_start_time[start_time])
 182                     old_nodes_set = \
 183                         set(\
 184                         current_nodes_reserved_by_start_time[start_time])
 185                     removed_nodes = \
 186                         old_nodes_set.difference(\
 187                         requested_nodes_by_start_time[start_time])
 188                     logger.debug("IOTLABSLICES verify_slice_leases \
 189                         shared_nodes %s  added_nodes %s removed_nodes %s"\
 190                         %(shared_nodes, added_nodes,removed_nodes ))
 191                     #If the lease is modified, delete it before
 192                     #creating it again.
 193                     #Add the deleted lease job id in the list
 194                     #WARNING :rescheduling does not work if there is already
 195                     # 2 running/scheduled jobs because deleting a job
 196                     #takes time SA 18/10/2012
 197                     if added_nodes or removed_nodes:
 198                         deleted_leases.append(\
 199                             leases_by_start_time[start_time]['lease_id'])
 200                         #Reschedule the job
 201                         if added_nodes or shared_nodes:
 202                             reschedule_jobs_dict[str(start_time)] = \
 203                                         requested_jobs_dict[str(start_time)]
 204
 205             else:
 206                     #New lease
 207
 208                 job = requested_jobs_dict[str(start_time)]
 209                 logger.debug("IOTLABSLICES \
 210                               NEWLEASE slice %s  job %s"
 211                              % (sfa_slice, job))
 212                 job_id = self.driver.AddLeases(
 213                     job['hostname'],
 214                     sfa_slice, int(job['start_time']),
 215                     int(job['duration']))
 216
 217                 # Removed by jordan
 218                 #if job_id is not None:
 219                 #    new_leases = self.driver.GetLeases(login=
 220                 #        sfa_slice['login'])
 221                 #    for new_lease in new_leases:
 222                 #        leases.append(new_lease)
 223
 224         #Deleted leases are the ones with lease id not declared in the Rspec
 225         if deleted_leases:
 226             self.driver.testbed_shell.DeleteLeases(deleted_leases,
 227                                                 sfa_slice['login'])
 228             #self.driver.testbed_shell.DeleteLeases(deleted_leases,
 229             #                                    sfa_slice['user']['uid'])
 230             logger.debug("IOTLABSLICES \
 231                           verify_slice_leases slice %s deleted_leases %s"
 232                          % (sfa_slice, deleted_leases))
 233
 234         if reschedule_jobs_dict:
 235             for start_time in reschedule_jobs_dict:
 236                 job = reschedule_jobs_dict[start_time]
 237                 self.driver.AddLeases(
 238                     job['hostname'],
 239                     sfa_slice, int(job['start_time']),
 240                     int(job['duration']))
 241
 242         # Added by Jordan: until we find a better solution, always update the list of leases
 243         return self.driver.GetLeases(login= sfa_slice['login'])
 244         #return leases
 245
 246     def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
 247         """Check for wanted and unwanted nodes in the slice.
 248
 249         Removes nodes and associated leases that the user does not want anymore
 250         by deleteing the associated job in OAR (DeleteSliceFromNodes).
 251         Returns the nodes' hostnames that are going to be in the slice.
 252
 253         :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
 254
 255         :param requested_slivers: list of requested nodes' hostnames.
 256         :param peer: unused so far.
 257
 258         :type sfa_slice: dict
 259         :type requested_slivers: list
 260         :type peer: string
 261
 262         :returns: list requested nodes hostnames
 263         :rtype: list
 264
 265         .. warning:: UNUSED SQA 24/07/13
 266         .. seealso:: DeleteSliceFromNodes
 267         .. todo:: check what to do with the peer? Can not remove peer nodes from
 268             slice here. Anyway, in this case, the peer should have gotten the
 269             remove request too.
 270
 271         """
 272         current_slivers = []
 273         deleted_nodes = []
 274
 275         if 'node_ids' in sfa_slice:
 276             nodes = self.driver.testbed_shell.GetNodes(
 277                 sfa_slice['list_node_ids'],
 278                 ['hostname'])
 279             current_slivers = [node['hostname'] for node in nodes]
 280
 281             # remove nodes not in rspec
 282             deleted_nodes = list(set(current_slivers).
 283                                  difference(requested_slivers))
 284
 285             logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
 286                                          \r\n \r\n deleted_nodes %s"
 287                          % (sfa_slice, deleted_nodes))
 288
 289             if deleted_nodes:
 290                 #Delete the entire experience
 291                 self.driver.testbed_shell.DeleteSliceFromNodes(sfa_slice)
 292             return nodes
 293
 294     def verify_slice(self, slice_hrn, slice_record, sfa_peer):
 295         """Ensures slice record exists.
 296
 297         The slice record must exist either in Iotlab or in the other
 298         federated testbed (sfa_peer). If the slice does not belong to Iotlab,
 299         check if the user already exists in LDAP. In this case, adds the slice
 300         to the sfa DB and associates its LDAP user.
 301
 302         :param slice_hrn: slice's name
 303         :param slice_record: sfa record of the slice
 304         :param sfa_peer: name of the peer authority if any.(not Iotlab).
 305
 306         :type slice_hrn: string
 307         :type slice_record: dictionary
 308         :type sfa_peer: string
 309
 310         .. seealso:: AddSlice
 311
 312
 313         """
 314
 315         slicename = slice_hrn
 316         sfa_slice = None
 317
 318         # check if slice belongs to Iotlab
 319         if slicename.startswith("iotlab"):
 320             slices_list = self.driver.GetSlices(slice_filter=slicename,
 321                                                 slice_filter_type='slice_hrn')
 322
 323             if slices_list:
 324                 for sl in slices_list:
 325
 326                     logger.debug("IOTLABSLICES \t verify_slice slicename %s \
 327                                     slices_list %s sl %s \r slice_record %s"
 328                                  % (slicename, slices_list, sl, slice_record))
 329                     sfa_slice = sl
 330                     sfa_slice.update(slice_record)
 331
 332         else:
 333             #Search for user in ldap based on email SA 14/11/12
 334             ldap_user = self.driver.testbed_shell.ldap.LdapFindUser(\
 335                                                     slice_record['user'])
 336             logger.debug(" IOTLABSLICES \tverify_slice Oups \
 337                         slice_record %s sfa_peer %s ldap_user %s"
 338                         % (slice_record, sfa_peer, ldap_user))
 339             #User already registered in ldap, meaning user should be in SFA db
 340             #and hrn = sfa_auth+ uid
 341             sfa_slice = {'hrn': slicename,
 342                          'node_list': [],
 343                          'authority': slice_record['authority'],
 344                          'gid': slice_record['gid'],
 345                          #'slice_id': slice_record['record_id'],
 346                          'reg-researchers': slice_record['reg-researchers'],
 347                          'urn': hrn_to_urn(slicename,'slice'),
 348                          #'peer_authority': str(sfa_peer)
 349                          }
 350
 351             if ldap_user:
 352 #                hrn = self.driver.testbed_shell.root_auth + '.' \
 353 #                                                + ldap_user['uid']
 354                 for hrn in slice_record['reg-researchers']:
 355                     user = self.driver.get_user_record(hrn)
 356                     if user:
 357                         break
 358
 359                 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
 360                              % (hrn, user))
 361
 362                 # add the external slice to the local SFA iotlab DB
 363                 if sfa_slice:
 364                     self.driver.AddSlice(sfa_slice, user)
 365
 366             logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
 367         return sfa_slice
 368
 369
 370     def verify_persons(self, slice_hrn, slice_record, users, options=None):
 371         """Ensures the users in users list exist and are enabled in LDAP. Adds
 372         person if needed (AddPerson).
 373
 374         Checking that a user exist is based on the user's email. If the user is
 375         still not found in the LDAP, it means that the user comes from another
 376         federated testbed. In this case an account has to be created in LDAP
 377         so as to enable the user to use the testbed, since we trust the testbed
 378         he comes from. This is done by calling AddPerson.
 379
 380         :param slice_hrn: slice name
 381         :param slice_record: record of the slice_hrn
 382         :param users: users is a record list. Records can either be
 383             local records or users records from known and trusted federated
 384             sites.If the user is from another site that iotlab doesn't trust
 385             yet, then Resolve will raise an error before getting to allocate.
 386
 387         :type slice_hrn: string
 388         :type slice_record: string
 389         :type users: list
 390
 391         .. seealso:: AddPerson
 392         .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
 393         """
 394         if options is None: options={}
 395         user = slice_record['user']
 396         logger.debug("IOTLABSLICES \tverify_persons \tuser  %s " % user)
 397         person = {
 398                 'peer_person_id': None,
 399                 'mail'      : user['email'],
 400                 'email'     : user['email'],
 401                 'key_ids'   : user.get('key_ids', []),
 402                 'hrn'       : user['hrn'],
 403         }
 404         if 'first_name' in user:
 405             person['first_name'] = user['first_name']
 406         if 'last_name' in user:
 407             person['last_name'] = user['last_name']
 408         if 'person_id' in user:
 409             person['person_id'] = user['person_id']
 410         if user['keys']:
 411             # Only one key is kept for IoTLAB
 412             person['pkey'] = user['keys'][0]
 413         # SFA DB (if user already exist we do nothing)
 414         self.driver.add_person_to_db(person)
 415         # Iot-LAB LDAP (if user already exist we do nothing)
 416         ret = self.driver.AddPerson(person)
 417         # user uid information is only in LDAP
 418         # Be carreful : global scope of dict slice_record in driver
 419         slice_record['login'] = ret['uid']
 420         return person
 421
 422
 423
 424     def verify_keys(self, persons, users, peer, options=None):
 425         """
 426         .. warning:: unused
 427         """
 428         if options is None: options={}
 429         # existing keys
 430         key_ids = []
 431         for person in persons:
 432             key_ids.extend(person['key_ids'])
 433         keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
 434
 435         keydict = {}
 436         for key in keylist:
 437             keydict[key['key']] = key['key_id']
 438         existing_keys = keydict.keys()
 439
 440         persondict = {}
 441         for person in persons:
 442             persondict[person['email']] = person
 443
 444         # add new keys
 445         requested_keys = []
 446         updated_persons = []
 447         users_by_key_string = {}
 448         for user in users:
 449             user_keys = user.get('keys', [])
 450             updated_persons.append(user)
 451             for key_string in user_keys:
 452                 users_by_key_string[key_string] = user
 453                 requested_keys.append(key_string)
 454                 if key_string not in existing_keys:
 455                     key = {'key': key_string, 'key_type': 'ssh'}
 456                     #try:
 457                         ##if peer:
 458                             #person = persondict[user['email']]
 459                             #self.driver.testbed_shell.UnBindObjectFromPeer(
 460                                 # 'person',person['person_id'],
 461                                 # peer['shortname'])
 462                     ret = self.driver.testbed_shell.AddPersonKey(
 463                         user['email'], key)
 464                         #if peer:
 465                             #key_index = user_keys.index(key['key'])
 466                             #remote_key_id = user['key_ids'][key_index]
 467                             #self.driver.testbed_shell.BindObjectToPeer('key', \
 468                                             #key['key_id'], peer['shortname'], \
 469                                             #remote_key_id)
 470
 471         # remove old keys (only if we are not appending)
 472         append = options.get('append', True)
 473         if append is False:
 474             removed_keys = set(existing_keys).difference(requested_keys)
 475             for key in removed_keys:
 476                     #if peer:
 477                         #self.driver.testbed_shell.UnBindObjectFromPeer('key', \
 478                                         #key, peer['shortname'])
 479
 480                 user = users_by_key_string[key]
 481                 self.driver.testbed_shell.DeleteKey(user, key)
 482
 483         return