sfa/iotlab/iotlabslices.py

   1 from sfa.util.xrn import get_authority, urn_to_hrn
   2 from sfa.util.sfalogging import logger
   3
   4
   5 MAXINT =  2L**31-1
   6
   7 class IotlabSlices:
   8
   9     rspec_to_slice_tag = {'max_rate':'net_max_rate'}
  10
  11
  12     def __init__(self, driver):
  13         """
  14         Get the reference to the driver here.
  15         """
  16         self.driver = driver
  17
  18
  19     def get_peer(self, xrn):
  20         """
  21         Find the authority of a resources based on its xrn.
  22         If the authority is Iotlab (local) return None,
  23         Otherwise, look up in the DB if Iotlab is federated with this site
  24         authority and returns its DB record if it is the case,
  25         """
  26         hrn, hrn_type = urn_to_hrn(xrn)
  27         #Does this slice belong to a local site or a peer iotlab site?
  28         peer = None
  29
  30         # get this slice's authority (site)
  31         slice_authority = get_authority(hrn)
  32         #Iotlab stuff
  33         #This slice belongs to the current site
  34         if slice_authority ==  self.driver.iotlab_api.root_auth:
  35             site_authority = slice_authority
  36             return None
  37
  38         site_authority = get_authority(slice_authority).lower()
  39         # get this site's authority (sfa root authority or sub authority)
  40
  41         logger.debug("IOTLABSLICES \ get_peer slice_authority  %s \
  42                     site_authority %s hrn %s" %(slice_authority, \
  43                                         site_authority, hrn))
  44
  45
  46         # check if we are already peered with this site_authority
  47         #if so find the peer record
  48         peers = self.driver.iotlab_api.GetPeers(peer_filter = site_authority)
  49         for peer_record in peers:
  50
  51             if site_authority == peer_record.hrn:
  52                 peer = peer_record
  53         logger.debug(" IOTLABSLICES \tget_peer peer  %s " %(peer))
  54         return peer
  55
  56     def get_sfa_peer(self, xrn):
  57         hrn, hrn_type = urn_to_hrn(xrn)
  58
  59         # return the authority for this hrn or None if we are the authority
  60         sfa_peer = None
  61         slice_authority = get_authority(hrn)
  62         site_authority = get_authority(slice_authority)
  63
  64         if site_authority != self.driver.hrn:
  65             sfa_peer = site_authority
  66
  67         return sfa_peer
  68
  69
  70     def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
  71         """
  72         Compare requested leases with the leases already scheduled/
  73         running in OAR. If necessary, delete and recreate modified leases,
  74         and delete no longer requested ones.
  75
  76         :param sfa_slice: sfa slice record
  77         :param requested_jobs_dict: dictionary of requested leases
  78         :param peer: sfa peer
  79
  80         :type sfa_slice: dict
  81         :type requested_jobs_dict: dict
  82         :type peer:
  83         :return: leases list of dictionary
  84         :rtype: list
  85
  86         """
  87
  88         logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s \
  89                         "%( sfa_slice))
  90         #First get the list of current leases from OAR
  91         leases = self.driver.iotlab_api.GetLeases({'name':sfa_slice['hrn']})
  92         logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
  93                         leases %s "%(requested_jobs_dict, leases ))
  94
  95         current_nodes_reserved_by_start_time = {}
  96         requested_nodes_by_start_time = {}
  97         leases_by_start_time = {}
  98         reschedule_jobs_dict = {}
  99
 100
 101         #Create reduced dictionary with key start_time and value
 102         # the list of nodes
 103         #-for the leases already registered by OAR first
 104         # then for the new leases requested by the user
 105
 106         #Leases already scheduled/running in OAR
 107         for lease in leases :
 108             current_nodes_reserved_by_start_time[lease['t_from']] = \
 109                     lease['reserved_nodes']
 110             leases_by_start_time[lease['t_from']] = lease
 111
 112         #First remove job whose duration is too short
 113         for job in requested_jobs_dict.values():
 114             if job['duration'] < self.driver.iotlab_api.GetLeaseGranularity():
 115                 del requested_jobs_dict[job['start_time']]
 116
 117         #Requested jobs
 118         for start_time in requested_jobs_dict:
 119             requested_nodes_by_start_time[int(start_time)]  = \
 120                     requested_jobs_dict[start_time]['hostname']
 121         #Check if there is any difference between the leases already
 122         #registered in OAR and the requested jobs.
 123         #Difference could be:
 124         #-Lease deleted in the requested jobs
 125         #-Added/removed nodes
 126         #-Newly added lease
 127
 128         logger.debug("IOTLABSLICES verify_slice_leases \
 129                         requested_nodes_by_start_time %s \
 130                         "%(requested_nodes_by_start_time ))
 131         #Find all deleted leases
 132         start_time_list = \
 133             list(set(leases_by_start_time.keys()).\
 134             difference(requested_nodes_by_start_time.keys()))
 135         deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
 136                             for start_time in start_time_list]
 137
 138
 139
 140         #Find added or removed nodes in exisiting leases
 141         for start_time in requested_nodes_by_start_time:
 142             logger.debug("IOTLABSLICES verify_slice_leases  start_time %s \
 143                          "%( start_time))
 144             if start_time in current_nodes_reserved_by_start_time:
 145
 146                 if requested_nodes_by_start_time[start_time] == \
 147                     current_nodes_reserved_by_start_time[start_time]:
 148                     continue
 149
 150                 else:
 151                     update_node_set = \
 152                             set(requested_nodes_by_start_time[start_time])
 153                     added_nodes = \
 154                         update_node_set.difference(\
 155                         current_nodes_reserved_by_start_time[start_time])
 156                     shared_nodes = \
 157                         update_node_set.intersection(\
 158                         current_nodes_reserved_by_start_time[start_time])
 159                     old_nodes_set = \
 160                         set(\
 161                         current_nodes_reserved_by_start_time[start_time])
 162                     removed_nodes = \
 163                         old_nodes_set.difference(\
 164                         requested_nodes_by_start_time[start_time])
 165                     logger.debug("IOTLABSLICES verify_slice_leases \
 166                         shared_nodes %s  added_nodes %s removed_nodes %s"\
 167                         %(shared_nodes, added_nodes,removed_nodes ))
 168                     #If the lease is modified, delete it before
 169                     #creating it again.
 170                     #Add the deleted lease job id in the list
 171                     #WARNING :rescheduling does not work if there is already
 172                     # 2 running/scheduled jobs because deleting a job
 173                     #takes time SA 18/10/2012
 174                     if added_nodes or removed_nodes:
 175                         deleted_leases.append(\
 176                             leases_by_start_time[start_time]['lease_id'])
 177                         #Reschedule the job
 178                         if added_nodes or shared_nodes:
 179                             reschedule_jobs_dict[str(start_time)] = \
 180                                         requested_jobs_dict[str(start_time)]
 181
 182             else:
 183                     #New lease
 184
 185                 job = requested_jobs_dict[str(start_time)]
 186                 logger.debug("IOTLABSLICES \
 187                 NEWLEASE slice %s  job %s"\
 188                 %(sfa_slice, job))
 189                 self.driver.iotlab_api.AddLeases(job['hostname'], \
 190                         sfa_slice, int(job['start_time']), \
 191                         int(job['duration']))
 192
 193         #Deleted leases are the ones with lease id not declared in the Rspec
 194         if deleted_leases:
 195             self.driver.iotlab_api.DeleteLeases(deleted_leases, sfa_slice['hrn'])
 196             logger.debug("IOTLABSLICES \
 197                     verify_slice_leases slice %s deleted_leases %s"\
 198                     %(sfa_slice, deleted_leases))
 199
 200
 201         if reschedule_jobs_dict :
 202             for start_time in  reschedule_jobs_dict:
 203                 job = reschedule_jobs_dict[start_time]
 204                 self.driver.iotlab_api.AddLeases(job['hostname'], \
 205                     sfa_slice, int(job['start_time']), \
 206                     int(job['duration']))
 207         return leases
 208
 209     def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
 210         current_slivers = []
 211         deleted_nodes = []
 212
 213         if 'node_ids' in sfa_slice:
 214             nodes = self.driver.iotlab_api.GetNodes(sfa_slice['list_node_ids'], \
 215                 ['hostname'])
 216             current_slivers = [node['hostname'] for node in nodes]
 217
 218             # remove nodes not in rspec
 219             deleted_nodes = list(set(current_slivers).\
 220                                                 difference(requested_slivers))
 221             # add nodes from rspec
 222             #added_nodes = list(set(requested_slivers).\
 223                                         #difference(current_slivers))
 224
 225
 226             logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
 227                                          \r\n \r\n deleted_nodes %s"\
 228                                         %(sfa_slice, deleted_nodes))
 229
 230             if deleted_nodes:
 231                 #Delete the entire experience
 232                 self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice)
 233                 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
 234                                                                 #deleted_nodes)
 235             return nodes
 236
 237
 238
 239     def free_egre_key(self):
 240         used = set()
 241         for tag in self.driver.iotlab_api.GetSliceTags({'tagname': 'egre_key'}):
 242             used.add(int(tag['value']))
 243
 244         for i in range(1, 256):
 245             if i not in used:
 246                 key = i
 247                 break
 248         else:
 249             raise KeyError("No more EGRE keys available")
 250
 251         return str(key)
 252
 253
 254
 255
 256
 257
 258     def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
 259
 260         #login_base = slice_hrn.split(".")[0]
 261         slicename = slice_hrn
 262         slices_list = self.driver.iotlab_api.GetSlices(slice_filter = slicename, \
 263                                             slice_filter_type = 'slice_hrn')
 264         sfa_slice = None
 265         if slices_list:
 266             for sl in slices_list:
 267
 268                 logger.debug("SLABSLICE \tverify_slice slicename %s \
 269                                     slices_list %s sl %s \ slice_record %s"\
 270                                     %(slicename, slices_list,sl, \
 271                                     slice_record))
 272                 sfa_slice = sl
 273                 sfa_slice.update(slice_record)
 274
 275         else:
 276             #Search for user in ldap based on email SA 14/11/12
 277             ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\
 278                                                     slice_record['user'])
 279             logger.debug(" IOTLABSLICES \tverify_slice Oups \
 280                         slice_record %s sfa_peer %s ldap_user %s"\
 281                         %(slice_record, sfa_peer, ldap_user ))
 282             #User already registered in ldap, meaning user should be in SFA db
 283             #and hrn = sfa_auth+ uid
 284             sfa_slice = {'hrn': slicename,
 285                      #'url': slice_record.get('url', slice_hrn),
 286                      #'description': slice_record.get('description', slice_hrn)
 287                      'node_list' : [],
 288                      'authority' : slice_record['authority'],
 289                      'gid':slice_record['gid'],
 290                      #'record_id_user' : user.record_id,
 291                      'slice_id' : slice_record['record_id'],
 292                      'reg-researchers':slice_record['reg-researchers'],
 293                      #'record_id_slice': slice_record['record_id'],
 294                      'peer_authority':str(sfa_peer)
 295
 296                      }
 297             if ldap_user :
 298                 hrn = self.driver.iotlab_api.root_auth +'.'+ ldap_user['uid']
 299
 300                 user = self.driver.get_user_record(hrn)
 301
 302                 logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" \
 303                                                             %(hrn, user))
 304                 #sfa_slice = {'slice_hrn': slicename,
 305                      ##'url': slice_record.get('url', slice_hrn),
 306                      ##'description': slice_record.get('description', slice_hrn)
 307                      #'node_list' : [],
 308                      #'authority' : slice_record['authority'],
 309                      #'gid':slice_record['gid'],
 310                      ##'record_id_user' : user.record_id,
 311                      #'slice_id' : slice_record['record_id'],
 312                      #'reg-researchers':slice_record['reg-researchers'],
 313                      ##'record_id_slice': slice_record['record_id'],
 314                      #'peer_authority':str(peer.hrn)
 315
 316                      #}
 317                      # add the slice
 318                 if sfa_slice :
 319                     self.driver.iotlab_api.AddSlice(sfa_slice, user)
 320
 321                 if peer:
 322                     sfa_slice['slice_id'] = slice_record['record_id']
 323
 324             #slice['slice_id'] = self.driver.iotlab_api.AddSlice(slice)
 325             logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
 326             #slice['node_ids']=[]
 327             #slice['person_ids'] = []
 328             #if peer:
 329                 #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None)
 330             # mark this slice as an sfa peer record
 331             #if sfa_peer:
 332                 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
 333                              #'peer_authority': sfa_peer, 'pointer': \
 334                                                     #slice['slice_id']}
 335                 #self.registry.register_peer_object(self.credential, peer_dict)
 336
 337
 338
 339         return sfa_slice
 340
 341
 342     def verify_persons(self, slice_hrn, slice_record, users,  peer, sfa_peer, \
 343                                                                 options={}):
 344         """
 345         users is a record list. Records can either be local records
 346         or users records from known and trusted federated sites.
 347         If the user is from another site that iotlab doesn't trust yet,
 348         then Resolve will raise an error before getting to create_sliver.
 349         """
 350         #TODO SA 21/08/12 verify_persons Needs review
 351
 352         logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn  %s  \
 353         \t slice_record %s\r\n users %s \t peer %s "\
 354         %( slice_hrn, slice_record, users,  peer))
 355         users_by_id = {}
 356         #users_by_hrn = {}
 357         users_by_email = {}
 358         #users_dict : dict whose keys can either be the user's hrn or its id.
 359         #Values contains only id and hrn
 360         users_dict = {}
 361
 362         #First create dicts by hrn and id for each user in the user record list:
 363         for info in users:
 364
 365             if 'slice_record' in info :
 366                 slice_rec = info['slice_record']
 367                 user = slice_rec['user']
 368
 369             if 'email' in user:
 370                 users_by_email[user['email']] = user
 371                 users_dict[user['email']] = user
 372
 373             #if 'hrn' in user:
 374                 #users_by_hrn[user['hrn']] = user
 375                 #users_dict[user['hrn']] = user
 376
 377         logger.debug( "SLABSLICE.PY \t verify_person  \
 378                         users_dict %s \r\n user_by_email %s \r\n \
 379                         \tusers_by_id %s " \
 380                         %(users_dict,users_by_email, users_by_id))
 381
 382         existing_user_ids = []
 383         #existing_user_hrns = []
 384         existing_user_emails = []
 385         existing_users = []
 386         # Check if user is in Iotlab LDAP using its hrn.
 387         # Assuming Iotlab is centralised :  one LDAP for all sites,
 388         # user'as record_id unknown from LDAP
 389         # LDAP does not provide users id, therefore we rely on hrns containing
 390         # the login of the user.
 391         # If the hrn is not a iotlab hrn, the user may not be in LDAP.
 392
 393         if users_by_email :
 394             #Construct the list of filters (list of dicts) for GetPersons
 395             filter_user = []
 396             for email in users_by_email :
 397                 filter_user.append (users_by_email[email])
 398             #Check user's in LDAP with GetPersons
 399             #Needed because what if the user has been deleted in LDAP but
 400             #is still in SFA?
 401             existing_users = self.driver.iotlab_api.GetPersons(filter_user)
 402             logger.debug(" \r\n SLABSLICE.PY \tverify_person  filter_user \
 403                                                 %s existing_users %s " \
 404                                                 %(filter_user, existing_users))
 405             #User's in iotlab LDAP
 406             if existing_users:
 407                 for user in existing_users :
 408                     users_dict[user['email']].update(user)
 409                     existing_user_emails.append(\
 410                                         users_dict[user['email']]['email'])
 411
 412
 413             # User from another known trusted federated site. Check
 414             # if a iotlab account matching the email has already been created.
 415             else:
 416                 req = 'mail='
 417                 if isinstance(users, list):
 418
 419                     req += users[0]['email']
 420                 else:
 421                     req += users['email']
 422
 423                 ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req)
 424
 425                 if ldap_reslt:
 426                     logger.debug(" SLABSLICE.PY \tverify_person users \
 427                                 USER already in Iotlab \t ldap_reslt %s \
 428                                 "%( ldap_reslt))
 429                     existing_users.append(ldap_reslt[1])
 430
 431                 else:
 432                     #User not existing in LDAP
 433                     #TODO SA 21/08/12 raise smthg to add user or add it auto ?
 434                     #new_record = {}
 435                     #new_record['pkey'] = users[0]['keys'][0]
 436                     #new_record['mail'] = users[0]['email']
 437
 438                     logger.debug(" SLABSLICE.PY \tverify_person users \
 439                                 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
 440                                 ldap_reslt %s "  %(users, ldap_reslt))
 441
 442         requested_user_emails = users_by_email.keys()
 443         requested_user_hrns = \
 444                         [users_by_email[user]['hrn'] for user in users_by_email]
 445         logger.debug("SLABSLICE.PY \tverify_person  \
 446                        users_by_email  %s " %( users_by_email))
 447         #logger.debug("SLABSLICE.PY \tverify_person  \
 448                         #user_by_hrn %s " %( users_by_hrn))
 449
 450
 451         #Check that the user of the slice in the slice record
 452         #matches one of the existing users
 453         try:
 454             if slice_record['PI'][0] in requested_user_hrns:
 455             #if slice_record['record_id_user'] in requested_user_ids and \
 456                                 #slice_record['PI'][0] in requested_user_hrns:
 457                 logger.debug(" SLABSLICE  \tverify_person ['PI']\
 458                                             slice_record %s" %(slice_record))
 459
 460         except KeyError:
 461             pass
 462
 463
 464         # users to be added, removed or updated
 465         #One user in one iotlab slice : there should be no need
 466         #to remove/ add any user from/to a slice.
 467         #However a user from SFA which is not registered in Iotlab yet
 468         #should be added to the LDAP.
 469         added_user_emails = set(requested_user_emails).\
 470                                         difference(set(existing_user_emails))
 471
 472
 473         #self.verify_keys(existing_slice_users, updated_users_list, \
 474                                                             #peer, append)
 475
 476         added_persons = []
 477         # add new users
 478
 479         #requested_user_email is in existing_user_emails
 480         if len(added_user_emails) == 0:
 481
 482             slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
 483             logger.debug(" SLABSLICE  \tverify_person QUICK DIRTY %s" \
 484                         %(slice_record))
 485
 486
 487         for added_user_email in added_user_emails:
 488             #hrn, type = urn_to_hrn(added_user['urn'])
 489             added_user = users_dict[added_user_email]
 490             logger.debug(" SLABSLICE \r\n \r\n  \t THE SECOND verify_person \
 491                                          added_user %s" %(added_user))
 492             person = {}
 493             person['peer_person_id'] =  None
 494             k_list  = ['first_name', 'last_name','person_id']
 495             for k in k_list:
 496                 if k in added_user:
 497                     person[k] = added_user[k]
 498
 499             person['pkey'] = added_user['keys'][0]
 500             person['mail'] = added_user['email']
 501             person['email'] = added_user['email']
 502             person['key_ids'] =  added_user.get('key_ids', [])
 503             #person['urn'] =   added_user['urn']
 504
 505             #person['person_id'] = self.driver.iotlab_api.AddPerson(person)
 506             ret = self.driver.iotlab_api.AddPerson(person)
 507             if type(ret) == int :
 508                 person['uid'] = ret
 509
 510             logger.debug(" SLABSLICE \r\n \r\n  \t THE SECOND verify_person\
 511                                              personne  %s" %(person))
 512             #Update slice_Record with the id now known to LDAP
 513             slice_record['login'] = person['uid']
 514
 515             added_persons.append(person)
 516
 517
 518         return added_persons
 519
 520     #Unused
 521     def verify_keys(self, persons, users, peer, options={}):
 522         # existing keys
 523         key_ids = []
 524         for person in persons:
 525             key_ids.extend(person['key_ids'])
 526         keylist = self.driver.iotlab_api.GetKeys(key_ids, ['key_id', 'key'])
 527
 528         keydict = {}
 529         for key in keylist:
 530             keydict[key['key']] = key['key_id']
 531         existing_keys = keydict.keys()
 532
 533         persondict = {}
 534         for person in persons:
 535             persondict[person['email']] = person
 536
 537         # add new keys
 538         requested_keys = []
 539         updated_persons = []
 540         users_by_key_string = {}
 541         for user in users:
 542             user_keys = user.get('keys', [])
 543             updated_persons.append(user)
 544             for key_string in user_keys:
 545                 users_by_key_string[key_string] = user
 546                 requested_keys.append(key_string)
 547                 if key_string not in existing_keys:
 548                     key = {'key': key_string, 'key_type': 'ssh'}
 549                     #try:
 550                         ##if peer:
 551                             #person = persondict[user['email']]
 552                             #self.driver.iotlab_api.UnBindObjectFromPeer('person',
 553                                         #person['person_id'], peer['shortname'])
 554                     ret = self.driver.iotlab_api.AddPersonKey(\
 555                                                             user['email'], key)
 556                         #if peer:
 557                             #key_index = user_keys.index(key['key'])
 558                             #remote_key_id = user['key_ids'][key_index]
 559                             #self.driver.iotlab_api.BindObjectToPeer('key', \
 560                                             #key['key_id'], peer['shortname'], \
 561                                             #remote_key_id)
 562
 563                     #finally:
 564                         #if peer:
 565                             #self.driver.iotlab_api.BindObjectToPeer('person', \
 566                                     #person['person_id'], peer['shortname'], \
 567                                     #user['person_id'])
 568
 569         # remove old keys (only if we are not appending)
 570         append = options.get('append', True)
 571         if append == False:
 572             removed_keys = set(existing_keys).difference(requested_keys)
 573             for key in removed_keys:
 574                     #if peer:
 575                         #self.driver.iotlab_api.UnBindObjectFromPeer('key', \
 576                                         #key, peer['shortname'])
 577
 578                 user = users_by_key_string[key]
 579                 self.driver.iotlab_api.DeleteKey(user, key)
 580
 581         return