Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3

[sfa.git] / sfa / iotlab / iotlabdriver.py

diff --git a/sfa/iotlab/iotlabdriver.py b/sfa/iotlab/iotlabdriver.py
index 9c27275..aaeb36b 100644 (file)
--- a/sfa/iotlab/iotlabdriver.py
+++ b/sfa/iotlab/iotlabdriver.py
@@ -135,7 +135,7 @@ class IotlabDriver(Driver):
 
 
 
 
 
 

-    def AddPerson(self, record):
+    def AddPerson(self, record, add_to_ldap = True):

         """
 
         Adds a new account. Any fields specified in records are used,
         """
 
         Adds a new account. Any fields specified in records are used,


@@ -150,6 +150,10 @@ class IotlabDriver(Driver):
         :rtype: dict
 
         """
         :rtype: dict
 
         """

+        if not add_to_ldap:
+            ret = self.__add_person_to_db(record)
+            return ret
+

         ret = self.testbed_shell.ldap.LdapAddUser(record)
 
         if ret['bool'] is True:
         ret = self.testbed_shell.ldap.LdapAddUser(record)
 
         if ret['bool'] is True:


@@ -173,18 +177,19 @@ class IotlabDriver(Driver):
         """
         query = self.api.dbsession().query(RegUser)
         check_if_exists = query.filter_by(email = user_dict['email']).first()
         """
         query = self.api.dbsession().query(RegUser)
         check_if_exists = query.filter_by(email = user_dict['email']).first()

+        logger.debug("LOIC __add_person_to_db %s" % check_if_exists)

         #user doesn't exists
         if not check_if_exists:
             logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
                                             " %(user_dict))
             hrn = user_dict['hrn']
             person_urn = hrn_to_urn(hrn, 'user')
         #user doesn't exists
         if not check_if_exists:
             logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
                                             " %(user_dict))
             hrn = user_dict['hrn']
             person_urn = hrn_to_urn(hrn, 'user')

-            pubkey = user_dict['pkey']

             try:
             try:

+                pubkey = user_dict['pkey']

                 pkey = convert_public_key(pubkey)
             except TypeError:
                 #key not good. create another pkey
                 pkey = convert_public_key(pubkey)
             except TypeError:
                 #key not good. create another pkey

-                logger.warn('__add_person_to_db: unable to convert public \
+                logger.warn('__add_person_to_db: no public key or unable to convert public \

                                     key for %s' %(hrn ))
                 pkey = Keypair(create=True)
 
                                     key for %s' %(hrn ))
                 pkey = Keypair(create=True)
 


@@ -202,7 +207,7 @@ class IotlabDriver(Driver):
             user_record = RegUser(hrn=hrn , pointer= '-1', \
                                     authority=get_authority(hrn), \
                                     email=user_dict['email'], gid = person_gid)
             user_record = RegUser(hrn=hrn , pointer= '-1', \
                                     authority=get_authority(hrn), \
                                     email=user_dict['email'], gid = person_gid)

-            user_record.reg_keys = [RegKey(user_dict['pkey'])]
+            #user_record.reg_keys = [RegKey(user_dict['pkey'])]

             user_record.just_created()
             self.api.dbsession().add (user_record)
             self.api.dbsession().commit()
             user_record.just_created()
             self.api.dbsession().add (user_record)
             self.api.dbsession().commit()


@@ -214,7 +219,7 @@ class IotlabDriver(Driver):
         """
         Get the slice record based on the slice hrn. Fetch the record of the
         user associated with the slice by using joinedload based on the
         """
         Get the slice record based on the slice hrn. Fetch the record of the
         user associated with the slice by using joinedload based on the

-        reg_researcher relationship.
+        reg_researchers relationship.

 
         :param slice_filter: the slice hrn we are looking for
         :type slice_filter: string
 
         :param slice_filter: the slice hrn we are looking for
         :type slice_filter: string


@@ -233,7 +238,7 @@ class IotlabDriver(Driver):
         raw_slicerec = self.api.dbsession().query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
         #raw_slicerec = self.api.dbsession().query(RegRecord).filter_by(hrn = slice_filter).first()
         if raw_slicerec:
         raw_slicerec = self.api.dbsession().query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
         #raw_slicerec = self.api.dbsession().query(RegRecord).filter_by(hrn = slice_filter).first()
         if raw_slicerec:

-            #load_reg_researcher
+            #load_reg_researchers

             #raw_slicerec.reg_researchers
             raw_slicerec = raw_slicerec.__dict__
             logger.debug(" IOTLAB_API \t  _sql_get_slice_info slice_filter %s  \
             #raw_slicerec.reg_researchers
             raw_slicerec = raw_slicerec.__dict__
             logger.debug(" IOTLAB_API \t  _sql_get_slice_info slice_filter %s  \


@@ -305,6 +310,7 @@ class IotlabDriver(Driver):
         .. seealso::_sql_get_slice_info_from_user
         .. seealso:: _sql_get_slice_info
         """
         .. seealso::_sql_get_slice_info_from_user
         .. seealso:: _sql_get_slice_info
         """

+        logger.debug("JORDAN get_slice_records slice_filter=%r slice_filter_type=%r" % (slice_filter, slice_filter_type))

 
         #Get list of slices based on the slice hrn
         if slice_filter_type == 'slice_hrn':
 
         #Get list of slices based on the slice hrn
         if slice_filter_type == 'slice_hrn':


@@ -363,8 +369,10 @@ class IotlabDriver(Driver):
 
         #First try to get information on the slice based on the filter provided
         if slice_filter_type in authorized_filter_types_list:
 
         #First try to get information on the slice based on the filter provided
         if slice_filter_type in authorized_filter_types_list:

+            logger.debug("JORDAN GET SLICES 1")

             fixed_slicerec_dict = self._get_slice_records(slice_filter,
                                                     slice_filter_type)
             fixed_slicerec_dict = self._get_slice_records(slice_filter,
                                                     slice_filter_type)

+            logger.debug("JORDAN GET SLICE RECORDS %r" % fixed_slicerec_dict)

             # if the slice was not found in the sfa db
             if fixed_slicerec_dict is None:
                 return return_slicerec_dictlist
             # if the slice was not found in the sfa db
             if fixed_slicerec_dict is None:
                 return return_slicerec_dictlist


@@ -386,12 +394,14 @@ class IotlabDriver(Driver):
             #If no job is running or no job scheduled
             #return only the slice record
             if leases_list == [] and fixed_slicerec_dict:
             #If no job is running or no job scheduled
             #return only the slice record
             if leases_list == [] and fixed_slicerec_dict:

+                logger.debug("JORDAN CASE 1")

                 return_slicerec_dictlist.append(fixed_slicerec_dict)
 
             # if the jobs running don't belong to the user/slice we are looking
             # for
             leases_hrn = [lease['slice_hrn'] for lease in leases_list]
             if slice_hrn not in leases_hrn:
                 return_slicerec_dictlist.append(fixed_slicerec_dict)
 
             # if the jobs running don't belong to the user/slice we are looking
             # for
             leases_hrn = [lease['slice_hrn'] for lease in leases_list]
             if slice_hrn not in leases_hrn:

+                logger.debug("JORDAN CASE 2")

                 return_slicerec_dictlist.append(fixed_slicerec_dict)
             #If several jobs for one slice , put the slice record into
             # each lease information dict
                 return_slicerec_dictlist.append(fixed_slicerec_dict)
             #If several jobs for one slice , put the slice record into
             # each lease information dict


@@ -436,6 +446,7 @@ class IotlabDriver(Driver):
 
 
         else:
 
 
         else:

+            logger.debug("JORDAN GET SLICES 2")

             #Get all slices from the iotlab sfa database ,
             #put them in dict format
             #query_slice_list = dbsession.query(RegRecord).all()
             #Get all slices from the iotlab sfa database ,
             #put them in dict format
             #query_slice_list = dbsession.query(RegRecord).all()


@@ -704,15 +715,18 @@ class IotlabDriver(Driver):
 
         sfa_record = RegSlice(hrn=slice_record['hrn'],
                               gid=slice_record['gid'],
 
         sfa_record = RegSlice(hrn=slice_record['hrn'],
                               gid=slice_record['gid'],

-                              pointer=slice_record['slice_id'],
+                              #pointer=slice_record['slice_id'],

                               authority=slice_record['authority'])
         logger.debug("IOTLAB_API.PY AddSlice  sfa_record %s user_record %s"
                      % (sfa_record, user_record))
         sfa_record.just_created()
         self.api.dbsession().add(sfa_record)
         self.api.dbsession().commit()
                               authority=slice_record['authority'])
         logger.debug("IOTLAB_API.PY AddSlice  sfa_record %s user_record %s"
                      % (sfa_record, user_record))
         sfa_record.just_created()
         self.api.dbsession().add(sfa_record)
         self.api.dbsession().commit()

-        #Update the reg-researcher dependance table
-        sfa_record.reg_researchers = [user_record]
+        #Update the reg-researchers dependency table
+        if user_record is not None:
+            sfa_record.reg_researchers = [user_record]
+        else:
+            sfa_record.reg_researchers = slice_record['reg-researchers']

         self.api.dbsession().commit()
 
         return
         self.api.dbsession().commit()
 
         return


@@ -1088,7 +1102,7 @@ class IotlabDriver(Driver):
 
 
 
 
 
 

-    def delete(self, slice_urns, options={}):
+    def delete(self, slice_urns, options=None):

         """
         Deletes the lease associated with the slice hrn and the credentials
             if the slice belongs to iotlab. Answer to DeleteSliver.
         """
         Deletes the lease associated with the slice hrn and the credentials
             if the slice belongs to iotlab. Answer to DeleteSliver.


@@ -1106,6 +1120,7 @@ class IotlabDriver(Driver):
         .. note:: creds are unused, and are not used either in the dummy driver
              delete_sliver .
         """
         .. note:: creds are unused, and are not used either in the dummy driver
              delete_sliver .
         """

+        if options is None: options={}

         # collect sliver ids so we can update sliver allocation states after
         # we remove the slivers.
         aggregate = IotlabAggregate(self)
         # collect sliver ids so we can update sliver allocation states after
         # we remove the slivers.
         aggregate = IotlabAggregate(self)


@@ -1398,16 +1413,19 @@ class IotlabDriver(Driver):
             'geni_ad_rspec_versions': ad_rspec_versions}
 
     # first 2 args are None in case of resource discovery
             'geni_ad_rspec_versions': ad_rspec_versions}
 
     # first 2 args are None in case of resource discovery

-    def list_resources (self, version=None, options={}):
+    def list_resources (self, version=None, options=None):
+        if options is None: options={}

         aggregate = IotlabAggregate(self)
         rspec =  aggregate.list_resources(version=version, options=options)
         return rspec
 
         aggregate = IotlabAggregate(self)
         rspec =  aggregate.list_resources(version=version, options=options)
         return rspec
 

-    def describe(self, urns, version, options={}):
+    def describe(self, urns, version, options=None):
+        if options is None: options={}

         aggregate = IotlabAggregate(self)
         return aggregate.describe(urns, version=version, options=options)
 
         aggregate = IotlabAggregate(self)
         return aggregate.describe(urns, version=version, options=options)
 

-    def status (self, urns, options={}):
+    def status (self, urns, options=None):
+        if options is None: options={}

         aggregate = IotlabAggregate(self)
         desc =  aggregate.describe(urns, version='GENI 3')
         status = {'geni_urn': desc['geni_urn'],
         aggregate = IotlabAggregate(self)
         desc =  aggregate.describe(urns, version='GENI 3')
         status = {'geni_urn': desc['geni_urn'],


@@ -1415,7 +1433,8 @@ class IotlabDriver(Driver):
         return status
 
 
         return status
 
 

-    def allocate (self, urn, rspec_string, expiration, options={}):
+    def allocate (self, urn, rspec_string, expiration, options=None):
+        if options is None: options={}

         xrn = Xrn(urn)
         aggregate = IotlabAggregate(self)
 
         xrn = Xrn(urn)
         aggregate = IotlabAggregate(self)
 


@@ -1423,16 +1442,38 @@ class IotlabDriver(Driver):
         peer = slices.get_peer(xrn.get_hrn())
         sfa_peer = slices.get_sfa_peer(xrn.get_hrn())
 
         peer = slices.get_peer(xrn.get_hrn())
         sfa_peer = slices.get_sfa_peer(xrn.get_hrn())
 

+        caller_hrn = options.get('actual_caller_hrn', [])
+        caller_xrn = Xrn(caller_hrn)
+        caller_urn = caller_xrn.get_urn()

 
 

-        slice_record = None
-        users = options.get('geni_users', [])
+        logger.debug("IOTLABDRIVER.PY :: Allocate caller = %s" % (caller_urn))

 
 

+        slice_record = {}
+        users = options.get('geni_users', [])

         sfa_users = options.get('sfa_users', [])
         sfa_users = options.get('sfa_users', [])

+        

         if sfa_users:
         if sfa_users:

-            slice_record = sfa_users[0].get('slice_record', [])
-            slice_record['user'] = {'keys': users[0]['keys'],
-                                    'email': users[0]['email'],
-                                    'hrn': slice_record['reg-researchers'][0]}
+            user = None
+            # Looking for the user who actually called the Allocate function in the list of users of the slice
+            for u in sfa_users:
+                if 'urn' in u and u['urn'] == caller_urn:
+                    user = u
+                    logger.debug("user = %s" % u)
+            # If we find the user in the list we use it, else we take the 1st in the list as before
+            if user:
+                user_hrn = caller_hrn
+            else:
+                user = sfa_users[0]
+                # XXX Always empty ??? no slice_record in the Allocate call
+                #slice_record = sfa_users[0].get('slice_record', [])
+                user_xrn = Xrn(sfa_users[0]['urn'])
+                user_hrn = user_xrn.get_hrn()
+
+            slice_record = user.get('slice_record', {})
+            slice_record['user'] = {'keys': user['keys'],
+                                    'email': user['email'],
+                                    'hrn': user_hrn}
+            slice_record['authority'] = xrn.get_authority_hrn() 

 
         logger.debug("IOTLABDRIVER.PY \t urn %s allocate options  %s "
                      % (urn, options))
 
         logger.debug("IOTLABDRIVER.PY \t urn %s allocate options  %s "
                      % (urn, options))


@@ -1443,28 +1484,59 @@ class IotlabDriver(Driver):
 
         # ensure site record exists
 
 
         # ensure site record exists
 

-        # ensure slice record exists
-
-        current_slice = slices.verify_slice(xrn.hrn, slice_record, sfa_peer)
-        logger.debug("IOTLABDRIVER.PY \t ===============allocate \t\
-                            \r\n \r\n  current_slice %s" % (current_slice))

         # ensure person records exists
 
         # oui c'est degueulasse, le slice_record se retrouve modifie
         # dans la methode avec les infos du user, els infos sont propagees
         # dans verify_slice_leases
         # ensure person records exists
 
         # oui c'est degueulasse, le slice_record se retrouve modifie
         # dans la methode avec les infos du user, els infos sont propagees
         # dans verify_slice_leases

-        persons = slices.verify_persons(xrn.hrn, slice_record, users,
-                                        options=options)
+        logger.debug("IOTLABDRIVER.PY  BEFORE slices.verify_persons")
+        # XXX JORDAN XXX slice_record devrait recevoir le caller_xrn...
+        logger.debug("LOIC users = %r" % users)
+        # XXX LOIC XXX Trying here to bypass the verify_persons function
+        # But it doesn't work, It seems we have to add users in LDAP anyway...
+        for user in users:
+            # XXX LOIC using hrn is a workaround because the function 
+            # Xrn.get_urn returns 'urn:publicid:IDN+onelab:upmc+timur_friedman'
+            # Instead of this     'urn:publicid:IDN+onelab:upmc+user+timur_friedman'
+            user['hrn'] = urn_to_hrn(user['urn'])[0]
+            # XXX LOIC adding the users of the slice to reg-researchers
+            # reg-researchers is used in iotlabslices.py verify_slice in order to add the slice
+            if 'reg-researchers' not in slice_record:
+                slice_record['reg-researchers'] = list()
+            slice_record['reg-researchers'].append(user['hrn'])
+            if caller_hrn == user['hrn']:
+                #hierarchical_user = user['hrn'].split(".")
+                #user['login'] = hierarchical_user[-1]            
+                #slice_record['login'] = user['login']
+                slice_record['user']=user
+
+        # XXX LOIC XXX Need to re-activate this function and understand exactly what is required
+
+        persons = slices.verify_persons(xrn.hrn, slice_record, [slice_record['user']], options=options)
+        logger.debug("IOTLABDRIVER.PY  AFTER slices.verify_persons")
+        logger.debug("LOIC - IOTLABDRIVER.PY - AFTER slices.verify_persons slice_record = %r" % slice_record)

         # ensure slice attributes exists
         # ensure slice attributes exists

-        # slices.verify_slice_attributes(slice, requested_attributes,
-                                    # options=options)
+        # XXX LOIC XXX This method doesn't exist !!!
+        #slices.verify_slice_attributes(slice, requested_attributes, options=options)
+
+        # XXX LOIC !!! Essayons d'inverser : d'abord verify_persons ensuite verify_slice
+        
+        # ensure slice record exists
+        # XXX LOIC !!! verify_slice uniquement pour iotlab - les slices externes ne doivent pas être vérifiés = Ils ne sont pas dans le registry IOTLAB
+        current_slice = None
+        #current_slice = slices.verify_slice(xrn.hrn, slice_record, sfa_peer)
+        #logger.debug("IOTLABDRIVER.PY \t ===============allocate \t\
+        #                    \r\n \r\n  current_slice %s" % (current_slice))

 
         # add/remove slice from nodes
 
         # add/remove slice from nodes

+        # XXX JORDAN ensure requested_xp_dict returns a dict with all new leases

         requested_xp_dict = self._process_requested_xp_dict(rspec)
 
         logger.debug("IOTLABDRIVER.PY \tallocate  requested_xp_dict %s "
                      % (requested_xp_dict))
         request_nodes = rspec.version.get_nodes_with_slivers()
         requested_xp_dict = self._process_requested_xp_dict(rspec)
 
         logger.debug("IOTLABDRIVER.PY \tallocate  requested_xp_dict %s "
                      % (requested_xp_dict))
         request_nodes = rspec.version.get_nodes_with_slivers()

+
+        # JORDAN: nodes_list will contain a list of newly allocated nodes

         nodes_list = []
         for start_time in requested_xp_dict:
             lease = requested_xp_dict[start_time]
         nodes_list = []
         for start_time in requested_xp_dict:
             lease = requested_xp_dict[start_time]


@@ -1479,35 +1551,66 @@ class IotlabDriver(Driver):
         rspec_requested_leases = rspec.version.get_leases()
         leases = slices.verify_slice_leases(slice_record,
                                                 requested_xp_dict, peer)
         rspec_requested_leases = rspec.version.get_leases()
         leases = slices.verify_slice_leases(slice_record,
                                                 requested_xp_dict, peer)

+        # JORDAN: 
+        #   leases = already in slice
+        #   rspec_requested_leases = newly requested

         logger.debug("IOTLABDRIVER.PY \tallocate leases  %s \
                         rspec_requested_leases %s" % (leases,
                         rspec_requested_leases))
         logger.debug("IOTLABDRIVER.PY \tallocate leases  %s \
                         rspec_requested_leases %s" % (leases,
                         rspec_requested_leases))

-         # update sliver allocations
+
+        # XXX LOIC !!! What is in the slice_record?
+        # Where is affected reg_researchers value???
+        logger.debug("LOIC - IOTLABDRIVER.PY - After verify_slice_leases slice_record = %r" %slice_record)
+        # update sliver allocations
+        # JORDAN Here we loop over newly allocated nodes

         for hostname in nodes_list:
         for hostname in nodes_list:

+            logger.debug("FORLOOP JORDAN hostname=%r" % hostname)

             client_id = hostname
             node_urn = xrn_object(self.testbed_shell.root_auth, hostname).urn
             component_id = node_urn
             client_id = hostname
             node_urn = xrn_object(self.testbed_shell.root_auth, hostname).urn
             component_id = node_urn

-            slice_urn = current_slice['reg-urn']
-            for lease in leases:
+            if current_slice is not None:
+                if 'reg-urn' in current_slice:
+                    slice_urn = current_slice['reg-urn']
+                else:
+                    slice_urn = current_slice['urn']
+            else:
+                slice_urn = slice_record['urn']
+            # JORDAN: We loop over leases previously in the slice
+            for lease in leases: # rspec_requested_leases ?????? XXX
+                logger.debug("FOR LEASE LOOP JORDAN lease=%r" % lease)
+                logger.debug("JORDAN hostname=%r lease['reserved_nodes']=%r, bool=%r" % (hostname, lease['reserved_nodes'], hostname in lease['reserved_nodes']))

                 if hostname in lease['reserved_nodes']:
                 if hostname in lease['reserved_nodes']:

+                    logger.debug("JORDAN IF OK")

                     index = lease['reserved_nodes'].index(hostname)
                     index = lease['reserved_nodes'].index(hostname)

+                    logger.debug("JORDAN index=%r" % index)

                     sliver_hrn = '%s.%s-%s' % (self.hrn, lease['lease_id'],
                                    lease['resource_ids'][index] )
                     sliver_hrn = '%s.%s-%s' % (self.hrn, lease['lease_id'],
                                    lease['resource_ids'][index] )

-            sliver_id = Xrn(sliver_hrn, type='sliver').urn
-            record = SliverAllocation(sliver_id=sliver_id, client_id=client_id,
-                                      component_id=component_id,
-                                      slice_urn = slice_urn,
-                                      allocation_state='geni_allocated')
-            record.sync(self.api.dbsession())
-
-        return aggregate.describe([xrn.get_urn()], version=rspec.version)

 
 

-    def provision(self, urns, options={}):
+                    logger.debug("LOIC sliver_hrn=%r" % sliver_hrn)
+                    sliver_id = Xrn(sliver_hrn, type='sliver').urn
+
+                    logger.debug("LOIC sliver_id=%r" % sliver_id)
+                    record = SliverAllocation(sliver_id=sliver_id, client_id=client_id,
+                                              component_id=component_id,
+                                              slice_urn = slice_urn,
+                                              allocation_state='geni_allocated')
+                    record.sync(self.api.dbsession())
+
+        # JORDAN : added describe_options which was not specified at all
+        describe_options = {
+            'geni_slice_urn': urn,
+            'list_leases': 'all',
+        }
+        return aggregate.describe([xrn.get_urn()], version=rspec.version, options=describe_options)
+
+    def provision(self, urns, options=None):
+        if options is None: options={}

         # update users
         slices = IotlabSlices(self)
         aggregate = IotlabAggregate(self)
         slivers = aggregate.get_slivers(urns)
         current_slice = slivers[0]
         # update users
         slices = IotlabSlices(self)
         aggregate = IotlabAggregate(self)
         slivers = aggregate.get_slivers(urns)
         current_slice = slivers[0]

+        logger.debug("Provision current slice: %r" % (current_slice,))

         peer = slices.get_peer(current_slice['hrn'])
         sfa_peer = slices.get_sfa_peer(current_slice['hrn'])
         users = options.get('geni_users', [])
         peer = slices.get_peer(current_slice['hrn'])
         sfa_peer = slices.get_sfa_peer(current_slice['hrn'])
         users = options.get('geni_users', [])


@@ -1522,4 +1625,10 @@ class IotlabDriver(Driver):
         version_manager = VersionManager()
         rspec_version = version_manager.get_version(options[
                                                         'geni_rspec_version'])
         version_manager = VersionManager()
         rspec_version = version_manager.get_version(options[
                                                         'geni_rspec_version'])

-        return self.describe(urns, rspec_version, options=options)
+        # JORDAN : added describe_options instead of options
+        # urns at the begining ???
+        describe_options = {
+            'geni_slice_urn': current_slice['urn'],
+            'list_leases': 'all',
+        }
+        return self.describe(urns, rspec_version, options=describe_options)