Propagating clast changes.

[sfa.git] / sfa / iotlab / iotlabapi.py

diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py
index bbae92b..4f8bcfe 100644 (file)
--- a/sfa/iotlab/iotlabapi.py
+++ b/sfa/iotlab/iotlabapi.py
@@ -11,7 +11,7 @@ from sfa.util.sfalogging import logger
 from sfa.storage.alchemy import dbsession
 from sqlalchemy.orm import joinedload
 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
 from sfa.storage.alchemy import dbsession
 from sqlalchemy.orm import joinedload
 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey

-from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP
+from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP

 from sfa.iotlab.OARrestapi import OARrestapi
 from sfa.iotlab.LDAPapi import LDAPapi
 
 from sfa.iotlab.OARrestapi import OARrestapi
 from sfa.iotlab.LDAPapi import LDAPapi
 


@@ -26,7 +26,7 @@ from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
 class IotlabTestbedAPI():
     """ Class enabled to use LDAP and OAR api calls. """
 
 class IotlabTestbedAPI():
     """ Class enabled to use LDAP and OAR api calls. """
 

-    _MINIMUM_DURATION = 600
+    _MINIMUM_DURATION = 10  # 10 units of granularity 60 s, 10 mins

 
     def __init__(self, config):
         """Creates an instance of OARrestapi and LDAPapi which will be used to
 
     def __init__(self, config):
         """Creates an instance of OARrestapi and LDAPapi which will be used to


@@ -37,12 +37,12 @@ class IotlabTestbedAPI():
         :param config: configuration object from sfa.util.config
         :type config: Config object
         """
         :param config: configuration object from sfa.util.config
         :type config: Config object
         """

-        self.iotlab_db = IotlabDB(config)
+        self.iotlab_db = TestbedAdditionalSfaDB(config)

         self.oar = OARrestapi()
         self.ldap = LDAPapi()
         self.time_format = "%Y-%m-%d %H:%M:%S"
         self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
         self.oar = OARrestapi()
         self.ldap = LDAPapi()
         self.time_format = "%Y-%m-%d %H:%M:%S"
         self.root_auth = config.SFA_REGISTRY_ROOT_AUTH

-        self.grain = 1  # 10 mins lease minimum, 1 sec granularity
+        self.grain = 60  # 10 mins lease minimum, 60 sec granularity

         #import logging, logging.handlers
         #from sfa.util.sfalogging import _SfaLogger
         #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \
         #import logging, logging.handlers
         #from sfa.util.sfalogging import _SfaLogger
         #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \


@@ -50,7 +50,7 @@ class IotlabTestbedAPI():
         return
 
     @staticmethod
         return
 
     @staticmethod

-    def GetMinExperimentDurationInSec():
+    def GetMinExperimentDurationInGranularity():

         """ Returns the minimum allowed duration for an experiment on the
         testbed. In seconds.
 
         """ Returns the minimum allowed duration for an experiment on the
         testbed. In seconds.
 


@@ -69,8 +69,7 @@ class IotlabTestbedAPI():
 
         existing_records = {}
         existing_hrns_by_types = {}
 
         existing_records = {}
         existing_hrns_by_types = {}

-        logger.debug("IOTLAB_API \tGetPeers peer_filter %s, \
-                    " %(peer_filter))
+        logger.debug("IOTLAB_API \tGetPeers peer_filter %s " % (peer_filter))

         all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
 
         for record in all_records:
         all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
 
         for record in all_records:


@@ -80,20 +79,20 @@ class IotlabTestbedAPI():
             else:
                 existing_hrns_by_types[record.type].append(record.hrn)
 
             else:
                 existing_hrns_by_types[record.type].append(record.hrn)
 

-
-        logger.debug("IOTLAB_API \tGetPeer\texisting_hrns_by_types %s "\
-                                             %( existing_hrns_by_types))
+        logger.debug("IOTLAB_API \tGetPeer\texisting_hrns_by_types %s "
+                     % (existing_hrns_by_types))

         records_list = []
 
         try:
             if peer_filter:
         records_list = []
 
         try:
             if peer_filter:

-                records_list.append(existing_records[(peer_filter,'authority')])
+                records_list.append(existing_records[(peer_filter,
+                                                     'authority')])

             else:
                 for hrn in existing_hrns_by_types['authority']:
             else:
                 for hrn in existing_hrns_by_types['authority']:

-                    records_list.append(existing_records[(hrn,'authority')])
+                    records_list.append(existing_records[(hrn, 'authority')])

 
 

-            logger.debug("IOTLAB_API \tGetPeer \trecords_list  %s " \
-                                            %(records_list))
+            logger.debug("IOTLAB_API \tGetPeer \trecords_list  %s "
+                         % (records_list))

 
         except KeyError:
             pass
 
         except KeyError:
             pass


@@ -112,10 +111,11 @@ class IotlabTestbedAPI():
         If a filter is specified, looks for the user whose properties match
         the filter, otherwise returns the whole enabled users'list.
 
         If a filter is specified, looks for the user whose properties match
         the filter, otherwise returns the whole enabled users'list.
 

-        :param person_filter: Must be a list of dictionnaries
-            with users properties when not set to None.
-        :param person_filter: list of dict
-        :returns:Returns a list of users whose accounts are enabled
+        :param person_filter: Must be a list of dictionnaries with users
+            properties when not set to None.
+        :type person_filter: list of dict
+
+        :returns: Returns a list of users whose accounts are enabled

             found in ldap.
         :rtype: list of dicts
 
             found in ldap.
         :rtype: list of dicts
 


@@ -244,6 +244,7 @@ class IotlabTestbedAPI():
 
         :returns: dicionary with nodes' hostnames belonging to the job.
         :rtype: dict
 
         :returns: dicionary with nodes' hostnames belonging to the job.
         :rtype: dict

+        .. warning: Unused. SA 16/10/13

         """
 
         req = "GET_jobs_id_resources"
         """
 
         req = "GET_jobs_id_resources"


@@ -434,8 +435,13 @@ class IotlabTestbedAPI():
 
     def GetSites(self, site_filter_name_list=None, return_fields_list=None):
         """Returns the list of Iotlab's sites with the associated nodes and
 
     def GetSites(self, site_filter_name_list=None, return_fields_list=None):
         """Returns the list of Iotlab's sites with the associated nodes and

-        their properties as dictionaries.
+        the sites' properties as dictionaries.

 
 

+        Site properties:
+        ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids',
+        'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids',
+        'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created',
+        'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name']

         Uses the OAR request GET_sites to find the Iotlab's sites.
 
         :param site_filter_name_list: used to specify specific sites
         Uses the OAR request GET_sites to find the Iotlab's sites.
 
         :param site_filter_name_list: used to specify specific sites


@@ -443,7 +449,7 @@ class IotlabTestbedAPI():
         :type site_filter_name_list: list
         :type return_fields_list: list
 
         :type site_filter_name_list: list
         :type return_fields_list: list
 

-        .. warning:: unused
+

         """
         site_dict = self.oar.parser.SendRequest("GET_sites")
         #site_dict : dict where the key is the sit ename
         """
         site_dict = self.oar.parser.SendRequest("GET_sites")
         #site_dict : dict where the key is the sit ename


@@ -520,7 +526,7 @@ class IotlabTestbedAPI():
         Add a federated user straight to db when the user issues a lease
         request with iotlab nodes and that he has not registered with iotlab
         yet (that is he does not have a LDAP entry yet).
         Add a federated user straight to db when the user issues a lease
         request with iotlab nodes and that he has not registered with iotlab
         yet (that is he does not have a LDAP entry yet).

-        Uses parts of the routines in SlabImport when importing user from LDAP.
+        Uses parts of the routines in IotlabImport when importing user from LDAP.

         Called by AddPerson, right after LdapAddUser.
         :param user_dict: Must contain email, hrn and pkey to get a GID
         and be added to the SFA db.
         Called by AddPerson, right after LdapAddUser.
         :param user_dict: Must contain email, hrn and pkey to get a GID
         and be added to the SFA db.


@@ -830,13 +836,13 @@ class IotlabTestbedAPI():
         logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \
                 %(type(slice_record['hrn']), type(job_id), type(end_time)))
 
         logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \
                 %(type(slice_record['hrn']), type(job_id), type(end_time)))
 

-        iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id,
+        iotlab_ex_row = LeaseTableXP(slice_hrn = slice_record['hrn'], job_id=job_id,

                                  end_time= end_time)
 
         logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \
                 %(iotlab_ex_row))
                                  end_time= end_time)
 
         logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \
                 %(iotlab_ex_row))

-        self.iotlab_db.iotlab_session.add(iotlab_ex_row)
-        self.iotlab_db.iotlab_session.commit()
+        self.iotlab_db.testbed_session.add(iotlab_ex_row)
+        self.iotlab_db.testbed_session.commit()

 
         logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s " \
                 %(start_time))
 
         logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s " \
                 %(start_time))


@@ -886,7 +892,7 @@ class IotlabTestbedAPI():
 
 
     # @staticmethod
 
 
     # @staticmethod

-    # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql):
+    # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql):

     #     """ Cleans the iotlab db by deleting expired and cancelled jobs.
     #     Compares the list of job ids given by OAR with the job ids that
     #     are already in the database, deletes the jobs that are no longer in
     #     """ Cleans the iotlab db by deleting expired and cancelled jobs.
     #     Compares the list of job ids given by OAR with the job ids that
     #     are already in the database, deletes the jobs that are no longer in


@@ -900,16 +906,40 @@ class IotlabTestbedAPI():
     #     set_jobs_psql = set(jobs_psql)
 
     #     kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
     #     set_jobs_psql = set(jobs_psql)
 
     #     kept_jobs = set(job_oar_list).intersection(set_jobs_psql)

-    #     logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \
+    #     logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \

     #         job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
     #     deleted_jobs = set_jobs_psql.difference(kept_jobs)
     #     deleted_jobs = list(deleted_jobs)
     #     if len(deleted_jobs) > 0:
     #         job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
     #     deleted_jobs = set_jobs_psql.difference(kept_jobs)
     #     deleted_jobs = list(deleted_jobs)
     #     if len(deleted_jobs) > 0:

-    #         self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
-    #         self.iotlab_db.iotlab_session.commit()
+    #         self.iotlab_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
+    #         self.iotlab_db.testbed_session.commit()

 
     #     return
 
 
     #     return
 

+    @staticmethod
+    def filter_lease_name(reservation_list, filter_value):
+        filtered_reservation_list = list(reservation_list)
+        logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \
+                        % (reservation_list))
+        for reservation in reservation_list:
+            if 'slice_hrn' in reservation and \
+                reservation['slice_hrn'] != filter_value:
+                filtered_reservation_list.remove(reservation)
+
+        logger.debug("IOTLAB_API \t filter_lease_name filtered_reservation_list %s" \
+                        % (filtered_reservation_list))
+        return filtered_reservation_list
+
+    @staticmethod
+    def filter_lease_start_time(reservation_list, filter_value):
+        filtered_reservation_list = list(reservation_list)
+
+        for reservation in reservation_list:
+            if 't_from' in reservation and \
+                reservation['t_from'] > filter_value:
+                filtered_reservation_list.remove(reservation)
+
+        return filtered_reservation_list

 
     def GetLeases(self, lease_filter_dict=None, login=None):
         """
 
     def GetLeases(self, lease_filter_dict=None, login=None):
         """


@@ -919,7 +949,7 @@ class IotlabTestbedAPI():
             Two purposes:
             -Fetch all the jobs from OAR (running, waiting..)
             complete the reservation information with slice hrn
             Two purposes:
             -Fetch all the jobs from OAR (running, waiting..)
             complete the reservation information with slice hrn

-            found in iotlab_xp table. If not available in the table,
+            found in testbed_xp table. If not available in the table,

             assume it is a iotlab slice.
             -Updates the iotlab table, deleting jobs when necessary.
 
             assume it is a iotlab slice.
             -Updates the iotlab table, deleting jobs when necessary.
 


@@ -930,7 +960,7 @@ class IotlabTestbedAPI():
 
         """
 
 
         """
 

-        unfiltered_reservation_list = self.GetReservedNodes()
+        unfiltered_reservation_list = self.GetReservedNodes(login)

 
         reservation_list = []
         #Find the slice associated with this user iotlab ldap uid
 
         reservation_list = []
         #Find the slice associated with this user iotlab ldap uid


@@ -941,7 +971,7 @@ class IotlabTestbedAPI():
         #the same user in LDAP SA 27/07/12
         job_oar_list = []
 
         #the same user in LDAP SA 27/07/12
         job_oar_list = []
 

-        jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all()
+        jobs_psql_query = self.iotlab_db.testbed_session.query(LeaseTableXP).all()

         jobs_psql_dict = dict([(row.job_id, row.__dict__)
                                for row in jobs_psql_query])
         #jobs_psql_dict = jobs_psql_dict)
         jobs_psql_dict = dict([(row.job_id, row.__dict__)
                                for row in jobs_psql_query])
         #jobs_psql_dict = jobs_psql_dict)


@@ -959,7 +989,7 @@ class IotlabTestbedAPI():
             if resa['lease_id'] in jobs_psql_dict:
                 job_info = jobs_psql_dict[resa['lease_id']]
                 logger.debug("IOTLAB_API \tGetLeases job_info %s"
             if resa['lease_id'] in jobs_psql_dict:
                 job_info = jobs_psql_dict[resa['lease_id']]
                 logger.debug("IOTLAB_API \tGetLeases job_info %s"

-                             % (job_info))
+                          % (job_info))

                 resa['slice_hrn'] = job_info['slice_hrn']
                 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
 
                 resa['slice_hrn'] = job_info['slice_hrn']
                 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
 


@@ -976,17 +1006,38 @@ class IotlabTestbedAPI():
                 iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
                 resa['component_id_list'].append(iotlab_xrn.urn)
 
                 iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
                 resa['component_id_list'].append(iotlab_xrn.urn)
 

-            if lease_filter_dict:
-                logger.debug("IOTLAB_API \tGetLeases resa_ %s \
-                        \r\n leasefilter %s" % (resa, lease_filter_dict))
+        if lease_filter_dict:
+            logger.debug("IOTLAB_API \tGetLeases  \
+                    \r\n leasefilter %s" % ( lease_filter_dict))
+
+            filter_dict_functions = {
+            'slice_hrn' : IotlabTestbedAPI.filter_lease_name,
+            't_from' : IotlabTestbedAPI.filter_lease_start_time
+            }
+            reservation_list = list(unfiltered_reservation_list)
+            for filter_type in lease_filter_dict:
+                logger.debug("IOTLAB_API \tGetLeases reservation_list %s" \
+                    % (reservation_list))
+                reservation_list = filter_dict_functions[filter_type](\
+                    reservation_list,lease_filter_dict[filter_type] )
+
+                # Filter the reservation list with a maximum timespan so that the
+                # leases and jobs running after this timestamp do not appear
+                # in the result leases.
+                # if 'start_time' in :
+                #     if resa['start_time'] < lease_filter_dict['start_time']:
+                #        reservation_list.append(resa)
+
+
+                # if 'name' in lease_filter_dict and \
+                #     lease_filter_dict['name'] == resa['slice_hrn']:
+                #     reservation_list.append(resa)

 
 

-                if lease_filter_dict['name'] == resa['slice_hrn']:
-                    reservation_list.append(resa)

 
         if lease_filter_dict is None:
             reservation_list = unfiltered_reservation_list
 
 
         if lease_filter_dict is None:
             reservation_list = unfiltered_reservation_list
 

-        self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list)
+        self.iotlab_db.update_experiments_in_additional_sfa_db(job_oar_list, jobs_psql_id_list)

 
         logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s"
                      % (reservation_list))
 
         logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s"
                      % (reservation_list))


@@ -1067,8 +1118,8 @@ class IotlabTestbedAPI():
 
         #"""
         ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
 
         #"""
         ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)

-        ##self.iotlab_db.iotlab_session.add(new_row)
-        ##self.iotlab_db.iotlab_session.commit()
+        ##self.iotlab_db.testbed_session.add(new_row)
+        ##self.iotlab_db.testbed_session.commit()

 
         #logger.debug("IOTLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
         #return
 
         #logger.debug("IOTLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
         #return


@@ -1148,7 +1199,7 @@ class IotlabTestbedAPI():
         #of the user otherwise will mess up the RegRecord in
         #Resolve, don't know why - SA 08/08/2012
 
         #of the user otherwise will mess up the RegRecord in
         #Resolve, don't know why - SA 08/08/2012
 

-        #Only one entry for one user  = one slice in iotlab_xp table
+        #Only one entry for one user  = one slice in testbed_xp table

         #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
         raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
         #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
         #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
         raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
         #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()