From 7575be18bc5a95e054f4edf71da6c8b4f42181b0 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Fri, 20 Sep 2013 14:16:59 +0200 Subject: [PATCH] Adding Rspec test files under /testbeds/iotlab/tests/tests_rspecs. Fixing problem in import nodes. Adding the filtering lease feature in iotlabapi. In the future, if filtering the leases depending on their start time is needed, that is only printing the leases scheduled in the next couple of hours/days/weeks and so on is needed, this is already implemented in the iotlab api(see GetLeases). --- sfa/importer/iotlabimporter.py | 109 +++++++++--------- sfa/iotlab/iotlabapi.py | 57 ++++++++- testbeds/iotlab/tests/driver_tests.py | 16 ++- testbeds/iotlab/tests/tests_rspecs/README | 36 ++++++ .../firexp_avakian_slice_iotlab.rspec | 23 ++++ .../firexp_iotlab_slice_all.rspec | 38 ++++++ .../firexp_iotlab_slice_iotlab.rspec | 23 ++++ .../iotlab_avakian_slice_all.rspec | 38 ++++++ .../iotlab_avakian_slice_iotlab.rspec | 23 ++++ .../iotlab_avakian_slice_iotlab2.rspec | 35 ++++++ .../iotlab_avakian_slice_plab.rspec | 16 +++ .../iotlab_user_slice_iotlab.rspec | 25 ++++ .../tests_rspecs/test_delete_all_leases.rspec | 9 ++ 13 files changed, 389 insertions(+), 59 deletions(-) create mode 100644 testbeds/iotlab/tests/tests_rspecs/README create mode 100644 testbeds/iotlab/tests/tests_rspecs/firexp_avakian_slice_iotlab.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_all.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_iotlab.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_all.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab2.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_plab.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/iotlab_user_slice_iotlab.rspec create mode 100644 testbeds/iotlab/tests/tests_rspecs/test_delete_all_leases.rspec diff --git a/sfa/importer/iotlabimporter.py b/sfa/importer/iotlabimporter.py index b95157a5..5bd03444 100644 --- a/sfa/importer/iotlabimporter.py +++ b/sfa/importer/iotlabimporter.py @@ -151,58 +151,6 @@ class IotlabImporter: return self.records_by_type_hrn[rec_tuple] = record - def import_sites_and_nodes(self, iotlabdriver): - """ - - Gets all the sites and nodes from OAR, process the information, - creates hrns and RegAuthority for sites, and feed them to the database. - For each site, import the site's nodes to the DB by calling - import_nodes. - - :param iotlabdriver: IotlabDriver object, used to have access to - iotlabdriver methods and fetching info on sites and nodes. - :type iotlabdriver: IotlabDriver - """ - - sites_listdict = iotlabdriver.iotlab_api.GetSites() - nodes_listdict = iotlabdriver.iotlab_api.GetNodes() - nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) - for site in sites_listdict: - site_hrn = site['name'] - site_record = self.find_record_by_type_hrn ('authority', site_hrn) - if not site_record: - try: - urn = hrn_to_urn(site_hrn, 'authority') - if not self.auth_hierarchy.auth_exists(urn): - self.auth_hierarchy.create_auth(urn) - - auth_info = self.auth_hierarchy.get_auth_info(urn) - site_record = \ - RegAuthority(hrn=site_hrn, - gid=auth_info.get_gid_object(), - pointer='-1', - authority=get_authority(site_hrn)) - site_record.just_created() - dbsession.add(site_record) - dbsession.commit() - self.logger.info("IotlabImporter: imported authority \ - (site) %s" % site_record) - self.update_just_added_records_dict(site_record) - except SQLAlchemyError: - # if the site import fails then there is no point in - # trying to import the - # site's child records(node, slices, persons), so skip them. - self.logger.log_exc("IotlabImporter: failed to import \ - site. Skipping child records") - continue - else: - # xxx update the record ... - pass - - site_record.stale = False - self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver) - - return def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver): """ @@ -277,6 +225,63 @@ class IotlabImporter: pass node_record.stale = False + def import_sites_and_nodes(self, iotlabdriver): + """ + + Gets all the sites and nodes from OAR, process the information, + creates hrns and RegAuthority for sites, and feed them to the database. + For each site, import the site's nodes to the DB by calling + import_nodes. + + :param iotlabdriver: IotlabDriver object, used to have access to + iotlabdriver methods and fetching info on sites and nodes. + :type iotlabdriver: IotlabDriver + """ + + sites_listdict = iotlabdriver.iotlab_api.GetSites() + nodes_listdict = iotlabdriver.iotlab_api.GetNodes() + nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) + for site in sites_listdict: + site_hrn = site['name'] + site_record = self.find_record_by_type_hrn ('authority', site_hrn) + self.logger.info("IotlabImporter: import_sites_and_nodes \ + (site) %s \r\n " % site_record) + if not site_record: + try: + urn = hrn_to_urn(site_hrn, 'authority') + if not self.auth_hierarchy.auth_exists(urn): + self.auth_hierarchy.create_auth(urn) + + auth_info = self.auth_hierarchy.get_auth_info(urn) + site_record = \ + RegAuthority(hrn=site_hrn, + gid=auth_info.get_gid_object(), + pointer='-1', + authority=get_authority(site_hrn)) + site_record.just_created() + dbsession.add(site_record) + dbsession.commit() + self.logger.info("IotlabImporter: imported authority \ + (site) %s" % site_record) + self.update_just_added_records_dict(site_record) + except SQLAlchemyError: + # if the site import fails then there is no point in + # trying to import the + # site's child records(node, slices, persons), so skip them. + self.logger.log_exc("IotlabImporter: failed to import \ + site. Skipping child records") + continue + else: + # xxx update the record ... + pass + + site_record.stale = False + self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver) + + return + + + def init_person_key(self, person, iotlab_key): """ diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py index 50592bd7..0ccced91 100644 --- a/sfa/iotlab/iotlabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -910,6 +910,30 @@ class IotlabTestbedAPI(): # return + @staticmethod + def filter_lease_name(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \ + % (reservation_list)) + for reservation in reservation_list: + if 'slice_hrn' in reservation and \ + reservation['slice_hrn'] != filter_value: + filtered_reservation_list.remove(reservation) + + logger.debug("IOTLAB_API \t filter_lease_name filtered_reservation_list %s" \ + % (filtered_reservation_list)) + return filtered_reservation_list + + @staticmethod + def filter_lease_start_time(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + + for reservation in reservation_list: + if 'start_time' in reservation and \ + reservation['start_time'] > filter_value: + filtered_reservation_list.remove(reservation) + + return filtered_reservation_list def GetLeases(self, lease_filter_dict=None, login=None): """ @@ -959,7 +983,7 @@ class IotlabTestbedAPI(): if resa['lease_id'] in jobs_psql_dict: job_info = jobs_psql_dict[resa['lease_id']] logger.debug("IOTLAB_API \tGetLeases job_info %s" - % (job_info)) + % (job_info)) resa['slice_hrn'] = job_info['slice_hrn'] resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') @@ -976,12 +1000,33 @@ class IotlabTestbedAPI(): iotlab_xrn = iotlab_xrn_object(self.root_auth, node) resa['component_id_list'].append(iotlab_xrn.urn) - if lease_filter_dict: - logger.debug("IOTLAB_API \tGetLeases resa_ %s \ - \r\n leasefilter %s" % (resa, lease_filter_dict)) + if lease_filter_dict: + logger.debug("IOTLAB_API \tGetLeases resa_ %s \ + \r\n leasefilter %s" % (resa, lease_filter_dict)) + + filter_dict_functions = { + 'slice_hrn' : IotlabTestbedAPI.filter_lease_name, + 'start_time' : IotlabTestbedAPI.filter_lease_start_time + } + reservation_list = list(unfiltered_reservation_list) + for filter_type in lease_filter_dict: + logger.debug("IOTLAB_API \tGetLeases reservation_list %s" \ + % (reservation_list)) + reservation_list = filter_dict_functions[filter_type](\ + reservation_list,lease_filter_dict[filter_type] ) + + # Filter the reservation list with a maximum timespan so that the + # leases and jobs running after this timestamp do not appear + # in the result leases. + # if 'start_time' in : + # if resa['start_time'] < lease_filter_dict['start_time']: + # reservation_list.append(resa) + + + # if 'name' in lease_filter_dict and \ + # lease_filter_dict['name'] == resa['slice_hrn']: + # reservation_list.append(resa) - if lease_filter_dict['name'] == resa['slice_hrn']: - reservation_list.append(resa) if lease_filter_dict is None: reservation_list = unfiltered_reservation_list diff --git a/testbeds/iotlab/tests/driver_tests.py b/testbeds/iotlab/tests/driver_tests.py index 25fd3fa3..bd5d571e 100644 --- a/testbeds/iotlab/tests/driver_tests.py +++ b/testbeds/iotlab/tests/driver_tests.py @@ -247,6 +247,15 @@ def TestOAR(job_id = None): return +def TestImporter(arg=None): + iotlabdriver = IotlabDriver(Config()) + + nodes_listdict = iotlabdriver.iotlab_api.GetNodes() + sites_listdict = iotlabdriver.iotlab_api.GetSites() + nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) + + # from sfa.importer.iotlabimporter import IotlabImporter + # importer = IotlabImporter() def TestIotlabDriver(job_id = None): if job_id is None: @@ -274,8 +283,12 @@ def TestIotlabDriver(job_id = None): leases = iotlabdriver.iotlab_api.GetLeases(login='avakian') print "\r\n \r\n GetLeases", leases + leases = iotlabdriver.iotlab_api.GetLeases(lease_filter_dict={'slice_hrn':'iotlab.avakian_slice'}) + print "\r\n \r\n GetLeases", leases + leases = iotlabdriver.iotlab_api.GetLeases(lease_filter_dict={'t_from':1405070000}) + print "\r\n \r\n GetLeases", leases def TestSfi(filename = None): if filename is None: @@ -386,7 +399,8 @@ supported_options = { 'driver': TestIotlabDriver, 'sfi':TestSfi, 'sql':TestSQL, - 'all' : RunAll } + 'all' : RunAll, + 'import': TestImporter } def main(): opts = parse_options() diff --git a/testbeds/iotlab/tests/tests_rspecs/README b/testbeds/iotlab/tests/tests_rspecs/README new file mode 100644 index 00000000..bf10e22b --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/README @@ -0,0 +1,36 @@ +Rspec file names +====================== +Rspec file names are constructed as follows : + slice name used in this rspec + network in which the reserved nodes are + +Networks can be : iotlab, plab, all (iotlab + plab) + +Slices and users +================= +user: +login iotlab : user +hrn iotlab: iotlab.user +hrn firexp: firexp.flab.iotlab_user +slice iotlab: iotlab.user_slice +slice firexp : firexp.flab.iotlab_slice + + +This special test user comes from Firexp and is considered as an +external user coming from a federated testbedd for Iotlab. + +user: +login iotlab: avakian +slice iotlab: iotlab.avakian_slice +hrn firexp : firexp.flab.avakian (?) +slice firexp : firexp.flab.avakian_slice (?) + +This user comes from iotlab. + + +Leases +====== + +The starting time of the leases in those RSpec files are +usually set to be in 2014, so that we don't have to keep the +date in mind and check that we are not scheduling a lease +in the past. diff --git a/testbeds/iotlab/tests/tests_rspecs/firexp_avakian_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/firexp_avakian_slice_iotlab.rspec new file mode 100644 index 00000000..51539f81 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/firexp_avakian_slice_iotlab.rspec @@ -0,0 +1,23 @@ + + + + + node5.devlille.senslab.info + + TRUE + + + + + node2.devlille.senslab.info + + TRUE + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_all.rspec b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_all.rspec new file mode 100644 index 00000000..aa1d3b57 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_all.rspec @@ -0,0 +1,38 @@ + + + + + effet.pl.sophia.inria.fr + TRUE + + + planetlab.test.plab.effet + + + + + + + + + node5.devlille.senslab.info + + TRUE + + + + + node2.devlille.senslab.info + + TRUE + + + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_iotlab.rspec new file mode 100644 index 00000000..93719f6e --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_iotlab.rspec @@ -0,0 +1,23 @@ + + + + + node5.devlille.senslab.info + + TRUE + + + + + node2.devlille.senslab.info + + TRUE + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_all.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_all.rspec new file mode 100644 index 00000000..bc0f78b9 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_all.rspec @@ -0,0 +1,38 @@ + + + + + effet.pl.sophia.inria.fr + TRUE + + + planetlab.test.plab.effet + + + + + + + + + node5.devlille.senslab.info + + TRUE + + + + + node2.devlille.senslab.info + + TRUE + + + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec new file mode 100644 index 00000000..20e50b50 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec @@ -0,0 +1,23 @@ + + + + + node5.devlille.senslab.info + + TRUE + + + + + node2.devlille.senslab.info + + TRUE + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab2.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab2.rspec new file mode 100644 index 00000000..c8851110 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab2.rspec @@ -0,0 +1,35 @@ + + + + + node11.devlille.senslab.info + + TRUE + + + + + node12.devlille.senslab.info + + TRUE + + + + + + + effet.pl.sophia.inria.fr + FALSE + + x86_64 + f14 + onelab + planetlab.test.plab.effet + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_plab.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_plab.rspec new file mode 100644 index 00000000..3bf9373f --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_plab.rspec @@ -0,0 +1,16 @@ + + + + + effet.pl.sophia.inria.fr + TRUE + + + planetlab.test.plab.effet + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_user_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_user_slice_iotlab.rspec new file mode 100644 index 00000000..75b1fdbb --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_user_slice_iotlab.rspec @@ -0,0 +1,25 @@ + + + + + node11.devlille.senslab.info + + TRUE + + + + + node12.devlille.senslab.info + + TRUE + + + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/test_delete_all_leases.rspec b/testbeds/iotlab/tests/tests_rspecs/test_delete_all_leases.rspec new file mode 100644 index 00000000..ad1d659c --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/test_delete_all_leases.rspec @@ -0,0 +1,9 @@ + + + + + + + + + -- 2.43.0