From a18ad5b1dbc2a1dd346784105c89a0714086ebc4 Mon Sep 17 00:00:00 2001 From: Mohamed Larabi Date: Mon, 29 Jul 2013 00:02:18 +0200 Subject: [PATCH] Adding and formatting documentation for Sphinx. Renaming folder senslab under /testbeds into iotlab. Changing bash_testsuite according to new naming convention. Cleaning files and replacing slab stuff by iotlab. Patching sliver_status Correcting missing bracket. Changing iotlabpostgres to remove global variable iotlab_dbsession and to put a singleton instead. Removing useless prints and correcting attribute in api. further fixes. Moving update_jobs_in_iotlabdb in iotlabpostgres.py. Removing now useless attribute db in iotlabdriver. Changing name of create_engine in iotlabpostgres to create_iotlab_engine. Adding forgotten import in importer. Cleaning and commenting mostly. Adding a new class in OARrestapi.py to put together all the parsing functions used for a GET_resources_full. Fixing bug in DeleteLeases. Documenting, cleaning. More cleaning and documenting. fixing getslices (sfi show node_ids was empty when done on a slice) and fixing create lease broken return Rspec. Correcting fill_record_info to add information on nodes. Modifying the test script (is now a python script). Fixing LdapSearch, as it appears that not all the entries in LDAP have first_name and last_name assigned, which failed (user is not found but no error raised whatsoever) when using a (cn=first_name last_name) when looking for a specific user. Fixing test script. Fixing yet another problem with GetSlices (slice returned is empty if of all the current leases none belong tto the slice we are looking for). --- sfa/generic/iotlab.py | 7 +- sfa/importer/iotlabimporter.py | 153 +--- sfa/iotlab/LDAPapi.py | 652 +++++++------- sfa/iotlab/OARrestapi.py | 775 ++++++++++++----- sfa/iotlab/iotlabaggregate.py | 353 ++++---- sfa/iotlab/iotlabapi.py | 798 +++++++----------- sfa/iotlab/iotlabdriver.py | 749 +++++++--------- sfa/iotlab/iotlabpostgres.py | 275 +++--- sfa/iotlab/iotlabslices.py | 399 +++++---- sfa/rspecs/elements/versions/iotlabv1Lease.py | 5 +- sfa/rspecs/elements/versions/iotlabv1Node.py | 14 +- sfa/rspecs/versions/iotlabv1.py | 2 +- 12 files changed, 2037 insertions(+), 2145 deletions(-) diff --git a/sfa/generic/iotlab.py b/sfa/generic/iotlab.py index 7ca346be..d5636a7c 100644 --- a/sfa/generic/iotlab.py +++ b/sfa/generic/iotlab.py @@ -30,13 +30,8 @@ class iotlab (Generic): # driver class for server-side services, talk to the whole testbed def driver_class (self): -<<<<<<< HEAD:sfa/generic/slab.py - import sfa.managers.v2_to_v3_adapter - return sfa.managers.v2_to_v3_adapter.V2ToV3Adapter -======= import sfa.iotlab.iotlabdriver return sfa.iotlab.iotlabdriver.IotlabDriver ->>>>>>> 7cb1e78... Renaming Senslab into Iotlab.:sfa/generic/iotlab.py # iotlab does not have a component manager yet # manager class @@ -44,4 +39,4 @@ class iotlab (Generic): return None # driver_class def component_driver_class (self): - return None + return None \ No newline at end of file diff --git a/sfa/importer/iotlabimporter.py b/sfa/importer/iotlabimporter.py index 62004356..0817dad8 100644 --- a/sfa/importer/iotlabimporter.py +++ b/sfa/importer/iotlabimporter.py @@ -2,7 +2,7 @@ from sfa.util.config import Config from sfa.util.xrn import Xrn, get_authority, hrn_to_urn from sfa.iotlab.iotlabdriver import IotlabDriver - +from sfa.iotlab.iotlabpostgres import IotlabDB, Iotlab_xp from sfa.trust.certificate import Keypair, convert_public_key from sfa.trust.gid import create_uuid @@ -15,17 +15,10 @@ from sqlalchemy.exc import SQLAlchemyError -<<<<<<< HEAD:sfa/importer/iotlabimporter.py class IotlabImporter: """ IotlabImporter class, generic importer_class. Used to populate the SFA DB with iotlab resources' records. -======= -class SlabImporter: - """ - SlabImporter class, generic importer_class. Used to populate the SFA DB - with senslab resources' records. ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py Used to update records when new resources, users or nodes, are added or deleted. """ @@ -151,11 +144,7 @@ class SlabImporter: return self.records_by_type_hrn [ rec_tuple ] = record -<<<<<<< HEAD:sfa/importer/iotlabimporter.py def import_sites_and_nodes(self, iotlabdriver): -======= - def import_sites_and_nodes(self, slabdriver): ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py """ Gets all the sites and nodes from OAR, process the information, @@ -163,7 +152,6 @@ class SlabImporter: For each site, import the site's nodes to the DB by calling import_nodes. -<<<<<<< HEAD:sfa/importer/iotlabimporter.py :param iotlabdriver: IotlabDriver object, used to have access to iotlabdriver methods and fetching info on sites and nodes. :type iotlabdriver: IotlabDriver @@ -171,15 +159,6 @@ class SlabImporter: sites_listdict = iotlabdriver.iotlab_api.GetSites() nodes_listdict = iotlabdriver.iotlab_api.GetNodes() -======= - :param slabdriver: SlabDriver object, used to have access to slabdriver - methods and fetching info on sites and nodes. - :type slabdriver: SlabDriver - """ - - sites_listdict = slabdriver.slab_api.GetSites() - nodes_listdict = slabdriver.slab_api.GetNodes() ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) for site in sites_listdict: site_hrn = site['name'] @@ -198,22 +177,14 @@ class SlabImporter: site_record.just_created() dbsession.add(site_record) dbsession.commit() -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.info("IotlabImporter: imported authority (site) \ -======= - self.logger.info("SlabImporter: imported authority (site) \ ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py %s" % site_record) self.update_just_added_records_dict(site_record) except SQLAlchemyError: # if the site import fails then there is no point in # trying to import the # site's child records(node, slices, persons), so skip them. -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.log_exc("IotlabImporter: failed to import site. \ -======= - self.logger.log_exc("SlabImporter: failed to import site. \ ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py Skipping child records") continue else: @@ -222,19 +193,11 @@ class SlabImporter: site_record.stale = False -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver) return def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver): -======= - self.import_nodes(site['node_ids'], nodes_by_id, slabdriver) - - return - - def import_nodes(self, site_node_ids, nodes_by_id, slabdriver): ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py """ Creates appropriate hostnames and RegNode records for @@ -249,11 +212,7 @@ class SlabImporter: :type nodes_by_id: dictionary :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver attributes. -<<<<<<< HEAD:sfa/importer/iotlabimporter.py :type iotlabdriver:IotlabDriver -======= - :type slabdriver:SlabDriver ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py """ @@ -267,11 +226,7 @@ class SlabImporter: escaped_hrn = \ self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth, \ node['hostname']) -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.info("IOTLABIMPORTER node %s " %(node)) -======= - self.logger.info("SLABIMPORTER node %s " %(node)) ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py hrn = node['hrn'] @@ -286,11 +241,7 @@ class SlabImporter: self.auth_hierarchy.create_gid(urn, \ create_uuid(), pkey) -<<<<<<< HEAD:sfa/importer/iotlabimporter.py def iotlab_get_authority(hrn): -======= - def slab_get_authority(hrn): ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py return hrn.split(".")[0] node_record = RegNode(hrn=hrn, gid=node_gid, @@ -301,44 +252,26 @@ class SlabImporter: node_record.just_created() dbsession.add(node_record) dbsession.commit() -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.info("IotlabImporter: imported node: %s" \ % node_record) self.update_just_added_records_dict(node_record) except SQLAlchemyError: self.logger.log_exc("IotlabImporter: \ -======= - self.logger.info("SlabImporter: imported node: %s" \ - % node_record) - self.update_just_added_records_dict(node_record) - except SQLAlchemyError: - self.logger.log_exc("SlabImporter: \ ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py failed to import node") else: #TODO: xxx update the record ... pass node_record.stale = False -<<<<<<< HEAD:sfa/importer/iotlabimporter.py def init_person_key (self, person, iotlab_key): -======= - - - def init_person_key (self, person, slab_key): ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py """ Returns a tuple pubkey and pkey. :param person Person's data. :type person: dict -<<<<<<< HEAD:sfa/importer/iotlabimporter.py :param iotlab_key: SSH public key, from LDAP user's data. -======= - :param slab_key: SSH public key, from LDAP user's data. ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py RSA type supported. :type iotlab_key: string :rtype (string, Keypair) @@ -346,11 +279,7 @@ class SlabImporter: pubkey = None if person['pkey']: # randomly pick first key in set -<<<<<<< HEAD:sfa/importer/iotlabimporter.py pubkey = iotlab_key -======= - pubkey = slab_key ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py try: pkey = convert_public_key(pubkey) @@ -370,11 +299,7 @@ class SlabImporter: return (pubkey, pkey) -<<<<<<< HEAD:sfa/importer/iotlabimporter.py def import_persons_and_slices(self, iotlabdriver): -======= - def import_persons_and_slices(self, slabdriver): ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py """ Gets user data from LDAP, process the information. @@ -385,31 +310,18 @@ class SlabImporter: import the user's slice onto the database as well by calling import_slice. -<<<<<<< HEAD:sfa/importer/iotlabimporter.py :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver -======= - :param slabdriver:SlabDriver object, used to have access to slabdriver ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py attributes. :type iotlabdriver:IotlabDriver """ -<<<<<<< HEAD:sfa/importer/iotlabimporter.py ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons() self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n" \ -======= - ldap_person_listdict = slabdriver.slab_api.GetPersons() - self.logger.info("SLABIMPORT \t ldap_person_listdict %s \r\n" \ ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py %(ldap_person_listdict)) # import persons for person in ldap_person_listdict : -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.info("IotlabImporter: person :" %(person)) -======= - self.logger.info("SlabImporter: person :" %(person)) ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py if 'ssh-rsa' not in person['pkey']: #people with invalid ssh key (ssh-dss, empty, bullshit keys...) #won't be imported @@ -423,21 +335,13 @@ class SlabImporter: person_urn = hrn_to_urn(person_hrn, 'user') -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.info("IotlabImporter: users_rec_by_email %s " \ -======= - self.logger.info("SlabImporter: users_rec_by_email %s " \ ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py %(self.users_rec_by_email)) #Check if user using person['email'] from LDAP is already registered #in SFA. One email = one person. In this case, do not create another #record for this person -<<<<<<< HEAD:sfa/importer/iotlabimporter.py #person_hrn returned by GetPerson based on iotlab root auth + -======= - #person_hrn returned by GetPerson based on senslab root auth + ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py #uid ldap user_record = self.find_record_by_type_hrn('user', person_hrn) @@ -449,11 +353,7 @@ class SlabImporter: slice_record = self.find_record_by_type_hrn ('slice', slice_hrn) -<<<<<<< HEAD:sfa/importer/iotlabimporter.py iotlab_key = person['pkey'] -======= - slab_key = person['pkey'] ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py # new person if not user_record: (pubkey, pkey) = self.init_person_key(person, iotlab_key) @@ -462,7 +362,7 @@ class SlabImporter: self.auth_hierarchy.create_gid(person_urn, \ create_uuid(), pkey) if person['email']: - self.logger.debug( "SLAB IMPORTER \ + self.logger.debug( "IOTLAB IMPORTER \ PERSON EMAIL OK email %s " %(person['email'])) person_gid.set_email(person['email']) user_record = RegUser(hrn=person_hrn, \ @@ -491,11 +391,7 @@ class SlabImporter: self.update_just_added_records_dict( user_record ) except SQLAlchemyError: -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.log_exc("IotlabImporter: \ -======= - self.logger.log_exc("SlabImporter: \ ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py failed to import person %s"%(person)) else: # update the record ? @@ -504,11 +400,7 @@ class SlabImporter: sfa_keys = user_record.reg_keys new_key = False -<<<<<<< HEAD:sfa/importer/iotlabimporter.py if iotlab_key is not sfa_keys : -======= - if slab_key is not sfa_keys : ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py new_key = True if new_key: self.logger.info("IotlabImporter: \t \t USER UPDATE \ @@ -531,11 +423,7 @@ class SlabImporter: dbsession.commit() user_record.stale = False except SQLAlchemyError: -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.log_exc("IotlabImporter: \ -======= - self.logger.log_exc("SlabImporter: \ ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py failed to update person %s"%(person)) self.import_slice(slice_hrn, slice_record, user_record) @@ -577,15 +465,9 @@ class SlabImporter: self.update_just_added_records_dict ( slice_record ) except SQLAlchemyError: -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.log_exc("IotlabImporter: failed to import slice") #No slice update upon import in iotlab -======= - self.logger.log_exc("SlabImporter: failed to import slice") - - #No slice update upon import in senslab ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py else: # xxx update the record ... self.logger.warning ("Slice update not yet implemented") @@ -598,11 +480,7 @@ class SlabImporter: dbsession.commit() slice_record.stale = False except SQLAlchemyError: -<<<<<<< HEAD:sfa/importer/iotlabimporter.py self.logger.log_exc("IotlabImporter: failed to update slice") -======= - self.logger.log_exc("SlabImporter: failed to update slice") ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py def run (self, options): @@ -616,13 +494,12 @@ class SlabImporter: """ config = Config() -<<<<<<< HEAD:sfa/importer/iotlabimporter.py iotlabdriver = IotlabDriver(config) - + iotlab_db = IotlabDB(config) #Create special slice table for iotlab - if not iotlabdriver.db.exists('iotlab_xp'): - iotlabdriver.db.createtable() + if not iotlab_db.exists('iotlab_xp'): + iotlab_db.createtable() self.logger.info ("IotlabImporter.run: iotlab_xp table created ") @@ -635,26 +512,6 @@ class SlabImporter: # special records must be preserved system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth, \ iotlabdriver.hrn+ '.slicemanager'] -======= - slabdriver = SlabDriver(config) - - #Create special slice table for senslab - - if not slabdriver.db.exists('slab_xp'): - slabdriver.db.createtable() - self.logger.info ("SlabImporter.run: slab_xp table created ") - - - # import site and node records in site into the SFA db. - self.import_sites_and_nodes(slabdriver) - #import users and slice into the SFA DB. - self.import_persons_and_slices(slabdriver) - - ### remove stale records - # special records must be preserved - system_hrns = [slabdriver.hrn, slabdriver.slab_api.root_auth, \ - slabdriver.hrn+ '.slicemanager'] ->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py for record in self.all_records: if record.hrn in system_hrns: record.stale = False diff --git a/sfa/iotlab/LDAPapi.py b/sfa/iotlab/LDAPapi.py index 98a72ec8..254f9942 100644 --- a/sfa/iotlab/LDAPapi.py +++ b/sfa/iotlab/LDAPapi.py @@ -1,19 +1,41 @@ +""" +This API is adapted for OpenLDAP. +The file contains all LDAP classes and methods needed to: + - Load the LDAP connection configuration file (login, address..) with + LdapConfig + - Connect to LDAP with ldap_co + - Create a unique LDAP login and password for a user based on his email or + last name and first name with LoginPassword. + - Manage entries in LDAP using SFA records with LDAPapi + (Search, Add, Delete, Modify) + +""" import random from passlib.hash import ldap_salted_sha1 as lssha + from sfa.util.xrn import get_authority -import ldap +from sfa.util.sfalogging import logger from sfa.util.config import Config - +import ldap import ldap.modlist as modlist -from sfa.util.sfalogging import logger -import os.path -#API for OpenLDAP +import os.path class LdapConfig(): - def __init__(self, config_file = '/etc/sfa/ldap_config.py'): + """ + Ldap configuration class loads the configuration file and sets the + ldap IP address, password, people dn, web dn, group dn. All these settings + were defined in a separate file ldap_config.py to avoid sharing them in + the SFA git as it contains sensible information. + + """ + def __init__(self, config_file='/etc/sfa/ldap_config.py'): + """Loads configuration from file /etc/sfa/ldap_config.py and set the + parameters for connection to LDAP. + + """ try: execfile(config_file, self.__dict__) @@ -23,13 +45,17 @@ class LdapConfig(): self.config_path = os.path.dirname(config_file) except IOError: raise IOError, "Could not find or load the configuration file: %s" \ - % config_file + % config_file class ldap_co: """ Set admin login and server configuration variables.""" def __init__(self): + """Fetch LdapConfig attributes (Ldap server connection parameters and + defines port , version and subtree scope. + + """ #Iotlab PROD LDAP parameters self.ldapserv = None ldap_config = LdapConfig() @@ -39,24 +65,20 @@ class ldap_co: self.ldapGroupDN = ldap_config.LDAP_GROUP_DN self.ldapAdminDN = ldap_config.LDAP_WEB_DN self.ldapAdminPassword = ldap_config.LDAP_WEB_PASSWORD - - self.ldapPort = ldap.PORT - self.ldapVersion = ldap.VERSION3 + self.ldapVersion = ldap.VERSION3 self.ldapSearchScope = ldap.SCOPE_SUBTREE - def connect(self, bind=True): - """ - Enables connection to the LDAP server. - :param bind : Set the bind parameter to True if a bind is needed - (for add/modify/delete operations). - Set to False otherwise. - - :type bind : boolean - :return: dictionary with status of the connection. True if Successful, - False if not and in this case the error message( {'bool', 'message'} ) - :rtype:dict + """Enables connection to the LDAP server. + + :param bind: Set the bind parameter to True if a bind is needed + (for add/modify/delete operations). Set to False otherwise. + :type bind: boolean + :returns: dictionary with status of the connection. True if Successful, + False if not and in this case the error + message( {'bool', 'message'} ). + :rtype: dict """ try: @@ -73,8 +95,9 @@ class ldap_co: def bind(self): """ Binding method. - :return: dictionary with the bind status. True if Successful, - False if not and in this case the error message( {'bool', 'message'} ) + + :returns: dictionary with the bind status. True if Successful, + False if not and in this case the error message({'bool','message'}) :rtype: dict """ @@ -84,8 +107,8 @@ class ldap_co: # Bind/authenticate with a user with apropriate #rights to add objects - self.ldapserv.simple_bind_s(self.ldapAdminDN, \ - self.ldapAdminPassword) + self.ldapserv.simple_bind_s(self.ldapAdminDN, + self.ldapAdminPassword) except ldap.LDAPError, error: return {'bool': False, 'message': error} @@ -96,6 +119,9 @@ class ldap_co: """ Close the LDAP connection. Can throw an exception if the unbinding fails. + :returns: dictionary with the bind status if fails. + False if not and in this case the error message({'bool','message'}) + :rtype: dict or None """ try: @@ -114,8 +140,8 @@ class LoginPassword(): def __init__(self): """ - Sets password and login maximum length, and defines the characters - that can be found in a random generated password. + Sets password and login maximum length, and defines the characters that + can be found in a random generated password. """ self.login_max_length = 8 @@ -134,13 +160,13 @@ class LoginPassword(): def clean_user_names(record): """ - Removes special characters such as - '-', '_' , '[', ']' and ' ' from the first name and last name. + Removes special characters such as '-', '_' , '[', ']' and ' ' from the + first name and last name. :param record: user's record - :type record:dict - :return: lower_first_name and lower_last_name if they were found - in the user's record. Return None, none otherwise. + :type record: dict + :returns: lower_first_name and lower_last_name if they were found + in the user's record. Return None, none otherwise. :rtype: string, string or None, None. """ @@ -161,16 +187,19 @@ class LoginPassword(): @staticmethod def extract_name_from_email(record): """ + When there is no valid first name and last name in the record, the email is used to generate the login. Here, we assume the email is firstname.lastname@something.smthg. The first name and last names are extracted from the email, special charcaters are removed and they are changed into lower case. + :param record: user's data - :type record:dict - :return: the first name and last name taken from the user's email. - lower_first_name, lower_last_name. + :type record: dict + :returns: the first name and last name taken from the user's email. + lower_first_name, lower_last_name. :rtype: string, string + """ email = record['email'] @@ -196,13 +225,18 @@ class LoginPassword(): return lower_first_name, lower_last_name def get_user_firstname_lastname(self, record): - """Get the user first name and last name from the information - we have in the record. + """ + + Get the user first name and last name from the information we have in + the record. + :param record: user's information :type record: dict - :return: the user's first name and last name. - ..seealso: clean_user_names - ..seealso: extract_name_from_email + :returns: the user's first name and last name. + + .. seealso:: clean_user_names + .. seealso:: extract_name_from_email + """ lower_first_name, lower_last_name = self.clean_user_names(record) @@ -210,62 +244,64 @@ class LoginPassword(): if lower_first_name is None and lower_last_name is None: lower_first_name, lower_last_name = \ - self.extract_name_from_email(record) + self.extract_name_from_email(record) return lower_first_name, lower_last_name - def choose_sets_chars_for_login(self, lower_first_name, lower_last_name): """ - Algorithm to select sets of characters from the first name and - last name, depending on the lenght of the last name and the - maximum login length which in our case is set to 8 charachetrs. + + Algorithm to select sets of characters from the first name and last + name, depending on the lenght of the last name and the maximum login + length which in our case is set to 8 characters. + :param lower_first_name: user's first name in lower case. :param lower_last_name: usr's last name in lower case. - :return: user's login - :rtype:string + :returns: user's login + :rtype: string + """ length_last_name = len(lower_last_name) self.login_max_length = 8 #Try generating a unique login based on first name and last name - if length_last_name >= self.login_max_length : + if length_last_name >= self.login_max_length: login = lower_last_name[0:self.login_max_length] index = 0 - logger.debug("login : %s index : %s" %(login, index)) - elif length_last_name >= 4 : + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name >= 4: login = lower_last_name index = 0 - logger.debug("login : %s index : %s" %(login, index)) - elif length_last_name == 3 : + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name == 3: login = lower_first_name[0:1] + lower_last_name index = 1 - logger.debug("login : %s index : %s" %(login, index)) + logger.debug("login : %s index : %s" % (login, index)) elif length_last_name == 2: - if len ( lower_first_name) >=2: + if len(lower_first_name) >= 2: login = lower_first_name[0:2] + lower_last_name index = 2 - logger.debug("login : %s index : %s" %(login, index)) + logger.debug("login : %s index : %s" % (login, index)) else: logger.error("LoginException : \ Generation login error with \ minimum four characters") - else : + else: logger.error("LDAP LdapGenerateUniqueLogin failed : \ - impossible to generate unique login for %s %s" \ - %(lower_first_name,lower_last_name)) + impossible to generate unique login for %s %s" + % (lower_first_name, lower_last_name)) return index, login - - def generate_password(self): + """ - """Generate a password upon adding a new user in LDAP Directory - (8 characters length). The generated password is composed of characters - from the charsPassword list - :return: the randomly generated password + Generate a password upon adding a new user in LDAP Directory + (8 characters length). The generated password is composed of characters + from the chars_password list. + + :returns: the randomly generated password :rtype: string """ @@ -273,27 +309,33 @@ class LoginPassword(): length = len(self.chars_password) for index in range(self.length_password): - char_index = random.randint(0, length-1) + char_index = random.randint(0, length - 1) password += self.chars_password[char_index] return password @staticmethod def encrypt_password(password): - """ Use passlib library to make a RFC2307 LDAP encrypted password - salt size = 8, use sha-1 algorithm. + """ + + Use passlib library to make a RFC2307 LDAP encrypted password salt size + is 8, use sha-1 algorithm. + :param password: password not encrypted. :type password: string - :return: Returns encrypted password. - :rtype:string + :returns: Returns encrypted password. + :rtype: string + """ #Keep consistency with Java Iotlab's LDAP API #RFC2307SSHAPasswordEncryptor so set the salt size to 8 bytes - return lssha.encrypt(password, salt_size = 8) + return lssha.encrypt(password, salt_size=8) +class LDAPapi: + """Defines functions to insert and search entries in the LDAP. -class LDAPapi : + """ def __init__(self): logger.setLevelDebug() @@ -302,24 +344,17 @@ class LDAPapi : config = Config() self.login_pwd = LoginPassword() self.authname = config.SFA_REGISTRY_ROOT_AUTH - self.conn = ldap_co() self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH - self.baseDN = self.conn.ldapPeopleDN - - - self.ldapShell = '/bin/bash' - def LdapGenerateUniqueLogin(self, record): """ -<<<<<<< HEAD:sfa/iotlab/LDAPapi.py Generate login for adding a new user in LDAP Directory (four characters minimum length). Get proper last name and @@ -327,8 +362,8 @@ class LDAPapi : :param record: Record must contain first_name and last_name. :param record: dict - :return: the generated login for the user described with record if the - login generation is successful, None if it fails. + :returns: the generated login for the user described with record if the + login generation is successful, None if it fails. :rtype: string or None """ @@ -337,18 +372,17 @@ class LDAPapi : record['email'] = record['mail'] lower_first_name, lower_last_name = \ - self.login_pwd.get_user_firstname_lastname(record) + self.login_pwd.get_user_firstname_lastname(record) + index, login = self.login_pwd.choose_sets_chars_for_login( + lower_first_name, lower_last_name) - index, login = self.login_pwd.choose_sets_chars_for_login( \ - lower_first_name, \ - lower_last_name) login_filter = '(uid=' + login + ')' get_attrs = ['uid'] - try : + try: #Check if login already in use - while (len(self.LdapSearch(login_filter, get_attrs)) is not 0 ): + while (len(self.LdapSearch(login_filter, get_attrs)) is not 0): index += 1 if index >= 9: @@ -357,26 +391,30 @@ class LDAPapi : else: try: login = \ - lower_first_name[0:index] + \ - lower_last_name[0:self.login_pwd.login_max_length-index] - login_filter = '(uid='+ login+ ')' + lower_first_name[0:index] + \ + lower_last_name[0: + self.login_pwd.login_max_length + - index] + login_filter = '(uid=' + login + ')' except KeyError: print "lower_first_name - lower_last_name too short" - logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s"%(login)) + logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s" + % (login)) return login - except ldap.LDAPError, error : - logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" %error) + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error)) return None - def find_max_uidNumber(self): + """Find the LDAP max uidNumber (POSIX uid attribute). - """Find the LDAP max uidNumber (POSIX uid attribute) . Used when adding a new user in LDAP Directory - :return: max uidNumber + 1 - :rtype:string + + :returns: max uidNumber + 1 + :rtype: string + """ #First, get all the users in the LDAP get_attrs = "(uidNumber=*)" @@ -388,10 +426,9 @@ class LDAPapi : max_uidnumber = self.ldapUserUidNumberMin #Otherwise, get the highest uidNumber else: - - uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data ] + uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data] logger.debug("LDAPapi.py \tfind_max_uidNumber \ - uidNumberList %s " %(uidNumberList)) + uidNumberList %s " % (uidNumberList)) max_uidnumber = max(uidNumberList) + 1 return str(max_uidnumber) @@ -407,29 +444,31 @@ class LDAPapi : @staticmethod #TODO Handle OR filtering in the ldap query when #dealing with a list of records instead of doing a for loop in GetPersons - def make_ldap_filters_from_record( record=None): - """ - Helper function to make LDAP filter requests out of SFA records. + def make_ldap_filters_from_record(record=None): + """Helper function to make LDAP filter requests out of SFA records. + :param record: user's sfa record. Should contain first_name,last_name, - email or mail, and if the record is enabled or not. If the dict - record does not have all of these, must at least contain the user's - email. + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. :type record: dict - :return: LDAP request + :returns: LDAP request :rtype: string + """ req_ldap = '' req_ldapdict = {} if record : - if 'first_name' in record and 'last_name' in record: - req_ldapdict['cn'] = str(record['first_name'])+" "\ - + str(record['last_name']) - if 'email' in record : + if 'first_name' in record and 'last_name' in record: + if record['first_name'] != record['last_name']: + req_ldapdict['cn'] = str(record['first_name'])+" "\ + + str(record['last_name']) + if 'email' in record: req_ldapdict['mail'] = record['email'] if 'mail' in record: req_ldapdict['mail'] = record['mail'] if 'enabled' in record: - if record['enabled'] == True : + if record['enabled'] is True: req_ldapdict['shadowExpire'] = '-1' else: req_ldapdict['shadowExpire'] = '0' @@ -440,45 +479,42 @@ class LDAPapi : #Plus, the SFA user may already have an account with iotlab #using another login. - - logger.debug("\r\n \t LDAP.PY make_ldap_filters_from_record \ - record %s req_ldapdict %s" \ - %(record, req_ldapdict)) + record %s req_ldapdict %s" + % (record, req_ldapdict)) for k in req_ldapdict: - req_ldap += '('+ str(k)+ '=' + str(req_ldapdict[k]) + ')' - if len(req_ldapdict.keys()) >1 : + req_ldap += '(' + str(k) + '=' + str(req_ldapdict[k]) + ')' + if len(req_ldapdict.keys()) >1 : req_ldap = req_ldap[:0]+"(&"+req_ldap[0:] size = len(req_ldap) - req_ldap = req_ldap[:(size-1)] +')'+ req_ldap[(size-1):] + req_ldap = req_ldap[:(size-1)] + ')' + req_ldap[(size-1):] else: req_ldap = "(cn=*)" return req_ldap def make_ldap_attributes_from_record(self, record): - """When adding a new user to Iotlab's LDAP, creates an attributes - dictionnary from the SFA record understandable by LDAP. - Generates the user's LDAP login. - User is automatically validated (account enabled) and described - as a SFA USER FROM OUTSIDE SENSLAB'. + """ + + When adding a new user to Iotlab's LDAP, creates an attributes + dictionnary from the SFA record understandable by LDAP. Generates the + user's LDAP login.User is automatically validated (account enabled) + and described as a SFA USER FROM OUTSIDE IOTLAB. + :param record: must contain the following keys and values: - first_name, last_name, mail, pkey (ssh key). + first_name, last_name, mail, pkey (ssh key). :type record: dict - - :return: dictionary of attributes using LDAP data structure - model. + :returns: dictionary of attributes using LDAP data structure model. :rtype: dict """ attrs = {} - attrs['objectClass'] = ["top", "person", "inetOrgPerson", \ - "organizationalPerson", "posixAccount", \ - "shadowAccount", "systemQuotas", \ - "ldapPublicKey"] - + attrs['objectClass'] = ["top", "person", "inetOrgPerson", + "organizationalPerson", "posixAccount", + "shadowAccount", "systemQuotas", + "ldapPublicKey"] attrs['uid'] = self.LdapGenerateUniqueLogin(record) try: @@ -493,7 +529,6 @@ class LDAPapi : attrs['cn'] = attrs['uid'] attrs['gecos'] = attrs['uid'] - attrs['quota'] = self.ldapUserQuotaNFS attrs['homeDirectory'] = self.ldapUserHomePath + attrs['uid'] attrs['loginShell'] = self.ldapShell @@ -533,28 +568,27 @@ class LDAPapi : def LdapAddUser(self, record) : """Add SFA user to LDAP if it is not in LDAP yet. - :param record: dictionnary with the user's data. - :return: a dictionary with the status (Fail= False, Success= True) - and the uid of the newly added user if successful, or the error - meassage it is not. Dict has keys bool and message in case of failure, - and bool uid in case of success. + :param record: dictionnary with the user's data. + :returns: a dictionary with the status (Fail= False, Success= True) + and the uid of the newly added user if successful, or the error + meassage it is not. Dict has keys bool and message in case of + failure, and bool uid in case of success. :rtype: dict - ..seealso: make_ldap_filters_from_record + .. seealso:: make_ldap_filters_from_record """ logger.debug(" \r\n \t LDAP LdapAddUser \r\n\r\n ================\r\n ") user_ldap_attrs = self.make_ldap_attributes_from_record(record) - #Check if user already in LDAP wih email, first name and last name filter_by = self.make_ldap_filters_from_record(user_ldap_attrs) user_exist = self.LdapSearch(filter_by) if user_exist: logger.warning(" \r\n \t LDAP LdapAddUser user %s %s \ - already exists" %(user_ldap_attrs['sn'], \ - user_ldap_attrs['mail'])) + already exists" % (user_ldap_attrs['sn'], + user_ldap_attrs['mail'])) return {'bool': False} #Bind to the server @@ -563,40 +597,38 @@ class LDAPapi : if(result['bool']): # A dict to help build the "body" of the object - - logger.debug(" \r\n \t LDAP LdapAddUser attrs %s " %user_ldap_attrs) + logger.debug(" \r\n \t LDAP LdapAddUser attrs %s " + % user_ldap_attrs) # The dn of our new entry/object dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN try: ldif = modlist.addModlist(user_ldap_attrs) - logger.debug("LDAPapi.py add attrs %s \r\n ldif %s"\ - %(user_ldap_attrs, ldif) ) + logger.debug("LDAPapi.py add attrs %s \r\n ldif %s" + % (user_ldap_attrs, ldif)) self.conn.ldapserv.add_s(dn, ldif) - logger.info("Adding user %s login %s in LDAP" \ - %(user_ldap_attrs['cn'] , user_ldap_attrs['uid'])) - - + logger.info("Adding user %s login %s in LDAP" + % (user_ldap_attrs['cn'], user_ldap_attrs['uid'])) except ldap.LDAPError, error: - logger.log_exc("LDAP Add Error %s" %error) + logger.log_exc("LDAP Add Error %s" % error) return {'bool': False, 'message': error} self.conn.close() - return {'bool': True, 'uid':user_ldap_attrs['uid']} + return {'bool': True, 'uid': user_ldap_attrs['uid']} else: return result - def LdapDelete(self, person_dn): - """ - Deletes a person in LDAP. Uses the dn of the user. + """Deletes a person in LDAP. Uses the dn of the user. + :param person_dn: user's ldap dn. :type person_dn: string - :return: dictionary with bool True if successful, bool False - and the error if not. - :rtype:dict + :returns: dictionary with bool True if successful, bool False + and the error if not. + :rtype: dict + """ #Connect and bind result = self.conn.connect() @@ -607,26 +639,27 @@ class LDAPapi : return {'bool': True} except ldap.LDAPError, error: - logger.log_exc("LDAP Delete Error %s" %error) + logger.log_exc("LDAP Delete Error %s" % error) return {'bool': False, 'message': error} - def LdapDeleteUser(self, record_filter): - """ - Deletes a SFA person in LDAP, based on the user's hrn. + """Deletes a SFA person in LDAP, based on the user's hrn. + :param record_filter: Filter to find the user to be deleted. Must - contain at least the user's email. + contain at least the user's email. :type record_filter: dict - :return: dict with bool True if successful, bool False and error message - otherwise - :rtype:dict - ..seealso: LdapFindUser docstring for more info on record filter. - ..seealso: LdapDelete for user deletion + :returns: dict with bool True if successful, bool False and error + message otherwise. + :rtype: dict + + .. seealso:: LdapFindUser docstring for more info on record filter. + .. seealso:: LdapDelete for user deletion + """ #Find uid of the person person = self.LdapFindUser(record_filter, []) - logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s" \ - %(record_filter, person)) + logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s" + % (record_filter, person)) if person: dn = 'uid=' + person['uid'] + "," + self.baseDN @@ -636,20 +669,21 @@ class LDAPapi : result = self.LdapDelete(dn) return result - def LdapModify(self, dn, old_attributes_dict, new_attributes_dict): """ Modifies a LDAP entry, replaces user's old attributes with the new ones given. + :param dn: user's absolute name in the LDAP hierarchy. :param old_attributes_dict: old user's attributes. Keys must match - the ones used in the LDAP model. + the ones used in the LDAP model. :param new_attributes_dict: new user's attributes. Keys must match - the ones used in the LDAP model. + the ones used in the LDAP model. :type dn: string :type old_attributes_dict: dict :type new_attributes_dict: dict - :return: dict bool True if Successful, bool False if not. - :rtype:dict + :returns: dict bool True if Successful, bool False if not. + :rtype: dict + """ ldif = modlist.modifyModlist(old_attributes_dict, new_attributes_dict) @@ -659,28 +693,32 @@ class LDAPapi : try: self.conn.ldapserv.modify_s(dn, ldif) self.conn.close() - return {'bool' : True } + return {'bool': True} except ldap.LDAPError, error: - logger.log_exc("LDAP LdapModify Error %s" %error) - return {'bool' : False } + logger.log_exc("LDAP LdapModify Error %s" % error) + return {'bool': False} def LdapModifyUser(self, user_record, new_attributes_dict): """ - Gets the record from one user based on the user sfa record - and changes the attributes according to the specified new_attributes. - Do not use this if we need to modify the uid. Use a ModRDN - #operation instead ( modify relative DN ) + + Gets the record from one user based on the user sfa recordand changes + the attributes according to the specified new_attributes. Do not use + this if we need to modify the uid. Use a ModRDN operation instead + ( modify relative DN ). + :param user_record: sfa user record. :param new_attributes_dict: new user attributes, keys must be the - same as the LDAP model. + same as the LDAP model. :type user_record: dict :type new_attributes_dict: dict - :return: bool True if successful, bool False if not. + :returns: bool True if successful, bool False if not. :rtype: dict - ..seealso: make_ldap_filters_from_record for info on what is mandatory - in the user_record. - ..seealso: make_ldap_attributes_from_record for the LDAP objectclass. + + .. seealso:: make_ldap_filters_from_record for info on what is mandatory + in the user_record. + .. seealso:: make_ldap_attributes_from_record for the LDAP objectclass. + """ if user_record is None: logger.error("LDAP \t LdapModifyUser Need user record ") @@ -690,30 +728,31 @@ class LDAPapi : #person = self.LdapFindUser(record_filter,[]) req_ldap = self.make_ldap_filters_from_record(user_record) person_list = self.LdapSearch(req_ldap, []) - logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s" \ - %(person_list)) - if person_list and len(person_list) > 1 : + logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s" + % (person_list)) + + if person_list and len(person_list) > 1: logger.error("LDAP \t LdapModifyUser Too many users returned") return {'bool': False} - if person_list is None : - logger.error("LDAP \t LdapModifyUser User %s doesn't exist "\ - %(user_record)) + if person_list is None: + logger.error("LDAP \t LdapModifyUser User %s doesn't exist " + % (user_record)) return {'bool': False} # The dn of our existing entry/object #One result only from ldapSearch person = person_list[0][1] - dn = 'uid=' + person['uid'][0] + "," + self.baseDN + dn = 'uid=' + person['uid'][0] + "," + self.baseDN if new_attributes_dict: old = {} for k in new_attributes_dict: if k not in person: - old[k] = '' - else : + old[k] = '' + else: old[k] = person[k] - logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s"\ - %( new_attributes_dict)) + logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s" + % (new_attributes_dict)) result = self.LdapModify(dn, old, new_attributes_dict) return result else: @@ -721,17 +760,21 @@ class LDAPapi : return {'bool': False} - - def LdapMarkUserAsDeleted(self, record): """ - Sets shadowExpire to 0, disabling the user in LDAP. - Calls LdapModifyUser to change the shadowExpire of the user. + + Sets shadowExpire to 0, disabling the user in LDAP. Calls LdapModifyUser + to change the shadowExpire of the user. + :param record: the record of the user who has to be disabled. + Should contain first_name,last_name, email or mail, and if the + record is enabled or not. If the dict record does not have all of + these, must at least contain the user's email. :type record: dict - :return: bool True if successful or bool False if not - :rtype:dict - ..seealso: LdapModifyUser + :returns: {bool: True} if successful or {bool: False} if not + :rtype: dict + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record """ new_attrs = {} @@ -741,40 +784,58 @@ class LDAPapi : ret = self.LdapModifyUser(record, new_attrs) return ret - def LdapResetPassword(self, record): - """ - Resets password for the user whose record is the parameter and changes - the corresponding entry in the LDAP. + """Resets password for the user whose record is the parameter and + changes the corresponding entry in the LDAP. + + :param record: user's sfa record whose Ldap password must be reset. + Should contain first_name,last_name, + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. + :type record: dict + :returns: return value of LdapModifyUser. True if successful, False + otherwise. + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record """ password = self.login_pwd.generate_password() attrs = {} attrs['userPassword'] = self.login_pwd.encrypt_password(password) - logger.debug("LDAP LdapResetPassword encrypt_password %s"\ - %(attrs['userPassword'])) + logger.debug("LDAP LdapResetPassword encrypt_password %s" + % (attrs['userPassword'])) result = self.LdapModifyUser(record, attrs) return result - def LdapSearch (self, req_ldap = None, expected_fields = None ): + def LdapSearch(self, req_ldap=None, expected_fields=None): """ - Used to search directly in LDAP, by using ldap filters and - return fields. - When req_ldap is None, returns all the entries in the LDAP. + Used to search directly in LDAP, by using ldap filters and return + fields. When req_ldap is None, returns all the entries in the LDAP. + + :param req_ldap: ldap style request, with appropriate filters, + example: (cn=*). + :param expected_fields: Fields in the user ldap entry that has to be + returned. If None is provided, will return 'mail', 'givenName', + 'sn', 'uid', 'sshPublicKey', 'shadowExpire'. + :type req_ldap: string + :type expected_fields: list + + .. seealso:: make_ldap_filters_from_record for req_ldap format. """ - result = self.conn.connect(bind = False) - if (result['bool']) : + result = self.conn.connect(bind=False) + if (result['bool']): return_fields_list = [] - if expected_fields == None : - return_fields_list = ['mail', 'givenName', 'sn', 'uid', \ - 'sshPublicKey', 'shadowExpire'] - else : + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey', 'shadowExpire'] + else: return_fields_list = expected_fields #No specifc request specified, get the whole LDAP - if req_ldap == None: + if req_ldap is None: req_ldap = '(cn=*)' logger.debug("LDAP.PY \t LdapSearch req_ldap %s \ @@ -783,43 +844,44 @@ class LDAPapi : try: msg_id = self.conn.ldapserv.search( - self.baseDN,ldap.SCOPE_SUBTREE,\ - req_ldap, return_fields_list) + self.baseDN, ldap.SCOPE_SUBTREE, + req_ldap, return_fields_list) #Get all the results matching the search from ldap in one #shot (1 value) result_type, result_data = \ - self.conn.ldapserv.result(msg_id, 1) + self.conn.ldapserv.result(msg_id, 1) self.conn.close() - logger.debug("LDAP.PY \t LdapSearch result_data %s"\ - %(result_data)) + logger.debug("LDAP.PY \t LdapSearch result_data %s" + % (result_data)) return result_data - except ldap.LDAPError, error : - logger.log_exc("LDAP LdapSearch Error %s" %error) + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapSearch Error %s" % error) return [] else: - logger.error("LDAP.PY \t Connection Failed" ) + logger.error("LDAP.PY \t Connection Failed") return - def _process_ldap_info_for_all_users(self, result_data): - """ - Process the data of all enabled users in LDAP. + """Process the data of all enabled users in LDAP. + :param result_data: Contains information of all enabled users in LDAP - and is coming from LdapSearch. + and is coming from LdapSearch. :param result_data: list - ..seealso: LdapSearch + + .. seealso:: LdapSearch + """ results = [] - logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s " \ - %(result_data)) + logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s " + % (result_data)) for ldapentry in result_data: - logger.debug(" LDAP.py _process_ldap_info_for_all_users ldapentry name : %s " \ - %(ldapentry[1]['uid'][0])) + logger.debug(" LDAP.py _process_ldap_info_for_all_users \ + ldapentry name : %s " % (ldapentry[1]['uid'][0])) tmpname = ldapentry[1]['uid'][0] hrn = self.authname + "." + tmpname @@ -827,48 +889,50 @@ class LDAPapi : if ldapentry[1]['mail'][0] == "unknown": tmpemail = None - try: - results.append( { - 'type': 'user', - 'pkey': ldapentry[1]['sshPublicKey'][0], - #'uid': ldapentry[1]['uid'][0], - 'uid': tmpname , - 'email':tmpemail, - #'email': ldapentry[1]['mail'][0], - 'first_name': ldapentry[1]['givenName'][0], - 'last_name': ldapentry[1]['sn'][0], - #'phone': 'none', - 'serial': 'none', - 'authority': self.authname, - 'peer_authority': '', - 'pointer' : -1, - 'hrn': hrn, - } ) + results.append({ + 'type': 'user', + 'pkey': ldapentry[1]['sshPublicKey'][0], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname , + 'email':tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry[1]['givenName'][0], + 'last_name': ldapentry[1]['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': self.authname, + 'peer_authority': '', + 'pointer': -1, + 'hrn': hrn, + }) except KeyError, error: - logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s" \ - %(error)) + logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s" + % (error)) return return results def _process_ldap_info_for_one_user(self, record, result_data): """ + Put the user's ldap data into shape. Only deals with one user record and one user data from ldap. + :param record: user record :param result_data: Raw ldap data coming from LdapSearch - :return: user's data dict with 'type','pkey','uid', 'email', - 'first_name' 'last_name''serial''authority''peer_authority' - 'pointer''hrn' + :returns: user's data dict with 'type','pkey','uid', 'email', + 'first_name' 'last_name''serial''authority''peer_authority' + 'pointer''hrn' :type record: dict :type result_data: list :rtype :dict + """ #One entry only in the ldap data because we used a filter #to find one user only ldapentry = result_data[0][1] - logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" %(ldapentry)) + logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" % (ldapentry)) tmpname = ldapentry['uid'][0] tmpemail = ldapentry['mail'][0] @@ -891,42 +955,42 @@ class LDAPapi : else: hrn = None - - - results = { - 'type': 'user', - 'pkey': ldapentry['sshPublicKey'], - #'uid': ldapentry[1]['uid'][0], - 'uid': tmpname , - 'email':tmpemail, - #'email': ldapentry[1]['mail'][0], - 'first_name': ldapentry['givenName'][0], - 'last_name': ldapentry['sn'][0], - #'phone': 'none', - 'serial': 'none', - 'authority': parent_hrn, - 'peer_authority': peer_authority, - 'pointer' : -1, - 'hrn': hrn, + results = { + 'type': 'user', + 'pkey': ldapentry['sshPublicKey'], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname, + 'email': tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry['givenName'][0], + 'last_name': ldapentry['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': parent_hrn, + 'peer_authority': peer_authority, + 'pointer': -1, + 'hrn': hrn, } return results - def LdapFindUser(self, record=None, is_user_enabled=None, expected_fields=None): """ + Search a SFA user with a hrn. User should be already registered in Iotlab LDAP. + :param record: sfa user's record. Should contain first_name,last_name, - email or mail. If no record is provided, returns all the users found - in LDAP. + email or mail. If no record is provided, returns all the users found + in LDAP. :type record: dict :param is_user_enabled: is the user's iotlab account already valid. :type is_user_enabled: Boolean. - :return: LDAP entries from ldap matching the filter provided. Returns - a single entry if one filter has been given and a list of - entries otherwise. + :returns: LDAP entries from ldap matching the filter provided. Returns + a single entry if one filter has been given and a list of + entries otherwise. :rtype: dict or list + """ custom_record = {} if is_user_enabled: @@ -934,19 +998,18 @@ class LDAPapi : if record: custom_record.update(record) - req_ldap = self.make_ldap_filters_from_record(custom_record) return_fields_list = [] - if expected_fields == None : - return_fields_list = ['mail', 'givenName', 'sn', 'uid', \ - 'sshPublicKey'] - else : + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey'] + else: return_fields_list = expected_fields - result_data = self.LdapSearch(req_ldap, return_fields_list ) - logger.debug("LDAP.PY \t LdapFindUser result_data %s" %(result_data)) + result_data = self.LdapSearch(req_ldap, return_fields_list) + logger.debug("LDAP.PY \t LdapFindUser result_data %s" % (result_data)) - if len(result_data) is 0: + if len(result_data) == 0: return None #Asked for a specific user if record is not None: @@ -955,5 +1018,4 @@ class LDAPapi : else: #Asked for all users in ldap results = self._process_ldap_info_for_all_users(result_data) - return results - + return results \ No newline at end of file diff --git a/sfa/iotlab/OARrestapi.py b/sfa/iotlab/OARrestapi.py index 0223f472..18679080 100644 --- a/sfa/iotlab/OARrestapi.py +++ b/sfa/iotlab/OARrestapi.py @@ -1,32 +1,35 @@ -#import sys +""" +File used to handle issuing request to OAR and parse OAR's JSON responses. +Contains the following classes: +- JsonPage : handles multiple pages OAR answers. +- OARRestapi : handles issuing POST or GET requests to OAR. +- ParsingResourcesFull : dedicated to parsing OAR's answer to a get resources +full request. +- OARGETParser : handles parsing the Json answers to different GET requests. + +""" from httplib import HTTPConnection, HTTPException, NotConnected import json -#import datetime -#from time import gmtime, strftime -import os.path -import sys -#import urllib -#import urllib2 from sfa.util.config import Config -#from sfa.util.xrn import hrn_to_urn, get_authority, Xrn, get_leaf - from sfa.util.sfalogging import logger +import os.path -OAR_REQUEST_POST_URI_DICT = {'POST_job':{'uri': '/oarapi/jobs.json'}, - 'DELETE_jobs_id':{'uri':'/oarapi/jobs/id.json'}, - } - -POST_FORMAT = {'json' : {'content':"application/json", 'object':json},} +class JsonPage: -#OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ - #'workdir':"/home/", 'walltime':""} + """Class used to manipulate json pages given by OAR. + In case the json answer from a GET request is too big to fit in one json + page, this class provides helper methods to retrieve all the pages and + store them in a list before putting them into one single json dictionary, + facilitating the parsing. + """ -class JsonPage: - """Class used to manipulate jsopn pages given by OAR.""" def __init__(self): + """Defines attributes to manipulate and parse the json pages. + + """ #All are boolean variables self.concatenate = False #Indicates end of data, no more pages to be loaded. @@ -38,11 +41,12 @@ class JsonPage: self.raw_json = None def FindNextPage(self): - """ Gets next data page from OAR when the query's results - are too big to be transmitted in a single page. - Uses the "links' item in the json returned to check if - an additionnal page has to be loaded. - Returns : next page , next offset query + """ + Gets next data page from OAR when the query's results are too big to + be transmitted in a single page. Uses the "links' item in the json + returned to check if an additionnal page has to be loaded. Updates + object attributes next_page, next_offset, and end. + """ if "links" in self.raw_json: for page in self.raw_json['links']: @@ -50,10 +54,9 @@ class JsonPage: self.concatenate = True self.next_page = True self.next_offset = "?" + page['href'].split("?")[1] - print>>sys.stderr, "\r\n \t FindNextPage NEXT LINK" return - if self.concatenate : + if self.concatenate: self.end = True self.next_page = False self.next_offset = None @@ -70,6 +73,24 @@ class JsonPage: @staticmethod def ConcatenateJsonPages(saved_json_list): + """ + If the json answer is too big to be contained in a single page, + all the pages have to be loaded and saved before being appended to the + first page. + + :param saved_json_list: list of all the stored pages, including the + first page. + :type saved_json_list: list + :returns: Returns a dictionary with all the pages saved in the + saved_json_list. The key of the dictionary is 'items'. + :rtype: dict + + + .. seealso:: SendRequest + .. warning:: Assumes the apilib is 0.2.10 (with the 'items' key in the + raw json dictionary) + + """ #reset items list tmp = {} @@ -79,8 +100,13 @@ class JsonPage: tmp['items'].extend(page['items']) return tmp - def ResetNextPage(self): + """ + Resets all the Json page attributes (next_page, next_offset, + concatenate, end). Has to be done before getting another json answer + so that the previous page status does not affect the new json load. + + """ self.next_page = True self.next_offset = None self.concatenate = False @@ -88,10 +114,25 @@ class JsonPage: class OARrestapi: - def __init__(self, config_file = '/etc/sfa/oar_config.py'): - self.oarserver = {} + """Class used to connect to the OAR server and to send GET and POST + requests. + + """ + + # classes attributes + + OAR_REQUEST_POST_URI_DICT = {'POST_job': {'uri': '/oarapi/jobs.json'}, + 'DELETE_jobs_id': + {'uri': '/oarapi/jobs/id.json'}, + } + + POST_FORMAT = {'json': {'content': "application/json", 'object': json}} + #OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ + #'workdir':"/home/", 'walltime':""} + def __init__(self, config_file='/etc/sfa/oar_config.py'): + self.oarserver = {} self.oarserver['uri'] = None self.oarserver['postformat'] = 'json' @@ -104,30 +145,57 @@ class OARrestapi: except IOError: raise IOError, "Could not find or load the configuration file: %s" \ - % config_file + % config_file #logger.setLevelDebug() self.oarserver['ip'] = self.OAR_IP self.oarserver['port'] = self.OAR_PORT - self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', \ - 'toError', 'toAckReservation', 'Launching', \ - 'Finishing', 'Running', 'Suspended', 'Resuming',\ - 'Error'] + self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', + 'toError', 'toAckReservation', 'Launching', + 'Finishing', 'Running', 'Suspended', 'Resuming', + 'Error'] self.parser = OARGETParser(self) - def GETRequestToOARRestAPI(self, request, strval=None, next_page=None, username = None ): + def GETRequestToOARRestAPI(self, request, strval=None, + next_page=None, username=None): + + """Makes a GET request to OAR. + + Fetch the uri associated with the resquest stored in + OARrequests_uri_dict, adds the username if needed and if available, adds + strval to the request uri if needed, connects to OAR and issues the GET + request. Gets the json reply. + + :param request: One of the known get requests that are keys in the + OARrequests_uri_dict. + :param strval: used when a job id has to be specified. + :param next_page: used to tell OAR to send the next page for this + Get request. Is appended to the GET uri. + :param username: used when a username has to be specified, when looking + for jobs scheduled by a particular user for instance. + + :type request: string + :type strval: integer + :type next_page: boolean + :type username: string + :returns: a json dictionary if OAR successfully processed the GET + request. + + .. seealso:: OARrequests_uri_dict + """ self.oarserver['uri'] = \ - OARGETParser.OARrequests_uri_dict[request]['uri'] + OARGETParser.OARrequests_uri_dict[request]['uri'] #Get job details with username if 'owner' in OARGETParser.OARrequests_uri_dict[request] and username: - self.oarserver['uri'] += OARGETParser.OARrequests_uri_dict[request]['owner'] + username + self.oarserver['uri'] += \ + OARGETParser.OARrequests_uri_dict[request]['owner'] + username headers = {} data = json.dumps({}) - logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" %(request)) + logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" % (request)) if strval: self.oarserver['uri'] = self.oarserver['uri'].\ - replace("id",str(strval)) + replace("id", str(strval)) if next_page: self.oarserver['uri'] += next_page @@ -136,21 +204,21 @@ class OARrestapi: headers['X-REMOTE_IDENT'] = username logger.debug("OARrestapi: \t GETRequestToOARRestAPI \ - self.oarserver['uri'] %s strval %s" \ - %(self.oarserver['uri'], strval)) - try : + self.oarserver['uri'] %s strval %s" + % (self.oarserver['uri'], strval)) + try: #seems that it does not work if we don't add this headers['content-length'] = '0' - conn = HTTPConnection(self.oarserver['ip'], \ - self.oarserver['port']) + conn = HTTPConnection(self.oarserver['ip'], + self.oarserver['port']) conn.request("GET", self.oarserver['uri'], data, headers) - resp = ( conn.getresponse()).read() + resp = (conn.getresponse()).read() conn.close() - except HTTPException, error : - logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " \ - %(error)) + except HTTPException, error: + logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " + % (error)) #raise ServerError("GET_OAR_SRVR : Could not reach OARserver") try: js_dict = json.loads(resp) @@ -158,8 +226,8 @@ class OARrestapi: return js_dict except ValueError, error: - logger.log_exc("Failed to parse Server Response: %s ERROR %s"\ - %(js_dict, error)) + logger.log_exc("Failed to parse Server Response: %s ERROR %s" + % (js_dict, error)) #raise ServerError("Failed to parse Server Response:" + js) @@ -171,7 +239,8 @@ class OARrestapi: #first check that all params for are OK try: - self.oarserver['uri'] = OAR_REQUEST_POST_URI_DICT[request]['uri'] + self.oarserver['uri'] = \ + self.OAR_REQUEST_POST_URI_DICT[request]['uri'] except KeyError: logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \ @@ -184,7 +253,7 @@ class OARrestapi: data = json.dumps(datadict) headers = {'X-REMOTE_IDENT':username, \ - 'content-type': POST_FORMAT['json']['content'], \ + 'content-type': self.POST_FORMAT['json']['content'], \ 'content-length':str(len(data))} try : @@ -202,7 +271,7 @@ class OARrestapi: try: answer = json.loads(resp) - logger.debug("POSTRequestToOARRestAPI : answer %s" %(answer)) + logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer)) return answer except ValueError, error: @@ -211,79 +280,229 @@ class OARrestapi: #raise ServerError("Failed to parse Server Response:" + answer) +class ParsingResourcesFull(): + """ + Class dedicated to parse the json response from a GET_resources_full from + OAR. + + """ + def __init__(self): + """ + Set the parsing dictionary. Works like a switch case, if the key is + found in the dictionary, then the associated function is called. + This is used in ParseNodes to create an usable dictionary from + the Json returned by OAR when issuing a GET resources full request. + + .. seealso:: ParseNodes + + """ + self.resources_fulljson_dict = { + 'network_address': self.AddNodeNetworkAddr, + 'site': self.AddNodeSite, + # 'radio': self.AddNodeRadio, + 'mobile': self.AddMobility, + 'x': self.AddPosX, + 'y': self.AddPosY, + 'z': self.AddPosZ, + 'archi': self.AddHardwareType, + 'state': self.AddBootState, + 'id': self.AddOarNodeId, + } + + + + def AddOarNodeId(self, tuplelist, value): + """Adds Oar internal node id to the nodes' attributes. + + Appends tuple ('oar_id', node_id) to the tuplelist. Used by ParseNodes. + + .. seealso:: ParseNodes + + """ + + tuplelist.append(('oar_id', int(value))) + + + def AddNodeNetworkAddr(self, dictnode, value): + """First parsing function to be called to parse the json returned by OAR + answering a GET_resources (/oarapi/resources.json) request. + + When a new node is found in the json, this function is responsible for + creating a new entry in the dictionary for storing information on this + specific node. The key is the node network address, which is also the + node's hostname. + The value associated with the key is a tuple list.It contains all + the nodes attributes. The tuplelist will later be turned into a dict. + + :param dictnode: should be set to the OARGETParser atribute + node_dictlist. It will store the information on the nodes. + :param value: the node_id is the network_address in the raw json. + :type value: string + :type dictnode: dictionary + + .. seealso: ParseResources, ParseNodes + """ + + node_id = value + dictnode[node_id] = [('node_id', node_id),('hostname', node_id) ] + + return node_id + + def AddNodeSite(self, tuplelist, value): + """Add the site's node to the dictionary. + + + :param tuplelist: tuple list on which to add the node's site. + Contains the other node attributes as well. + :param value: value to add to the tuple list, in this case the node's + site. + :type tuplelist: list + :type value: string + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('site', str(value))) + + # def AddNodeRadio(tuplelist, value): + # """Add thenode's radio chipset type to the tuple list. + + # :param tuplelist: tuple list on which to add the node's mobility + # status. The tuplelist is the value associated with the node's + # id in the OARGETParser + # 's dictionary node_dictlist. + # :param value: name of the radio chipset on the node. + # :type tuplelist: list + # :type value: string + + # .. seealso:: AddNodeNetworkAddr + + # """ + # tuplelist.append(('radio', str(value))) + + + def AddMobility(self, tuplelist, value): + """Add if the node is a mobile node or not to the tuple list. + + :param tuplelist: tuple list on which to add the node's mobility status. + The tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: tells if a node is a mobile node or not. The value is found + in the json. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + if value is 0: + tuplelist.append(('mobile', 'False')) + else: + tuplelist.append(('mobile', 'True')) + + + def AddPosX(self, tuplelist, value): + """Add the node's position on the x axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position x. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('posx', value )) + + + + def AddPosY(self, tuplelist, value): + """Add the node's position on the y axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position y. -def AddOarNodeId(tuplelist, value): - """ Adds Oar internal node id to the nodes attributes """ + :type tuplelist: list + :type value: integer - tuplelist.append(('oar_id', int(value))) + .. seealso:: AddNodeNetworkAddr + """ + tuplelist.append(('posy', value)) + + + + def AddPosZ(self, tuplelist, value): + """Add the node's position on the z axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position z. -def AddNodeNetworkAddr(dictnode, value): - #Inserts new key. The value associated is a tuple list - node_id = value + :type tuplelist: list + :type value: integer - dictnode[node_id] = [('node_id', node_id),('hostname', node_id) ] + .. seealso:: AddNodeNetworkAddr - return node_id + """ + + tuplelist.append(('posz', value)) -def AddNodeSite(tuplelist, value): - tuplelist.append(('site', str(value))) -def AddNodeRadio(tuplelist, value): - tuplelist.append(('radio', str(value))) + def AddBootState(tself, tuplelist, value): + """Add the node's state, Alive or Suspected. -def AddMobility(tuplelist, value): - if value is 0: - tuplelist.append(('mobile', 'False')) - else : - tuplelist.append(('mobile', 'True')) + :param tuplelist: tuple list on which to add the node's state . The + tuplelist is the value associated with the node's id in the + OARGETParser 's dictionary node_dictlist. + :param value: node's state. -def AddPosX(tuplelist, value): - tuplelist.append(('posx', value)) + :type tuplelist: list + :type value: string -def AddPosY(tuplelist, value): - tuplelist.append(('posy', value)) + .. seealso:: AddNodeNetworkAddr -def AddPosZ(tuplelist, value): - tuplelist.append(('posz', value)) + """ + tuplelist.append(('boot_state', str(value))) -def AddBootState(tuplelist, value): - tuplelist.append(('boot_state', str(value))) -#Insert a new node into the dictnode dictionary -def AddNodeId(dictnode, value): - #Inserts new key. The value associated is a tuple list - node_id = int(value) + def AddHardwareType(self, tuplelist, value): + """Add the node's hardware model and radio chipset type to the tuple + list. - dictnode[node_id] = [('node_id', node_id)] - return node_id + :param tuplelist: tuple list on which to add the node's architecture + and radio chipset type. + :param value: hardware type: radio chipset. The value contains both the + architecture and the radio chipset, separated by a colon. + :type tuplelist: list + :type value: string -def AddHardwareType(tuplelist, value): - value_list = value.split(':') - tuplelist.append(('archi', value_list[0])) - tuplelist.append(('radio', value_list[1])) + .. seealso:: AddNodeNetworkAddr + + """ + + value_list = value.split(':') + tuplelist.append(('archi', value_list[0])) + tuplelist.append(('radio', value_list[1])) class OARGETParser: - resources_fulljson_dict = { - 'network_address' : AddNodeNetworkAddr, - 'site': AddNodeSite, - 'radio': AddNodeRadio, - 'mobile': AddMobility, - 'x': AddPosX, - 'y': AddPosY, - 'z':AddPosZ, - 'archi':AddHardwareType, - 'state':AddBootState, - 'id' : AddOarNodeId, - } + """Class providing parsing methods associated to specific GET requests. + """ - def __init__(self, srv) : + def __init__(self, srv): self.version_json_dict = { - 'api_version' : None , 'apilib_version' :None,\ - 'api_timezone': None, 'api_timestamp': None, 'oar_version': None ,} + 'api_version': None, 'apilib_version': None, + 'api_timezone': None, 'api_timestamp': None, 'oar_version': None} self.config = Config() self.interface_hrn = self.config.SFA_INTERFACE_HRN self.timezone_json_dict = { @@ -297,70 +516,116 @@ class OARGETParser: self.node_dictlist = {} self.json_page = JsonPage() - + self.parsing_resourcesfull = ParsingResourcesFull() self.site_dict = {} + self.jobs_list = [] self.SendRequest("GET_version") + def ParseVersion(self): + """Parses the OAR answer to the GET_version ( /oarapi/version.json.) + Finds the OAR apilib version currently used. Has an impact on the json + structure returned by OAR, so the version has to be known before trying + to parse the jsons returned after a get request has been issued. + Updates the attribute version_json_dict. + """ - def ParseVersion(self) : - #print self.json_page.raw_json - #print >>sys.stderr, self.json_page.raw_json - if 'oar_version' in self.json_page.raw_json : - self.version_json_dict.update(api_version = \ - self.json_page.raw_json['api_version'], - apilib_version = self.json_page.raw_json['apilib_version'], - api_timezone = self.json_page.raw_json['api_timezone'], - api_timestamp = self.json_page.raw_json['api_timestamp'], - oar_version = self.json_page.raw_json['oar_version'] ) - else : - self.version_json_dict.update(api_version = \ - self.json_page.raw_json['api'] , - apilib_version = self.json_page.raw_json['apilib'], - api_timezone = self.json_page.raw_json['api_timezone'], - api_timestamp = self.json_page.raw_json['api_timestamp'], - oar_version = self.json_page.raw_json['oar'] ) + if 'oar_version' in self.json_page.raw_json: + self.version_json_dict.update( + api_version=self.json_page.raw_json['api_version'], + apilib_version=self.json_page.raw_json['apilib_version'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar_version']) + else: + self.version_json_dict.update( + api_version=self.json_page.raw_json['api'], + apilib_version=self.json_page.raw_json['apilib'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar']) print self.version_json_dict['apilib_version'] - def ParseTimezone(self) : + def ParseTimezone(self): + """Get the timezone used by OAR. + + Get the timezone from the answer to the GET_timezone request. + :return: api_timestamp and api timezone. + :rype: integer, integer + + .. warning:: unused. + """ api_timestamp = self.json_page.raw_json['api_timestamp'] api_tz = self.json_page.raw_json['timezone'] return api_timestamp, api_tz - def ParseJobs(self) : + def ParseJobs(self): + """Called when a GET_jobs request has been issued to OAR. + + Corresponds to /oarapi/jobs.json uri. Currently returns the raw json + information dict. + :returns: json_page.raw_json + :rtype: dictionary + + .. warning:: Does not actually parse the information in the json. SA + 15/07/13. + + """ self.jobs_list = [] print " ParseJobs " return self.json_page.raw_json - def ParseJobsTable(self) : + def ParseJobsTable(self): + """In case we need to use the job table in the future. + + Associated with the GET_jobs_table : '/oarapi/jobs/table.json uri. + .. warning:: NOT USED. DOES NOTHING. + """ print "ParseJobsTable" - def ParseJobsDetails (self): - # currently, this function is not used a lot, - #so i have no idea what be usefull to parse, - #returning the full json. NT + def ParseJobsDetails(self): + """Currently only returns the same json in self.json_page.raw_json. + + .. todo:: actually parse the json + .. warning:: currently, this function is not used a lot, so I have no + idea what could be useful to parse, returning the full json. NT + """ + #logger.debug("ParseJobsDetails %s " %(self.json_page.raw_json)) return self.json_page.raw_json def ParseJobsIds(self): + """Associated with the GET_jobs_id OAR request. + + Parses the json dict (OAR answer) to the GET_jobs_id request + /oarapi/jobs/id.json. - job_resources = ['wanted_resources', 'name', 'id', 'start_time', \ - 'state','owner','walltime','message'] + :returns: dictionary whose keys are listed in the local variable + job_resources and values that are in the json dictionary returned + by OAR with the job information. + :rtype: dict - job_resources_full = ['launching_directory', 'links', \ - 'resubmit_job_id', 'owner', 'events', 'message', \ - 'scheduled_start', 'id', 'array_id', 'exit_code', \ - 'properties', 'state','array_index', 'walltime', \ - 'type', 'initial_request', 'stop_time', 'project',\ - 'start_time', 'dependencies','api_timestamp','submission_time', \ - 'reservation', 'stdout_file', 'types', 'cpuset_name', \ - 'name', 'wanted_resources','queue','stderr_file','command'] + """ + job_resources = ['wanted_resources', 'name', 'id', 'start_time', + 'state', 'owner', 'walltime', 'message'] + + # Unused variable providing the contents of the json dict returned from + # get job resources full request + job_resources_full = [ + 'launching_directory', 'links', + 'resubmit_job_id', 'owner', 'events', 'message', + 'scheduled_start', 'id', 'array_id', 'exit_code', + 'properties', 'state', 'array_index', 'walltime', + 'type', 'initial_request', 'stop_time', 'project', + 'start_time', 'dependencies', 'api_timestamp', 'submission_time', + 'reservation', 'stdout_file', 'types', 'cpuset_name', + 'name', 'wanted_resources', 'queue', 'stderr_file', 'command'] job_info = self.json_page.raw_json @@ -386,10 +651,9 @@ class OARGETParser: for resource in self.json_page.raw_json['items']: job_resources.append(resource['id']) - #logger.debug("OARESTAPI \tParseJobsIdResources %s" %(self.json_page.raw_json)) return job_resources - def ParseResources(self) : + def ParseResources(self): """ Parses the json produced by a get_resources request on oar.""" #logger.debug("OARESTAPI \tParseResources " ) @@ -398,31 +662,38 @@ class OARGETParser: self.ParseNodes() def ParseReservedNodes(self): - """ Returns an array containing the list of the reserved nodes """ + """ Returns an array containing the list of the jobs scheduled + with the reserved nodes if available. + + :returns: list of job dicts, each dict containing the following keys: + t_from, t_until, resources_ids (of the reserved nodes for this job). + If the information is not available, default values will be set for + these keys. The other keys are : state, lease_id and user. + :rtype: list + + """ #resources are listed inside the 'items' list from the json reservation_list = [] job = {} #Parse resources info - for json_element in self.json_page.raw_json['items']: + for json_element in self.json_page.raw_json['items']: #In case it is a real reservation (not asap case) if json_element['scheduled_start']: job['t_from'] = json_element['scheduled_start'] job['t_until'] = int(json_element['scheduled_start']) + \ - int(json_element['walltime']) + int(json_element['walltime']) #Get resources id list for the job - job['resource_ids'] = \ - [ node_dict['id'] for node_dict in json_element['resources']] + job['resource_ids'] = [node_dict['id'] for node_dict + in json_element['resources']] else: job['t_from'] = "As soon as possible" job['t_until'] = "As soon as possible" job['resource_ids'] = ["Undefined"] - job['state'] = json_element['state'] job['lease_id'] = json_element['id'] - job['user'] = json_element['owner'] #logger.debug("OARRestapi \tParseReservedNodes job %s" %(job)) reservation_list.append(job) @@ -434,16 +705,34 @@ class OARGETParser: """ Gets the list of nodes currently in use from the attributes of the running jobs. + :returns: list of hostnames, the nodes that are currently involved in + running jobs. + :rtype: list + + """ - logger.debug("OARESTAPI \tParseRunningJobs__________________________ ") + logger.debug("OARESTAPI \tParseRunningJobs_________________ ") #resources are listed inside the 'items' list from the json nodes = [] - for job in self.json_page.raw_json['items']: + for job in self.json_page.raw_json['items']: for node in job['nodes']: nodes.append(node['network_address']) return nodes + def ChangeRawJsonDependingOnApilibVersion(self): + """ + Check if the OAR apilib version is different from 0.2.10, in which case + the Json answer is also dict instead as a plain list. + + .. warning:: the whole code is assuming the json contains a 'items' key + .. seealso:: ConcatenateJsonPages, ParseJobs, ParseReservedNodes, + ParseJobsIdResources, ParseResources, ParseRunningJobs + .. todo:: Clean the whole code. Either suppose the apilib will always + provide the 'items' key, or handle different options. + """ + if self.version_json_dict['apilib_version'] != "0.2.10": + self.json_page.raw_json = self.json_page.raw_json['items'] def ParseDeleteJobs(self): """ No need to parse anything in this function.A POST @@ -452,36 +741,44 @@ class OARGETParser: """ return - def ParseResourcesFull(self) : + def ParseResourcesFull(self): """ This method is responsible for parsing all the attributes of all the nodes returned by OAR when issuing a get resources full. The information from the nodes and the sites are separated. Updates the node_dictlist so that the dictionnary of the platform's nodes is available afterwards. + :returns: node_dictlist, a list of dictionaries about the nodes and + their properties. + :rtype: list + """ - logger.debug("OARRESTAPI ParseResourcesFull________________________ ") + logger.debug("OARRESTAPI ParseResourcesFull___________ ") #print self.json_page.raw_json[1] #resources are listed inside the 'items' list from the json - if self.version_json_dict['apilib_version'] != "0.2.10" : - self.json_page.raw_json = self.json_page.raw_json['items'] + self.ChangeRawJsonDependingOnApilibVersion() self.ParseNodes() self.ParseSites() return self.node_dictlist - def ParseResourcesFullSites(self) : - """ UNUSED. Originally used to get information from the sites. - ParseResourcesFull is used instead. + def ParseResourcesFullSites(self): + """ Called by GetSites which is unused. + Originally used to get information from the sites, with for each site + the list of nodes it has, along with their properties. + + :return: site_dict, dictionary of sites + :rtype: dict + + .. warning:: unused + .. seealso:: GetSites (IotlabTestbedAPI) """ - if self.version_json_dict['apilib_version'] != "0.2.10" : - self.json_page.raw_json = self.json_page.raw_json['items'] + self.ChangeRawJsonDependingOnApilibVersion() self.ParseNodes() self.ParseSites() return self.site_dict - def ParseNodes(self): """ Parse nodes properties from OAR Put them into a dictionary with key = node id and value is a dictionary @@ -489,21 +786,23 @@ class OARGETParser: """ node_id = None - keys = self.resources_fulljson_dict.keys() + _resources_fulljson_dict = \ + self.parsing_resourcesfull.resources_fulljson_dict + keys = _resources_fulljson_dict.keys() keys.sort() for dictline in self.json_page.raw_json: node_id = None # dictionary is empty and/or a new node has to be inserted - node_id = self.resources_fulljson_dict['network_address'](\ - self.node_dictlist, dictline['network_address']) + node_id = _resources_fulljson_dict['network_address']( + self.node_dictlist, dictline['network_address']) for k in keys: if k in dictline: if k == 'network_address': continue - self.resources_fulljson_dict[k](\ - self.node_dictlist[node_id], dictline[k]) + _resources_fulljson_dict[k]( + self.node_dictlist[node_id], dictline[k]) #The last property has been inserted in the property tuple list, #reset node_id @@ -512,10 +811,20 @@ class OARGETParser: node_id = None @staticmethod - def iotlab_hostname_to_hrn( root_auth, hostname): - return root_auth + '.'+ hostname + def iotlab_hostname_to_hrn(root_auth, hostname): + """ + Transforms a node hostname into a SFA hrn. + :param root_auth: Name of the root authority of the SFA server. In + our case, it is set to iotlab. + :param hostname: node's hotname, given by OAR. + :type root_auth: string + :type hostname: string + :returns: inserts the root_auth and '.' before the hostname. + :rtype: string + """ + return root_auth + '.' + hostname def ParseSites(self): """ Returns a list of dictionnaries containing the sites' attributes.""" @@ -526,7 +835,7 @@ class OARGETParser: #%(self.node_dictlist)) # Create a list of nodes per site_id for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] + node = self.node_dictlist[node_id] if node['site'] not in nodes_per_site: nodes_per_site[node['site']] = [] @@ -535,112 +844,110 @@ class OARGETParser: if node['node_id'] not in nodes_per_site[node['site']]: nodes_per_site[node['site']].append(node['node_id']) - #Create a site dictionary whose key is site_login_base (name of the site) - # and value is a dictionary of properties, including the list - #of the node_ids + #Create a site dictionary whose key is site_login_base + # (name of the site) and value is a dictionary of properties, + # including the list of the node_ids for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] - #node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, \ - #node['site'],node['hostname'])}) - node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, node['hostname'])}) - self.node_dictlist.update({node_id:node}) + node = self.node_dictlist[node_id] + node.update({'hrn': self.iotlab_hostname_to_hrn(self.interface_hrn, + node['hostname'])}) + self.node_dictlist.update({node_id: node}) if node['site'] not in self.site_dict: self.site_dict[node['site']] = { - 'site':node['site'], - 'node_ids':nodes_per_site[node['site']], - 'latitude':"48.83726", - 'longitude':"- 2.10336",'name':config.SFA_REGISTRY_ROOT_AUTH, - 'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - 'max_slivers':None, 'is_public':True, 'peer_site_id': None, - 'abbreviated_name':"iotlab", 'address_ids': [], - 'url':"http,//www.senslab.info", 'person_ids':[], - 'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - 'date_created': None, 'peer_id': None } - #if node['site_login_base'] not in self.site_dict.keys(): - #self.site_dict[node['site_login_base']] = {'login_base':node['site_login_base'], - #'node_ids':nodes_per_site[node['site_login_base']], - #'latitude':"48.83726", - #'longitude':"- 2.10336",'name':"senslab", - #'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - #'max_slivers':None, 'is_public':True, 'peer_site_id': None, - #'abbreviated_name':"senslab", 'address_ids': [], - #'url':"http,//www.senslab.info", 'person_ids':[], - #'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - #'date_created': None, 'peer_id': None } - - - + 'site': node['site'], + 'node_ids': nodes_per_site[node['site']], + 'latitude': "48.83726", + 'longitude': "- 2.10336", + 'name': config.SFA_REGISTRY_ROOT_AUTH, + 'pcu_ids': [], 'max_slices': None, + 'ext_consortium_id': None, + 'max_slivers': None, 'is_public': True, + 'peer_site_id': None, + 'abbreviated_name': "iotlab", 'address_ids': [], + 'url': "https://portal.senslab.info", 'person_ids': [], + 'site_tag_ids': [], 'enabled': True, 'slice_ids': [], + 'date_created': None, 'peer_id': None + } OARrequests_uri_dict = { 'GET_version': - {'uri':'/oarapi/version.json', 'parse_func': ParseVersion}, + {'uri': '/oarapi/version.json', 'parse_func': ParseVersion}, + 'GET_timezone': - {'uri':'/oarapi/timezone.json' ,'parse_func': ParseTimezone }, + {'uri': '/oarapi/timezone.json', 'parse_func': ParseTimezone}, + 'GET_jobs': - {'uri':'/oarapi/jobs.json','parse_func': ParseJobs}, + {'uri': '/oarapi/jobs.json', 'parse_func': ParseJobs}, + 'GET_jobs_id': - {'uri':'/oarapi/jobs/id.json','parse_func': ParseJobsIds}, + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseJobsIds}, + 'GET_jobs_id_resources': - {'uri':'/oarapi/jobs/id/resources.json',\ - 'parse_func': ParseJobsIdResources}, + {'uri': '/oarapi/jobs/id/resources.json', + 'parse_func': ParseJobsIdResources}, + 'GET_jobs_table': - {'uri':'/oarapi/jobs/table.json','parse_func': ParseJobsTable}, + {'uri': '/oarapi/jobs/table.json', 'parse_func': ParseJobsTable}, + 'GET_jobs_details': - {'uri':'/oarapi/jobs/details.json',\ - 'parse_func': ParseJobsDetails}, - 'GET_reserved_nodes': - {'uri': - '/oarapi/jobs/details.json?state=Running,Waiting,Launching',\ - 'owner':'&user=', - 'parse_func':ParseReservedNodes}, + {'uri': '/oarapi/jobs/details.json', 'parse_func': ParseJobsDetails}, + 'GET_reserved_nodes': + {'uri': + '/oarapi/jobs/details.json?state=Running,Waiting,Launching', + 'owner': '&user=', 'parse_func': ParseReservedNodes}, 'GET_running_jobs': - {'uri':'/oarapi/jobs/details.json?state=Running',\ - 'parse_func':ParseRunningJobs}, + {'uri': '/oarapi/jobs/details.json?state=Running', + 'parse_func': ParseRunningJobs}, + 'GET_resources_full': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFull}, + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFull}, + 'GET_sites': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFullSites}, - 'GET_resources': - {'uri':'/oarapi/resources.json' ,'parse_func': ParseResources}, - 'DELETE_jobs_id': - {'uri':'/oarapi/jobs/id.json' ,'parse_func': ParseDeleteJobs} - } + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFullSites}, + 'GET_resources': + {'uri': '/oarapi/resources.json', 'parse_func': ParseResources}, + 'DELETE_jobs_id': + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseDeleteJobs}} - def SendRequest(self, request, strval = None , username = None): + def SendRequest(self, request, strval=None, username=None): """ Connects to OAR , sends the valid GET requests and uses the appropriate json parsing functions. + :returns: calls to the appropriate parsing function, associated with the + GET request + :rtype: depends on the parsing function called. + + .. seealso:: OARrequests_uri_dict """ save_json = None self.json_page.ResetNextPage() save_json = [] - if request in self.OARrequests_uri_dict : + if request in self.OARrequests_uri_dict: while self.json_page.next_page: - self.json_page.raw_json = self.server.GETRequestToOARRestAPI(\ - request, \ - strval, \ - self.json_page.next_offset, \ - username) + self.json_page.raw_json = self.server.GETRequestToOARRestAPI( + request, + strval, + self.json_page.next_offset, + username) self.json_page.FindNextPage() if self.json_page.concatenate: save_json.append(self.json_page.raw_json) - if self.json_page.concatenate and self.json_page.end : + if self.json_page.concatenate and self.json_page.end: self.json_page.raw_json = \ self.json_page.ConcatenateJsonPages(save_json) return self.OARrequests_uri_dict[request]['parse_func'](self) else: - logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " \ - %(request)) - + logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " + % (request)) diff --git a/sfa/iotlab/iotlabaggregate.py b/sfa/iotlab/iotlabaggregate.py index 73fca1bc..cffc1eff 100644 --- a/sfa/iotlab/iotlabaggregate.py +++ b/sfa/iotlab/iotlabaggregate.py @@ -1,42 +1,57 @@ -#import time +""" +File providing methods to generate valid RSpecs for the Iotlab testbed. +Contains methods to get information on slice, slivers, nodes and leases, +formatting them and turn it into a RSpec. +""" from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority from sfa.rspecs.rspec import RSpec #from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.hardware_type import HardwareType from sfa.rspecs.elements.login import Login -from sfa.rspecs.elements.services import ServicesElement +from sfa.rspecs.elements.services import Services from sfa.rspecs.elements.sliver import Sliver from sfa.rspecs.elements.lease import Lease from sfa.rspecs.elements.granularity import Granularity from sfa.rspecs.version_manager import VersionManager +from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, \ + IotlabNode, IotlabLocation -from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, IotlabNode, \ - IotlabLocation from sfa.util.sfalogging import logger - from sfa.util.xrn import Xrn + def iotlab_xrn_to_hostname(xrn): + """Returns a node's hostname from its xrn. + :param xrn: The nodes xrn identifier. + :type xrn: Xrn (from sfa.util.xrn) + + :returns: node's hostname. + :rtype: string + + """ return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf()) + def iotlab_xrn_object(root_auth, hostname): - """Attributes are urn and hrn. -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py - Get the hostname using iotlab_xrn_to_hostname on the urn. + """Creates a valid xrn object from the node's hostname and the authority + of the SFA server. - :return: the iotlab node's xrn -======= - Get the hostname using slab_xrn_to_hostname on the urn. + :param hostname: the node's hostname. + :param root_auth: the SFA root authority. + :type hostname: string + :type root_auth: string - :return: the senslab node's xrn ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py + :returns: the iotlab node's xrn :rtype: Xrn + """ - return Xrn('.'.join( [root_auth, Xrn.escape(hostname)]), type='node') + return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node') + class IotlabAggregate: + """Aggregate manager class for Iotlab. """ sites = {} nodes = {} @@ -55,30 +70,24 @@ class IotlabAggregate: def get_slice_and_slivers(self, slice_xrn, login=None): """ Get the slices and the associated leases if any from the iotlab - testbed. For each slice, get the nodes in the associated lease - and create a sliver with the necessary info and insertinto the sliver - dictionary, keyed on the node hostnames. - Returns a dict of slivers based on the sliver's node_id. - Called by get_rspec. + testbed. One slice can have mutliple leases. + For each slice, get the nodes in the associated lease + and create a sliver with the necessary info and insert it into the + sliver dictionary, keyed on the node hostnames. + Returns a dict of slivers based on the sliver's node_id. + Called by get_rspec. :param slice_xrn: xrn of the slice -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py :param login: user's login on iotlab ldap -======= - :param login: user's login on senslab ldap ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py :type slice_xrn: string :type login: string - :reutnr : a list of slices dict and a dictionary of Sliver object - :rtype: (list, dict) + :returns: a list of slices dict and a list of Sliver object + :rtype: (list, list) -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py - ..note: There is no slivers in iotlab, only leases. -======= - ..note: There is no slivers in senslab, only leases. ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py + .. note: There is no real slivers in iotlab, only leases. The goal + is to be consistent with the SFA standard. """ slivers = {} @@ -89,149 +98,116 @@ class IotlabAggregate: slice_hrn, _ = urn_to_hrn(slice_xrn) slice_name = slice_hrn - slices = self.driver.iotlab_api.GetSlices(slice_filter= str(slice_name), \ - slice_filter_type = 'slice_hrn', \ - login=login) + slices = self.driver.iotlab_api.GetSlices(slice_filter=str(slice_name), + slice_filter_type='slice_hrn', + login=login) - logger.debug("Slabaggregate api \tget_slice_and_slivers \ - sfa_slice %s \r\n slices %s self.driver.hrn %s" \ - %(sfa_slice, slices, self.driver.hrn)) - if slices == []: + logger.debug("IotlabAggregate api \tget_slice_and_slivers \ + sfa_slice %s \r\n slices %s self.driver.hrn %s" + % (sfa_slice, slices, self.driver.hrn)) + if slices == []: return (sfa_slice, slivers) - # sort slivers by node id , if there is a job #and therefore, node allocated to this slice for sfa_slice in slices: try: - node_ids_list = sfa_slice['node_ids'] + node_ids_list = sfa_slice['node_ids'] except KeyError: - logger.log_exc("SLABAGGREGATE \t \ - get_slice_and_slivers No nodes in the slice - KeyError ") + logger.log_exc("IOTLABAGGREGATE \t \ + get_slice_and_slivers No nodes in the slice \ + - KeyError ") continue for node in node_ids_list: sliver_xrn = Xrn(slice_urn, type='sliver', id=node) sliver_xrn.set_authority(self.driver.hrn) - sliver = Sliver({'sliver_id':sliver_xrn.urn, + sliver = Sliver({'sliver_id': sliver_xrn.urn, 'name': sfa_slice['hrn'], -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py 'type': 'iotlab-node', -======= - 'type': 'slab-node', ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py 'tags': []}) slivers[node] = sliver - #Add default sliver attribute : -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py #connection information for iotlab - if get_authority (sfa_slice['hrn']) == self.driver.iotlab_api.root_auth: -======= - #connection information for senslab - if get_authority (sfa_slice['hrn']) == self.driver.slab_api.root_auth: ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py + if get_authority(sfa_slice['hrn']) == self.driver.iotlab_api.root_auth: tmp = sfa_slice['hrn'].split('.') ldap_username = tmp[1].split('_')[0] ssh_access = None - slivers['default_sliver'] = {'ssh': ssh_access , \ - 'login': ldap_username} + slivers['default_sliver'] = {'ssh': ssh_access, + 'login': ldap_username} #TODO get_slice_and_slivers Find the login of the external user - logger.debug("SLABAGGREGATE api get_slice_and_slivers slivers %s "\ - %(slivers)) + logger.debug("IOTLABAGGREGATE api get_slice_and_slivers slivers %s " + % (slivers)) return (slices, slivers) - def get_nodes(self, slices=None, slivers=[], options=None): + """Returns the nodes in the slice using the rspec format, with all the + nodes' properties. + + Fetch the nodes ids in the slices dictionary and get all the nodes + properties from OAR. Makes a rspec dicitonary out of this and returns + it. If the slice does not have any job running or scheduled, that is + it has no reserved nodes, then returns an empty list. + + :param slices: list of slices (record dictionaries) + :param slivers: the list of slivers in all the slices + :type slices: list of dicts + :type slivers: list of Sliver object (dictionaries) + :returns: An empty list if the slice has no reserved nodes, a rspec + list with all the nodes and their properties (a dict per node) + otherwise. + :rtype: list + + .. seealso:: get_slice_and_slivers + + """ # NT: the semantic of this function is not clear to me : # if slice is not defined, then all the nodes should be returned # if slice is defined, we should return only the nodes that # are part of this slice # but what is the role of the slivers parameter ? # So i assume that slice['node_ids'] will be the same as slivers for us - #filter_dict = {} - #if slice_xrn: - #if not slices or not slices['node_ids']: - #return ([],[]) - #tags_filter = {} + slice_nodes_list = [] + if slices is not None: + for one_slice in slices: + try: + slice_nodes_list = one_slice['node_ids'] + # if we are dealing with a slice that has no node just + # return an empty list. In iotlab a slice can have multiple + # jobs scheduled, so it either has at least one lease or + # not at all. + except KeyError: + return [] # get the granularity in second for the reservation system -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py grain = self.driver.iotlab_api.GetLeaseGranularity() - nodes = self.driver.iotlab_api.GetNodes() -======= - grain = self.driver.slab_api.GetLeaseGranularity() - - - nodes = self.driver.slab_api.GetNodes() ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py - #geni_available = options.get('geni_available') - #if geni_available: - #filter['boot_state'] = 'boot' - - #filter.update({'peer_id': None}) -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py - #nodes = self.driver.iotlab_api.GetNodes(filter['hostname']) -======= - #nodes = self.driver.slab_api.GetNodes(filter['hostname']) ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py - - #site_ids = [] - #interface_ids = [] - #tag_ids = [] - nodes_dict = {} - - #for node in nodes: - #nodes_dict[node['node_id']] = node - #logger.debug("SLABAGGREGATE api get_nodes nodes %s "\ - #%(nodes )) - # get sites - #sites_dict = self.get_sites({'site_id': site_ids}) - # get interfaces - #interfaces = self.get_interfaces({'interface_id':interface_ids}) - # get tags - #node_tags = self.get_node_tags(tags_filter) + nodes_dict = {} #if slices, this means we got to list all the nodes given to this slice # Make a list of all the nodes in the slice before getting their #attributes rspec_nodes = [] - slice_nodes_list = [] - logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slices )) - if slices is not None: - for one_slice in slices: - try: - slice_nodes_list = one_slice['node_ids'] - except KeyError: - pass - #for node in one_slice['node_ids']: - #slice_nodes_list.append(node) -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py + logger.debug("IOTLABAGGREGATE api get_nodes slice_nodes_list %s " + % (slices)) + + reserved_nodes = self.driver.iotlab_api.GetNodesCurrentlyInUse() -======= - reserved_nodes = self.driver.slab_api.GetNodesCurrentlyInUse() ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py - logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slice_nodes_list)) + logger.debug("IOTLABAGGREGATE api get_nodes slice_nodes_list %s " + % (slice_nodes_list)) for node in nodes: nodes_dict[node['node_id']] = node if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py rspec_node = IotlabNode() -======= - rspec_node = SlabNode() ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py # xxx how to retrieve site['login_base'] #site_id=node['site_id'] #site=sites_dict[site_id] @@ -240,27 +216,16 @@ class IotlabAggregate: rspec_node['archi'] = node['archi'] rspec_node['radio'] = node['radio'] -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py - iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, \ - node['hostname']) + iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, + node['hostname']) rspec_node['component_id'] = iotlab_xrn.urn -======= - slab_xrn = slab_xrn_object(self.driver.slab_api.root_auth, \ - node['hostname']) - rspec_node['component_id'] = slab_xrn.urn ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = \ - hrn_to_urn(self.driver.iotlab_api.root_auth, \ + hrn_to_urn(self.driver.iotlab_api.root_auth, 'authority+sa') -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py # Iotlab's nodes are federated : there is only one authority # for all Iotlab sites, registered in SFA. -======= - # Senslab's nodes are federated : there is only one authority - # for all Senslab sites, registered in SFA. ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py # Removing the part including the site # in authority_id SA 27/07/12 rspec_node['authority_id'] = rspec_node['component_manager_id'] @@ -282,17 +247,13 @@ class IotlabAggregate: rspec_node['location'] = location -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py position = IotlabPosition() -======= - position = SlabPosition() ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py for field in position : try: position[field] = node[field] except KeyError, error : - logger.log_exc("SLABAGGREGATE\t get_nodes \ - position %s "%(error)) + logger.log_exc("IOTLABAGGREGATE\t get_nodes \ + position %s "% (error)) rspec_node['position'] = position #rspec_node['interfaces'] = [] @@ -317,17 +278,21 @@ class IotlabAggregate: rspec_nodes.append(rspec_node) return (rspec_nodes) - #def get_all_leases(self, slice_record = None): + def get_all_leases(self): """ + Get list of lease dictionaries which all have the mandatory keys ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration'). All the leases running or scheduled are returned. + :returns: rspec lease dictionary with keys lease_id, component_id, + slice_id, start_time, duration. + :rtype: dict - ..note::There is no filtering of leases within a given time frame. - All the running or scheduled leases are returned. options - removed SA 15/05/2013 + .. note::There is no filtering of leases within a given time frame. + All the running or scheduled leases are returned. options + removed SA 15/05/2013 """ @@ -341,7 +306,7 @@ class IotlabAggregate: #leases = self.driver.iotlab_api.GetLeases(lease_filter) leases = self.driver.iotlab_api.GetLeases() grain = self.driver.iotlab_api.GetLeaseGranularity() - site_ids = [] + # site_ids = [] rspec_leases = [] for lease in leases: #as many leases as there are nodes in the job @@ -349,7 +314,8 @@ class IotlabAggregate: rspec_lease = Lease() rspec_lease['lease_id'] = lease['lease_id'] #site = node['site_id'] - iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, node) + iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, + node) rspec_lease['component_id'] = iotlab_xrn.urn #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ #site, node['hostname']) @@ -360,49 +326,60 @@ class IotlabAggregate: pass rspec_lease['start_time'] = lease['t_from'] rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ - / grain + / grain rspec_leases.append(rspec_lease) return rspec_leases + def get_rspec(self, slice_xrn=None, login=None, version=None, + options=None): + """ + Returns xml rspec: + - a full advertisement rspec with the testbed resources if slice_xrn + is not specified.If a lease option is given, also returns the + leases scheduled on the testbed. + - a manifest Rspec with the leases and nodes in slice's leases + if slice_xrn is not None. + + :param slice_xrn: srn of the slice + :param login: user'uid (ldap login) on iotlab + :param version: can be set to sfa or iotlab + :param options: used to specify if the leases should also be included in + the returned rspec. + :type slice_xrn: string + :type login: string + :type version: RSpecVersion + :type options: dict + + :returns: Xml Rspec. + :rtype: XML -#from plc/aggregate.py - def get_rspec(self, slice_xrn=None, login=None, version = None, \ - options=None): + + """ rspec = None version_manager = VersionManager() version = version_manager.get_version(version) logger.debug("IotlabAggregate \t get_rspec ***version %s \ - version.type %s version.version %s options %s \r\n" \ - %(version,version.type,version.version,options)) + version.type %s version.version %s options %s \r\n" + % (version, version.type, version.version, options)) if slice_xrn is None: - rspec_version = version_manager._get_version(version.type, \ - version.version, 'ad') + rspec_version = version_manager._get_version(version.type, + version.version, 'ad') else: - rspec_version = version_manager._get_version(version.type, \ - version.version, 'manifest') + rspec_version = version_manager._get_version( + version.type, version.version, 'manifest') slices, slivers = self.get_slice_and_slivers(slice_xrn, login) -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py #at this point sliver may be empty if no iotlab job -======= - #at this point sliver may be empty if no senslab job ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py #is running for this user/slice. rspec = RSpec(version=rspec_version, user_options=options) - - #if slice and 'expires' in slice: - #rspec.xml.set('expires',\ - #datetime_to_string(utcparse(slice['expires'])) - # add sliver defaults - #nodes, links = self.get_nodes(slice, slivers) logger.debug("\r\n \r\n IotlabAggregate \tget_rspec *** \ - slice_xrn %s slices %s\r\n \r\n"\ - %(slice_xrn, slices)) + slice_xrn %s slices %s\r\n \r\n" + % (slice_xrn, slices)) if options is not None: lease_option = options['list_leases'] @@ -412,57 +389,41 @@ class IotlabAggregate: #if slice_xrn : #lease_option = 'all' - if lease_option in ['all', 'resources']: #if not options.get('list_leases') or options.get('list_leases') #and options['list_leases'] != 'leases': nodes = self.get_nodes(slices, slivers) -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py - logger.debug("\r\n \r\n IotlabAggregate \ lease_option %s \ -======= - logger.debug("\r\n \r\n SlabAggregate \ lease_option %s \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py - get rspec ******* nodes %s"\ - %(lease_option, nodes[0])) + logger.debug("\r\n \r\n IotlabAggregate \t lease_option %s \ + get rspec ******* nodes %s" + % (lease_option, nodes[0])) - sites_set = set([node['location']['site'] for node in nodes] ) + sites_set = set([node['location']['site'] for node in nodes]) #In case creating a job, slice_xrn is not set to None rspec.version.add_nodes(nodes) - if slice_xrn : + if slice_xrn: #Get user associated with this slice - #user = dbsession.query(RegRecord).filter_by(record_id = \ - #slices['record_id_user']).first() - - #ldap_username = (user.hrn).split('.')[1] - - #for one_slice in slices : ldap_username = slices[0]['hrn'] tmp = ldap_username.split('.') ldap_username = tmp[1].split('_')[0] - if version.type == "Slab": - rspec.version.add_connection_information(ldap_username, \ - sites_set) + if version.type == "Iotlab": + rspec.version.add_connection_information( + ldap_username, sites_set) default_sliver = slivers.get('default_sliver', []) if default_sliver: #default_sliver_attribs = default_sliver.get('tags', []) logger.debug("IotlabAggregate \tget_rspec **** \ - default_sliver%s \r\n" %(default_sliver)) + default_sliver%s \r\n" % (default_sliver)) for attrib in default_sliver: - rspec.version.add_default_sliver_attribute(attrib, \ - default_sliver[attrib]) + rspec.version.add_default_sliver_attribute( + attrib, default_sliver[attrib]) + if lease_option in ['all','leases']: - #leases = self.get_all_leases(slices) leases = self.get_all_leases() rspec.version.add_leases(leases) - -<<<<<<< HEAD:sfa/iotlab/iotlabaggregate.py - #logger.debug("IotlabAggregate \tget_rspec ******* rspec_toxml %s \r\n"\ -======= - #logger.debug("SlabAggregate \tget_rspec ******* rspec_toxml %s \r\n"\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabaggregate.py - #%(rspec.toxml())) + logger.debug("IotlabAggregate \tget_rspec **** \ + FINAL RSPEC %s \r\n" % (rspec.toxml())) return rspec.toxml() diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py index 66ffec96..0e3e9008 100644 --- a/sfa/iotlab/iotlabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -1,3 +1,9 @@ +""" +File containing the IotlabTestbedAPI, used to interact with nodes, users, +slices, leases and keys, as well as the dedicated iotlab database and table, +holding information about which slice is running which job. + +""" from datetime import datetime from sfa.util.sfalogging import logger @@ -5,9 +11,8 @@ from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession from sqlalchemy.orm import joinedload from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey -from sfa.iotlab.iotlabpostgres import iotlab_dbsession, IotlabXP - -from sfa.iotlab.OARrestapi import OARrestapi +from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP +from sfa.iotlab.OARrestapi import OARrestapi from sfa.iotlab.LDAPapi import LDAPapi from sfa.util.xrn import Xrn, hrn_to_urn, get_authority @@ -15,17 +20,14 @@ from sfa.util.xrn import Xrn, hrn_to_urn, get_authority from sfa.trust.certificate import Keypair, convert_public_key from sfa.trust.gid import create_uuid from sfa.trust.hierarchy import Hierarchy -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py -======= - -from sfa.senslab.slabaggregate import slab_xrn_object ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py from sfa.iotlab.iotlabaggregate import iotlab_xrn_object class IotlabTestbedAPI(): """ Class enabled to use LDAP and OAR api calls. """ + _MINIMUM_DURATION = 600 + def __init__(self, config): """Creates an instance of OARrestapi and LDAPapi which will be used to issue calls to OAR or LDAP methods. @@ -35,21 +37,25 @@ class IotlabTestbedAPI(): :param config: configuration object from sfa.util.config :type config: Config object """ + self.iotlab_db = IotlabDB(config) self.oar = OARrestapi() self.ldap = LDAPapi() self.time_format = "%Y-%m-%d %H:%M:%S" self.root_auth = config.SFA_REGISTRY_ROOT_AUTH - self.grain = 1 # 10 mins lease minimum + self.grain = 1 # 10 mins lease minimum, 1 sec granularity #import logging, logging.handlers #from sfa.util.sfalogging import _SfaLogger #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \ #level=logging.DEBUG) return - @staticmethod def GetMinExperimentDurationInSec(): - return 600 + """ Returns the minimum allowed duration for an experiment on the + testbed. In seconds. + + """ + return IotlabTestbedAPI._MINIMUM_DURATION @staticmethod def GetPeers (peer_filter=None ): @@ -57,7 +63,7 @@ class IotlabTestbedAPI(): if peer_filter is specified. :param peer_filter: name of the site authority looked for. :type peer_filter: string - :return: list of records. + :returns: list of records. """ @@ -75,74 +81,57 @@ class IotlabTestbedAPI(): existing_hrns_by_types[record.type].append(record.hrn) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\ -======= - logger.debug("SLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py %( existing_hrns_by_types)) records_list = [] try: if peer_filter: records_list.append(existing_records[(peer_filter,'authority')]) - else : + else: for hrn in existing_hrns_by_types['authority']: records_list.append(existing_records[(hrn,'authority')]) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER \tGetPeer \trecords_list %s " \ -======= - logger.debug("SLABDRIVER \tGetPeer \trecords_list %s " \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py %(records_list)) except KeyError: pass return_records = records_list - logger.debug("IOTLABDRIVER \tGetPeer return_records %s " \ - %(return_records)) + logger.debug("IOTLABDRIVER \tGetPeer return_records %s " + % (return_records)) return return_records - - #TODO : Handling OR request in make_ldap_filters_from_records #instead of the for loop #over the records' list def GetPersons(self, person_filter=None): """ -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py Get the enabled users and their properties from Iotlab LDAP. -======= - Get the enabled users and their properties from Senslab LDAP. ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py If a filter is specified, looks for the user whose properties match the filter, otherwise returns the whole enabled users'list. + :param person_filter: Must be a list of dictionnaries - with users properties when not set to None. + with users properties when not set to None. :param person_filter: list of dict - :return:Returns a list of users whose accounts are enabled - found in ldap. + :returns:Returns a list of users whose accounts are enabled + found in ldap. :rtype: list of dicts """ - logger.debug("IOTLABDRIVER \tGetPersons person_filter %s" \ - %(person_filter)) + logger.debug("IOTLABDRIVER \tGetPersons person_filter %s" + % (person_filter)) person_list = [] if person_filter and isinstance(person_filter, list): #If we are looking for a list of users (list of dict records) #Usually the list contains only one user record for searched_attributes in person_filter: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #Get only enabled user accounts in iotlab LDAP : -======= - #Get only enabled user accounts in senslab LDAP : ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #add a filter for make_ldap_filters_from_record - person = self.ldap.LdapFindUser(searched_attributes, \ - is_user_enabled=True) + person = self.ldap.LdapFindUser(searched_attributes, + is_user_enabled=True) #If a person was found, append it to the list if person: person_list.append(person) @@ -152,11 +141,7 @@ class IotlabTestbedAPI(): person_list = None else: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #Get only enabled user accounts in iotlab LDAP : -======= - #Get only enabled user accounts in senslab LDAP : ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #add a filter for make_ldap_filters_from_record person_list = self.ldap.LdapFindUser(is_user_enabled=True) @@ -171,29 +156,23 @@ class IotlabTestbedAPI(): #return server_timestamp, server_tz def DeleteJobs(self, job_id, username): + """ - """ Deletes the job with the specified job_id and username on OAR by - posting a delete request to OAR. + Deletes the job with the specified job_id and username on OAR by + posting a delete request to OAR. :param job_id: job id in OAR. -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py :param username: user's iotlab login in LDAP. -======= - :param username: user's senslab login in LDAP. ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - :type job_id:integer + :type job_id: integer :type username: string - :return: dictionary with the job id and if delete has been successful - (True) or no (False) + :returns: dictionary with the job id and if delete has been successful + (True) or no (False) :rtype: dict + """ -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s username %s "\ -======= - logger.debug("SLABDRIVER \tDeleteJobs jobid %s username %s "\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - %(job_id, username)) + logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s username %s " + % (job_id, username)) if not job_id or job_id is -1: return @@ -201,15 +180,14 @@ class IotlabTestbedAPI(): reqdict['method'] = "delete" reqdict['strval'] = str(job_id) - - answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', \ - reqdict,username) + answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', + reqdict, username) if answer['status'] == 'Delete request registered': - ret = {job_id : True } + ret = {job_id: True} else: - ret = {job_id :False } + ret = {job_id: False} logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s \r\n answer %s \ - username %s" %(job_id, answer, username)) + username %s" % (job_id, answer, username)) return ret @@ -239,11 +217,7 @@ class IotlabTestbedAPI(): #return None #except KeyError: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #logger.error("IOTLABDRIVER \tGetJobsId KeyError") -======= - #logger.error("SLABDRIVER \tGetJobsId KeyError") ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #return None #parsed_job_info = self.get_info_on_reserved_nodes(job_info, \ @@ -260,11 +234,7 @@ class IotlabTestbedAPI(): def GetJobsResources(self, job_id, username = None): """ Gets the list of nodes associated with the job_id and username if provided. -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py Transforms the iotlab hostnames to the corresponding -======= - Transforms the senslab hostnames to the corresponding ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py SFA nodes hrns. Rertuns dict key :'node_ids' , value : hostnames list :param username: user's LDAP login @@ -272,7 +242,7 @@ class IotlabTestbedAPI(): :type username: string :type job_id: integer - :return: dicionary with nodes' hostnames belonging to the job. + :returns: dicionary with nodes' hostnames belonging to the job. :rtype: dict """ @@ -281,11 +251,7 @@ class IotlabTestbedAPI(): #Get job resources list from OAR node_id_list = self.oar.parser.SendRequest(req, job_id, username) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER \t GetJobsResources %s " %(node_id_list)) -======= - logger.debug("SLABDRIVER \t GetJobsResources %s " %(node_id_list)) ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py hostname_list = \ self.__get_hostnames_from_oar_node_ids(node_id_list) @@ -317,19 +283,11 @@ class IotlabTestbedAPI(): #reserved_node_hostname_list[index] = \ #node_dict[job_info[node_list_name][index]]['hostname'] -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #logger.debug("IOTLABDRIVER \t get_info_on_reserved_nodes \ #reserved_node_hostname_list %s" \ #%(reserved_node_hostname_list)) #except KeyError: #logger.error("IOTLABDRIVER \t get_info_on_reserved_nodes KEYERROR " ) -======= - #logger.debug("SLABDRIVER \t get_info_on_reserved_nodes \ - #reserved_node_hostname_list %s" \ - #%(reserved_node_hostname_list)) - #except KeyError: - #logger.error("SLABDRIVER \t get_info_on_reserved_nodes KEYERROR " ) ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #return reserved_node_hostname_list @@ -369,7 +327,7 @@ class IotlabTestbedAPI(): OAR node identifier. :param username: user's LDAP login :type username: string - :return: list of reservations dict + :returns: list of reservations dict :rtype: dict list """ @@ -388,32 +346,30 @@ class IotlabTestbedAPI(): #del resa['resource_ids'] return reservation_dict_list - def GetNodes(self, node_filter_dict = None, return_fields_list = None): + def GetNodes(self, node_filter_dict=None, return_fields_list=None): """ -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py Make a list of iotlab nodes and their properties from information -======= - Make a list of senslab nodes and their properties from information ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - given by OAR. Search for specific nodes if some filters are specified. - Nodes properties returned if no return_fields_list given: - 'hrn','archi','mobile','hostname','site','boot_state','node_id', - 'radio','posx','posy','oar_id','posz'. - - :param node_filter_dict: dictionnary of lists with node properties + given by OAR. Search for specific nodes if some filters are + specified. Nodes properties returned if no return_fields_list given: + 'hrn','archi','mobile','hostname','site','boot_state','node_id', + 'radio','posx','posy','oar_id','posz'. + + :param node_filter_dict: dictionnary of lists with node properties. For + instance, if you want to look for a specific node with its hrn, + the node_filter_dict should be {'hrn': [hrn_of_the_node]} :type node_filter_dict: dict :param return_fields_list: list of specific fields the user wants to be - returned. + returned. :type return_fields_list: list - :return: list of dictionaries with node properties + :returns: list of dictionaries with node properties :rtype: list """ node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full") node_dict_list = node_dict_by_id.values() logger.debug (" IOTLABDRIVER GetNodes node_filter_dict %s \ - return_fields_list %s "%(node_filter_dict, return_fields_list)) + return_fields_list %s " % (node_filter_dict, return_fields_list)) #No filtering needed return the list directly if not (node_filter_dict or return_fields_list): return node_dict_list @@ -427,7 +383,7 @@ class IotlabTestbedAPI(): for value in node_filter_dict[filter_key]: for node in node_dict_list: if node[filter_key] == value: - if return_fields_list : + if return_fields_list: tmp = {} for k in return_fields_list: tmp[k] = node[k] @@ -445,42 +401,54 @@ class IotlabTestbedAPI(): @staticmethod def AddSlice(slice_record, user_record): - """Add slice to the local iotlab sfa tables if the slice comes - from a federated site and is not yet in the iotlab sfa DB, - although the user has already a LDAP login. - Called by verify_slice during lease/sliver creation. + """ + + Add slice to the local iotlab sfa tables if the slice comes + from a federated site and is not yet in the iotlab sfa DB, + although the user has already a LDAP login. + Called by verify_slice during lease/sliver creation. + :param slice_record: record of slice, must contain hrn, gid, slice_id - and authority of the slice. + and authority of the slice. :type slice_record: dictionary :param user_record: record of the user :type user_record: RegUser + """ sfa_record = RegSlice(hrn=slice_record['hrn'], gid=slice_record['gid'], pointer=slice_record['slice_id'], authority=slice_record['authority']) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER.PY AddSlice sfa_record %s user_record %s" -======= - logger.debug("SLABDRIVER.PY AddSlice sfa_record %s user_record %s" ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py % (sfa_record, user_record)) sfa_record.just_created() dbsession.add(sfa_record) dbsession.commit() #Update the reg-researcher dependance table - sfa_record.reg_researchers = [user_record] + sfa_record.reg_researchers = [user_record] dbsession.commit() return - def GetSites(self, site_filter_name_list = None, return_fields_list = None): + def GetSites(self, site_filter_name_list=None, return_fields_list=None): + """Returns the list of Iotlab's sites with the associated nodes and + their properties as dictionaries. + + Uses the OAR request GET_sites to find the Iotlab's sites. + + :param site_filter_name_list: used to specify specific sites + :param return_fields_list: field that has to be returned + :type site_filter_name_list: list + :type return_fields_list: list + + .. warning:: unused + """ site_dict = self.oar.parser.SendRequest("GET_sites") #site_dict : dict where the key is the sit ename return_site_list = [] - if not ( site_filter_name_list or return_fields_list): + if not (site_filter_name_list or return_fields_list): return_site_list = site_dict.values() return return_site_list @@ -492,48 +460,46 @@ class IotlabTestbedAPI(): try: tmp[field] = site_dict[site_filter_name][field] except KeyError: - logger.error("GetSites KeyError %s "%(field)) + logger.error("GetSites KeyError %s " % (field)) return None return_site_list.append(tmp) else: - return_site_list.append( site_dict[site_filter_name]) - + return_site_list.append(site_dict[site_filter_name]) return return_site_list - - - #TODO : Check rights to delete person def DeletePerson(self, person_record): -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - """ Disable an existing account in iotlab LDAP. -======= - """ Disable an existing account in senslab LDAP. ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py + """Disable an existing account in iotlab LDAP. + Users and techs can only delete themselves. PIs can only - delete themselves and other non-PIs at their sites. - ins can delete anyone. + delete themselves and other non-PIs at their sites. + ins can delete anyone. + :param person_record: user's record :type person_record: dict - :return: True if successful, False otherwise. + :returns: True if successful, False otherwise. :rtype: boolean + .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF. """ #Disable user account in iotlab LDAP ret = self.ldap.LdapMarkUserAsDeleted(person_record) - logger.warning("IOTLABDRIVER DeletePerson %s " %(person_record)) + logger.warning("IOTLABDRIVER DeletePerson %s " % (person_record)) return ret['bool'] - def DeleteSlice(self, slice_record): - """ Deletes the specified slice and kills the jobs associated with - the slice if any, using DeleteSliceFromNodes. + """Deletes the specified slice and kills the jobs associated with + the slice if any, using DeleteSliceFromNodes. - :return: True if all the jobs in the slice have been deleted, - or the list of jobs that could not be deleted otherwise. - :rtype: list or boolean + :param slice_record: record of the slice, must contain oar_job_id, user + :type slice_record: dict + :returns: True if all the jobs in the slice have been deleted, + or the list of jobs that could not be deleted otherwise. + :rtype: list or boolean + + .. seealso:: DeleteSliceFromNodes """ ret = self.DeleteSliceFromNodes(slice_record) @@ -544,11 +510,7 @@ class IotlabTestbedAPI(): delete_failed = [] delete_failed.append(job_id) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.info("IOTLABDRIVER DeleteSlice %s answer %s"%(slice_record, \ -======= - logger.info("SLABDRIVER DeleteSlice %s answer %s"%(slice_record, \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py delete_failed)) return delete_failed or True @@ -556,11 +518,7 @@ class IotlabTestbedAPI(): def __add_person_to_db(user_dict): """ Add a federated user straight to db when the user issues a lease -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py request with iotlab nodes and that he has not registered with iotlab -======= - request with senslab nodes and that he has not registered with senslab ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py yet (that is he does not have a LDAP entry yet). Uses parts of the routines in SlabImport when importing user from LDAP. Called by AddPerson, right after LdapAddUser. @@ -593,7 +551,7 @@ class IotlabTestbedAPI(): pkey) if user_dict['email']: logger.debug("__add_person_to_db \r\n \r\n \ - SLAB IMPORTER PERSON EMAIL OK email %s "\ + IOTLAB IMPORTER PERSON EMAIL OK email %s "\ %(user_dict['email'])) person_gid.set_email(user_dict['email']) @@ -608,13 +566,17 @@ class IotlabTestbedAPI(): def AddPerson(self, record): - """Adds a new account. Any fields specified in records are used, - otherwise defaults are used. Creates an appropriate login by calling - LdapAddUser. + """ + + Adds a new account. Any fields specified in records are used, + otherwise defaults are used. Creates an appropriate login by calling + LdapAddUser. + :param record: dictionary with the sfa user's properties. - :return: The uid of the added person if sucessful, otherwise returns - the error message from LDAP. + :returns: The uid of the added person if sucessful, otherwise returns + the error message from LDAP. :rtype: interger or string + """ ret = self.ldap.LdapAddUser(record) @@ -632,21 +594,18 @@ class IotlabTestbedAPI(): #TODO AddPersonKey 04/07/2012 SA def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict): """Adds a new key to the specified account. Adds the key to the - iotlab ldap, provided that the person_uid is valid. + iotlab ldap, provided that the person_uid is valid. + Non-admins can only modify their own keys. -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py :param person_uid: user's iotlab login in LDAP -======= - :param person_uid: user's senslab login in LDAP ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py :param old_attributes_dict: dict with the user's old sshPublicKey - :param new_key_dict:dict with the user's new sshPublicKey + :param new_key_dict: dict with the user's new sshPublicKey :type person_uid: string :rtype: Boolean - :return: True if the key has been modified, False otherwise. + :returns: True if the key has been modified, False otherwise. """ ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \ @@ -654,25 +613,26 @@ class IotlabTestbedAPI(): logger.warning("IOTLABDRIVER AddPersonKey EMPTY - DO NOTHING \r\n ") return ret['bool'] - def DeleteLeases(self, leases_id_list, slice_hrn ): + def DeleteLeases(self, leases_id_list, slice_hrn): """ + Deletes several leases, based on their job ids and the slice - they are associated with. Uses DeleteJobs to delete the jobs - on OAR. Note that one slice can contain multiple jobs, and in this case - all the jobs in the leases_id_list MUST belong to ONE slice, - since there is only one slice hrn provided here. + they are associated with. Uses DeleteJobs to delete the jobs + on OAR. Note that one slice can contain multiple jobs, and in this + case all the jobs in the leases_id_list MUST belong to ONE slice, + since there is only one slice hrn provided here. + :param leases_id_list: list of job ids that belong to the slice whose - slice hrn is provided. - :param slice_hrn: the slice hrn . - ..warning: Does not have a return value since there was no easy - way to handle failure when dealing with multiple job delete. Plus, - there was no easy way to report it to the user. + slice hrn is provided. + :param slice_hrn: the slice hrn. + :type slice_hrn: string + + .. warning:: Does not have a return value since there was no easy + way to handle failure when dealing with multiple job delete. Plus, + there was no easy way to report it to the user. + """ -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \ -======= - logger.debug("SLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py \r\n " %(leases_id_list, slice_hrn)) for job_id in leases_id_list: self.DeleteJobs(job_id, slice_hrn) @@ -795,29 +755,17 @@ class IotlabTestbedAPI(): lease_dict['time_format'] = self.time_format -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER.PY \tLaunchExperimentOnOAR slice_user %s\ -======= - logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR slice_user %s\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py \r\n " %(slice_user)) #Create the request for OAR reqdict = self._create_job_structure_request_for_OAR(lease_dict) # first step : start the OAR job and update the job -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\ -======= - logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py \r\n " %(reqdict)) answer = self.oar.POSTRequestToOARRestAPI('POST_job', \ reqdict, slice_user) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER \tLaunchExperimentOnOAR jobid %s " %(answer)) -======= - logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s " %(answer)) ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py try: jobid = answer['id'] except KeyError: @@ -841,11 +789,7 @@ class IotlabTestbedAPI(): lease_start_time, lease_duration): """Creates a job in OAR corresponding to the information provided -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py as parameters. Adds the job id and the slice hrn in the iotlab -======= - as parameters. Adds the job id and the slice hrn in the senslab ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py database so that we are able to know which slice has which nodes. :param hostname_list: list of nodes' OAR hostnames. @@ -877,7 +821,6 @@ class IotlabTestbedAPI(): end_time = lease_start_time + lease_duration -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ %s %s %s "%(slice_record['hrn'], job_id, end_time)) @@ -890,53 +833,32 @@ class IotlabTestbedAPI(): logger.debug("IOTLABDRIVER \r\n \r\n \t AddLeases iotlab_ex_row %s" \ %(iotlab_ex_row)) - iotlab_dbsession.add(iotlab_ex_row) - iotlab_dbsession.commit() + self.iotlab_db.iotlab_session.add(iotlab_ex_row) + self.iotlab_db.iotlab_session.commit() logger.debug("IOTLABDRIVER \t AddLeases hostname_list start_time %s " \ -======= - logger.debug("SLABDRIVER \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ - %s %s %s "%(slice_record['hrn'], job_id, end_time)) - - - logger.debug("SLABDRIVER \r\n \r\n \t AddLeases %s %s %s " \ - %(type(slice_record['hrn']), type(job_id), type(end_time))) - - slab_ex_row = SenslabXP(slice_hrn = slice_record['hrn'], \ - job_id = job_id, end_time= end_time) - - logger.debug("SLABDRIVER \r\n \r\n \t AddLeases slab_ex_row %s" \ - %(slab_ex_row)) - slab_dbsession.add(slab_ex_row) - slab_dbsession.commit() - - logger.debug("SLABDRIVER \t AddLeases hostname_list start_time %s " \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py %(start_time)) return -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #Delete the jobs from job_iotlab table -======= - #Delete the jobs from job_senslab table ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py def DeleteSliceFromNodes(self, slice_record): - """ Deletes all the running or scheduled jobs of a given slice - given its record. - :param slice_record: record of the slice + """ + + Deletes all the running or scheduled jobs of a given slice + given its record. + + :param slice_record: record of the slice, must contain oar_job_id, user :type slice_record: dict - :return: dict of the jobs'deletion status. Success= True, Failure= - False, for each job id. + :returns: dict of the jobs'deletion status. Success= True, Failure= + False, for each job id. :rtype: dict + """ -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - logger.debug("IOTLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record)) -======= - logger.debug("SLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record)) ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py + logger.debug("IOTLABDRIVER \t DeleteSliceFromNodes %s " + % (slice_record)) if isinstance(slice_record['oar_job_id'], list): oar_bool_answer = {} @@ -946,70 +868,64 @@ class IotlabTestbedAPI(): oar_bool_answer.update(ret) else: - oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], \ - slice_record['user'])] + oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], + slice_record['user'])] return oar_bool_answer def GetLeaseGranularity(self): -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py """ Returns the granularity of an experiment in the Iotlab testbed. -======= - """ Returns the granularity of an experiment in the Senslab testbed. ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py OAR uses seconds for experiments duration , the granulaity is also defined in seconds. Experiments which last less than 10 min (600 sec) are invalid""" return self.grain - @staticmethod - def update_jobs_in_iotlabdb( job_oar_list, jobs_psql): - """ Cleans the iotlab db by deleting expired and cancelled jobs. - Compares the list of job ids given by OAR with the job ids that - are already in the database, deletes the jobs that are no longer in - the OAR job id list. - :param job_oar_list: list of job ids coming from OAR - :type job_oar_list: list - :param job_psql: list of job ids cfrom the database. - type job_psql: list - """ - #Turn the list into a set - set_jobs_psql = set(jobs_psql) - - kept_jobs = set(job_oar_list).intersection(set_jobs_psql) - logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \ - job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) - deleted_jobs = set_jobs_psql.difference(kept_jobs) - deleted_jobs = list(deleted_jobs) - if len(deleted_jobs) > 0: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - iotlab_dbsession.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - iotlab_dbsession.commit() -======= - slab_dbsession.query(SenslabXP).filter(SenslabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - slab_dbsession.commit() ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - - return - + # @staticmethod + # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql): + # """ Cleans the iotlab db by deleting expired and cancelled jobs. + # Compares the list of job ids given by OAR with the job ids that + # are already in the database, deletes the jobs that are no longer in + # the OAR job id list. + # :param job_oar_list: list of job ids coming from OAR + # :type job_oar_list: list + # :param job_psql: list of job ids cfrom the database. + # type job_psql: list + # """ + # #Turn the list into a set + # set_jobs_psql = set(jobs_psql) + + # kept_jobs = set(job_oar_list).intersection(set_jobs_psql) + # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \ + # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) + # deleted_jobs = set_jobs_psql.difference(kept_jobs) + # deleted_jobs = list(deleted_jobs) + # if len(deleted_jobs) > 0: + # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + # self.iotlab_db.iotlab_session.commit() + + # return def GetLeases(self, lease_filter_dict=None, login=None): - """ Get the list of leases from OAR with complete information - about which slice owns which jobs and nodes. - Two purposes: - -Fetch all the jobs from OAR (running, waiting..) - complete the reservation information with slice hrn - found in iotlab_xp table. If not available in the table, - assume it is a iotlab slice. - -Updates the iotlab table, deleting jobs when necessary. - :return: reservation_list, list of dictionaries with 'lease_id', - 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', - 'slice_hrn', 'resource_ids', 't_from', 't_until' + """ + + Get the list of leases from OAR with complete information + about which slice owns which jobs and nodes. + Two purposes: + -Fetch all the jobs from OAR (running, waiting..) + complete the reservation information with slice hrn + found in iotlab_xp table. If not available in the table, + assume it is a iotlab slice. + -Updates the iotlab table, deleting jobs when necessary. + + :returns: reservation_list, list of dictionaries with 'lease_id', + 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', + 'slice_hrn', 'resource_ids', 't_from', 't_until' :rtype: list + """ unfiltered_reservation_list = self.GetReservedNodes(login) @@ -1017,74 +933,50 @@ class IotlabTestbedAPI(): reservation_list = [] #Find the slice associated with this user iotlab ldap uid logger.debug(" IOTLABDRIVER.PY \tGetLeases login %s\ - unfiltered_reservation_list %s " %(login, unfiltered_reservation_list)) + unfiltered_reservation_list %s " + % (login, unfiltered_reservation_list)) #Create user dict first to avoid looking several times for #the same user in LDAP SA 27/07/12 job_oar_list = [] -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - jobs_psql_query = iotlab_dbsession.query(IotlabXP).all() -======= - jobs_psql_query = slab_dbsession.query(SenslabXP).all() ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - jobs_psql_dict = dict([(row.job_id, row.__dict__ ) for row in jobs_psql_query ]) + jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all() + jobs_psql_dict = dict([(row.job_id, row.__dict__) + for row in jobs_psql_query]) #jobs_psql_dict = jobs_psql_dict) - logger.debug("IOTLABDRIVER \tGetLeases jobs_psql_dict %s"\ - %(jobs_psql_dict)) - jobs_psql_id_list = [ row.job_id for row in jobs_psql_query ] - - + logger.debug("IOTLABDRIVER \tGetLeases jobs_psql_dict %s" + % (jobs_psql_dict)) + jobs_psql_id_list = [row.job_id for row in jobs_psql_query] for resa in unfiltered_reservation_list: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - logger.debug("IOTLABDRIVER \tGetLeases USER %s"\ -======= - logger.debug("SLABDRIVER \tGetLeases USER %s"\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - %(resa['user'])) + logger.debug("IOTLABDRIVER \tGetLeases USER %s" + % (resa['user'])) #Construct list of jobs (runing, waiting..) in oar job_oar_list.append(resa['lease_id']) - #If there is information on the job in SLAB DB ] + #If there is information on the job in IOTLAB DB ] #(slice used and job id) if resa['lease_id'] in jobs_psql_dict: job_info = jobs_psql_dict[resa['lease_id']] -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - logger.debug("IOTLABDRIVER \tGetLeases job_info %s"\ -======= - logger.debug("SLABDRIVER \tGetLeases job_info %s"\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - %(job_info)) + logger.debug("IOTLABDRIVER \tGetLeases job_info %s" + % (job_info)) resa['slice_hrn'] = job_info['slice_hrn'] resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #otherwise, assume it is a iotlab slice: -======= - #otherwise, assume it is a senslab slice: ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py else: - resa['slice_id'] = hrn_to_urn(self.root_auth+'.'+ \ - resa['user'] +"_slice" , 'slice') + resa['slice_id'] = hrn_to_urn(self.root_auth + '.' + + resa['user'] + "_slice", 'slice') resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn() resa['component_id_list'] = [] #Transform the hostnames into urns (component ids) for node in resa['reserved_nodes']: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py iotlab_xrn = iotlab_xrn_object(self.root_auth, node) resa['component_id_list'].append(iotlab_xrn.urn) if lease_filter_dict: logger.debug("IOTLABDRIVER \tGetLeases resa_ %s \ -======= - slab_xrn = slab_xrn_object(self.root_auth, node) - resa['component_id_list'].append(slab_xrn.urn) - - if lease_filter_dict: - logger.debug("SLABDRIVER \tGetLeases resa_ %s \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - \r\n leasefilter %s" %(resa, lease_filter_dict)) + \r\n leasefilter %s" % (resa, lease_filter_dict)) if lease_filter_dict['name'] == resa['slice_hrn']: reservation_list.append(resa) @@ -1092,17 +984,10 @@ class IotlabTestbedAPI(): if lease_filter_dict is None: reservation_list = unfiltered_reservation_list + self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - self.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list) - - logger.debug(" IOTLABDRIVER.PY \tGetLeases reservation_list %s"\ -======= - self.update_jobs_in_slabdb(job_oar_list, jobs_psql_id_list) - - logger.debug(" SLABDRIVER.PY \tGetLeases reservation_list %s"\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - %(reservation_list)) + logger.debug(" IOTLABDRIVER.PY \tGetLeases reservation_list %s" + % (reservation_list)) return reservation_list @@ -1152,11 +1037,7 @@ class IotlabTestbedAPI(): #return ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py ##Funciton should delete and create another job since oin iotlab slice=job -======= - ##Funciton should delete and create another job since oin senslab slice=job ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None): #"""Updates the parameters of an existing slice with the values in #slice_fields. @@ -1169,20 +1050,12 @@ class IotlabTestbedAPI(): #FROM PLC API DOC #""" -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #logger.warning("IOTLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ") -======= - #logger.warning("SLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ") ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #return #Unused SA 30/05/13, we only update the user's key or we delete it. ##TODO UpdatePerson 04/07/2012 SA -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None): -======= - #def UpdatePerson(self, slab_hrn, federated_hrn, person_fields=None): ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #"""Updates a person. Only the fields specified in person_fields #are updated, all other fields are left untouched. #Users and techs can only update themselves. PIs can only update @@ -1191,19 +1064,11 @@ class IotlabTestbedAPI(): #FROM PLC API DOC #""" -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn) - ##iotlab_dbsession.add(new_row) - ##iotlab_dbsession.commit() + ##self.iotlab_db.iotlab_session.add(new_row) + ##self.iotlab_db.iotlab_session.commit() #logger.debug("IOTLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ") -======= - ##new_row = FederatedToSenslab(slab_hrn, federated_hrn) - ##slab_dbsession.add(new_row) - ##slab_dbsession.commit() - - #logger.debug("SLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ") ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #return @staticmethod @@ -1217,48 +1082,47 @@ class IotlabTestbedAPI(): Admin may query all keys. Non-admins may only query their own keys. FROM PLC API DOC - :return: dict with ssh key as key and dicts as value. + :returns: dict with ssh key as key and dicts as value. :rtype: dict """ if key_filter is None: keys = dbsession.query(RegKey).options(joinedload('reg_user')).all() - else : + else: keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all() key_dict = {} for key in keys: - key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, \ - 'email': key.reg_user.email, 'hrn':key.reg_user.hrn} + key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, + 'email': key.reg_user.email, + 'hrn': key.reg_user.hrn} #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ #for user in ldap_rslt) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - logger.debug("IOTLABDRIVER GetKeys -key_dict %s \r\n " %(key_dict)) -======= - logger.debug("SLABDRIVER GetKeys -key_dict %s \r\n " %(key_dict)) ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py + logger.debug("IOTLABDRIVER GetKeys -key_dict %s \r\n " % (key_dict)) return key_dict #TODO : test def DeleteKey(self, user_record, key_string): - """ Deletes a key in the LDAP entry of the specified user. + """Deletes a key in the LDAP entry of the specified user. + Removes the key_string from the user's key list and updates the LDAP - user's entry with the new key attributes. + user's entry with the new key attributes. + :param key_string: The ssh key to remove :param user_record: User's record :type key_string: string :type user_record: dict - :return: True if sucessful, False if not. + :returns: True if sucessful, False if not. :rtype: Boolean """ all_user_keys = user_record['keys'] all_user_keys.remove(key_string) - new_attributes = {'sshPublicKey':all_user_keys} + new_attributes = {'sshPublicKey':all_user_keys} ret = self.ldap.LdapModifyUser(user_record, new_attributes) - logger.debug("IOTLABDRIVER DeleteKey %s- "%(ret)) + logger.debug("IOTLABDRIVER DeleteKey %s- " % (ret)) return ret['bool'] @@ -1268,66 +1132,69 @@ class IotlabTestbedAPI(): def _sql_get_slice_info( slice_filter ): """ Get the slice record based on the slice hrn. Fetch the record of the - user associated with the slice by usingjoinedload based on t - he reg_researcher relationship. + user associated with the slice by using joinedload based on the + reg_researcher relationship. + :param slice_filter: the slice hrn we are looking for :type slice_filter: string - :return: the slice record enhanced with the user's information if the - slice was found, None it wasn't. + :returns: the slice record enhanced with the user's information if the + slice was found, None it wasn't. + :rtype: dict or None. """ #DO NOT USE RegSlice - reg_researchers to get the hrn #of the user otherwise will mess up the RegRecord in #Resolve, don't know why - SA 08/08/2012 -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #Only one entry for one user = one slice in iotlab_xp table -======= - #Only one entry for one user = one slice in slab_xp table ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() - raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn = slice_filter).first() + raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() if raw_slicerec: #load_reg_researcher #raw_slicerec.reg_researchers raw_slicerec = raw_slicerec.__dict__ logger.debug(" IOTLABDRIVER \t get_slice_info slice_filter %s \ - raw_slicerec %s"%(slice_filter, raw_slicerec)) + raw_slicerec %s" % (slice_filter, raw_slicerec)) slicerec = raw_slicerec #only one researcher per slice so take the first one #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] #del slicerec['reg_researchers']['_sa_instance_state'] return slicerec - else : + else: return None @staticmethod - def _sql_get_slice_info_from_user(slice_filter ): + def _sql_get_slice_info_from_user(slice_filter): """ Get the slice record based on the user recordid by using a joinedload on the relationship reg_slices_as_researcher. Format the sql record into a dict with the mandatory fields for user and slice. - :return: dict with slice record and user record if the record was found + :returns: dict with slice record and user record if the record was found based on the user's id, None if not.. :rtype:dict or None.. """ #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() - raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id = slice_filter).first() + raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() #Put it in correct order - user_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'email', 'pointer'] - slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'pointer'] + user_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'email', 'pointer'] + slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'pointer'] if raw_slicerec: #raw_slicerec.reg_slices_as_researcher raw_slicerec = raw_slicerec.__dict__ slicerec = {} slicerec = \ - dict([(k, raw_slicerec['reg_slices_as_researcher'][0].__dict__[k]) \ - for k in slice_needed_fields]) - slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) \ - for k in user_needed_fields]) + dict([(k, raw_slicerec[ + 'reg_slices_as_researcher'][0].__dict__[k]) + for k in slice_needed_fields]) + slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) + for k in user_needed_fields]) #TODO Handle multiple slices for one user SA 10/12/12 #for now only take the first slice record associated to the rec user ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__ @@ -1340,8 +1207,8 @@ class IotlabTestbedAPI(): else: return None - def _get_slice_records(self, slice_filter = None, \ - slice_filter_type = None): + def _get_slice_records(self, slice_filter=None, + slice_filter_type=None): """ Get the slice record depending on the slice filter and its type. :param slice_filter: Can be either the slice hrn or the user's record @@ -1350,10 +1217,10 @@ class IotlabTestbedAPI(): :param slice_filter_type: describes the slice filter type used, can be slice_hrn or record_id_user :type: string - :return: the slice record + :returns: the slice record :rtype:dict - ..seealso:_sql_get_slice_info_from_user - ..seealso: _sql_get_slice_info + .. seealso::_sql_get_slice_info_from_user + .. seealso:: _sql_get_slice_info """ #Get list of slices based on the slice hrn @@ -1365,7 +1232,7 @@ class IotlabTestbedAPI(): slicerec = self._sql_get_slice_info(slice_filter) if slicerec is None: - return None + return None #return login, None #Get slice based on user id @@ -1385,28 +1252,24 @@ class IotlabTestbedAPI(): return fixed_slicerec_dict - - def GetSlices(self, slice_filter = None, slice_filter_type = None, \ - login=None): -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py - """ Get the slice records from the iotlab db and add lease information -======= - """ Get the slice records from the slab db and add lease information ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - if any. + def GetSlices(self, slice_filter=None, slice_filter_type=None, + login=None): + """Get the slice records from the iotlab db and add lease information + if any. :param slice_filter: can be the slice hrn or slice record id in the db - depending on the slice_filter_type. + depending on the slice_filter_type. :param slice_filter_type: defines the type of the filtering used, Can be - either 'slice_hrn' or "record_id'. + either 'slice_hrn' or "record_id'. :type slice_filter: string :type slice_filter_type: string - :return: a slice dict if slice_filter and slice_filter_type - are specified and a matching entry is found in the db. The result - is put into a list.Or a list of slice dictionnaries if no filters are - specified. + :returns: a slice dict if slice_filter and slice_filter_type + are specified and a matching entry is found in the db. The result + is put into a list.Or a list of slice dictionnaries if no filters + arespecified. :rtype: list + """ #login = None authorized_filter_types_list = ['slice_hrn', 'record_id_user'] @@ -1414,18 +1277,14 @@ class IotlabTestbedAPI(): #First try to get information on the slice based on the filter provided if slice_filter_type in authorized_filter_types_list: - fixed_slicerec_dict = \ - self._get_slice_records(slice_filter, slice_filter_type) + fixed_slicerec_dict = self._get_slice_records(slice_filter, + slice_filter_type) slice_hrn = fixed_slicerec_dict['hrn'] -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug(" IOTLABDRIVER \tGetSlices login %s \ -======= - logger.debug(" SLABDRIVER \tGetSlices login %s \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py slice record %s slice_filter %s \ - slice_filter_type %s " %(login, \ - fixed_slicerec_dict, slice_filter, \ + slice_filter_type %s " % (login, + fixed_slicerec_dict, slice_filter, slice_filter_type)) @@ -1433,78 +1292,67 @@ class IotlabTestbedAPI(): #jobs associated to this slice leases_list = [] - leases_list = self.GetLeases(login = login) + leases_list = self.GetLeases(login=login) #If no job is running or no job scheduled #return only the slice record if leases_list == [] and fixed_slicerec_dict: return_slicerec_dictlist.append(fixed_slicerec_dict) + # if the jobs running don't belong to the user/slice we are looking + # for + leases_hrn = [lease['slice_hrn'] for lease in leases_list] + if slice_hrn not in leases_hrn: + return_slicerec_dictlist.append(fixed_slicerec_dict) #If several jobs for one slice , put the slice record into # each lease information dict - - - for lease in leases_list : + for lease in leases_list: slicerec_dict = {} -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER.PY \tGetSlices slice_filter %s \ -======= - logger.debug("SLABDRIVER.PY \tGetSlices slice_filter %s \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - \ lease['slice_hrn'] %s" \ - %(slice_filter, lease['slice_hrn'])) - if lease['slice_hrn'] == slice_hrn: - slicerec_dict['slice_hrn'] = lease['slice_hrn'] - slicerec_dict['hrn'] = lease['slice_hrn'] - slicerec_dict['user'] = lease['user'] + \t lease['slice_hrn'] %s" + % (slice_filter, lease['slice_hrn'])) + if lease['slice_hrn'] == slice_hrn: slicerec_dict['oar_job_id'] = lease['lease_id'] - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) - slicerec_dict.update({'node_ids':lease['reserved_nodes']}) - #Update lease dict with the slice record if fixed_slicerec_dict: fixed_slicerec_dict['oar_job_id'] = [] - fixed_slicerec_dict['oar_job_id'].append(slicerec_dict['oar_job_id']) + fixed_slicerec_dict['oar_job_id'].append( + slicerec_dict['oar_job_id']) slicerec_dict.update(fixed_slicerec_dict) #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) + slicerec_dict['slice_hrn'] = lease['slice_hrn'] + slicerec_dict['hrn'] = lease['slice_hrn'] + slicerec_dict['user'] = lease['user'] + slicerec_dict.update( + {'list_node_ids': + {'hostname': lease['reserved_nodes']}}) + slicerec_dict.update({'node_ids': lease['reserved_nodes']}) + + return_slicerec_dictlist.append(slicerec_dict) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER.PY \tGetSlices \ - OHOHOHOH %s" %(return_slicerec_dictlist )) + OHOHOHOH %s" %(return_slicerec_dictlist)) logger.debug("IOTLABDRIVER.PY \tGetSlices \ -======= - logger.debug("SLABDRIVER.PY \tGetSlices \ - OHOHOHOH %s" %(return_slicerec_dictlist )) - - logger.debug("SLABDRIVER.PY \tGetSlices \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py slicerec_dict %s return_slicerec_dictlist %s \ lease['reserved_nodes'] \ - %s" %(slicerec_dict, return_slicerec_dictlist, \ - lease['reserved_nodes'] )) + %s" % (slicerec_dict, return_slicerec_dictlist, + lease['reserved_nodes'])) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER.PY \tGetSlices RETURN \ -======= - logger.debug("SLABDRIVER.PY \tGetSlices RETURN \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py - return_slicerec_dictlist %s" \ - %(return_slicerec_dictlist)) + return_slicerec_dictlist %s" + % (return_slicerec_dictlist)) return return_slicerec_dictlist else: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #Get all slices from the iotlab sfa database , -======= - #Get all slices from the senslab sfa database , ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #put them in dict format #query_slice_list = dbsession.query(RegRecord).all() - query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() + query_slice_list = \ + dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() for record in query_slice_list: tmp = record.__dict__ @@ -1516,16 +1364,12 @@ class IotlabTestbedAPI(): #Get all the jobs reserved nodes leases_list = self.GetReservedNodes() - for fixed_slicerec_dict in return_slicerec_dictlist: slicerec_dict = {} -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #Check if the slice belongs to a iotlab user -======= - #Check if the slice belongs to a senslab user ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py if fixed_slicerec_dict['peer_authority'] is None: - owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + owner = fixed_slicerec_dict['hrn'].split( + ".")[1].split("_")[0] else: owner = None for lease in leases_list: @@ -1533,30 +1377,25 @@ class IotlabTestbedAPI(): slicerec_dict['oar_job_id'] = lease['lease_id'] #for reserved_node in lease['reserved_nodes']: - logger.debug("IOTLABDRIVER.PY \tGetSlices lease %s "\ - %(lease )) - - slicerec_dict.update({'node_ids':lease['reserved_nodes']}) - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) + logger.debug("IOTLABDRIVER.PY \tGetSlices lease %s " + % (lease)) slicerec_dict.update(fixed_slicerec_dict) + slicerec_dict.update({'node_ids': + lease['reserved_nodes']}) + slicerec_dict.update({'list_node_ids': + {'hostname': + lease['reserved_nodes']}}) + #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) #return_slicerec_dictlist.append(slicerec_dict) fixed_slicerec_dict.update(slicerec_dict) -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py logger.debug("IOTLABDRIVER.PY \tGetSlices RETURN \ -======= - logger.debug("SLABDRIVER.PY \tGetSlices RETURN \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py return_slicerec_dictlist %s \slice_filter %s " \ %(return_slicerec_dictlist, slice_filter)) return return_slicerec_dictlist -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py -======= - ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py @@ -1569,7 +1408,6 @@ class IotlabTestbedAPI(): #iotlab_record = {} ##for field in record: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py ## iotlab_record[field] = record[field] #if sfa_type == "slice": @@ -1577,50 +1415,26 @@ class IotlabTestbedAPI(): #if not "instantiation" in iotlab_record: #iotlab_record["instantiation"] = "iotlab-instantiated" ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn) - ##Unused hrn_to_pl_slicename because Slab's hrn already + ##Unused hrn_to_pl_slicename because Iotlab's hrn already ##in the appropriate form SA 23/07/12 #iotlab_record["hrn"] = hrn #logger.debug("IOTLABDRIVER.PY sfa_fields_to_iotlab_fields \ #iotlab_record %s " %(iotlab_record['hrn'])) -======= - ## slab_record[field] = record[field] - - #if sfa_type == "slice": - ##instantion used in get_slivers ? - #if not "instantiation" in slab_record: - #slab_record["instantiation"] = "senslab-instantiated" - ##slab_record["hrn"] = hrn_to_pl_slicename(hrn) - ##Unused hrn_to_pl_slicename because Slab's hrn already - ##in the appropriate form SA 23/07/12 - #slab_record["hrn"] = hrn - #logger.debug("SLABDRIVER.PY sfa_fields_to_slab_fields \ - #slab_record %s " %(slab_record['hrn'])) ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py #if "url" in record: #iotlab_record["url"] = record["url"] #if "description" in record: #iotlab_record["description"] = record["description"] #if "expires" in record: -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py #iotlab_record["expires"] = int(record["expires"]) -======= - #slab_record["expires"] = int(record["expires"]) ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py ##nodes added by OAR only and then imported to SFA ##elif type == "node": ##if not "hostname" in iotlab_record: ##if not "hostname" in record: ##raise MissingSfaInfo("hostname") -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py ##iotlab_record["hostname"] = record["hostname"] ##if not "model" in iotlab_record: ##iotlab_record["model"] = "geni" -======= - ##slab_record["hostname"] = record["hostname"] - ##if not "model" in slab_record: - ##slab_record["model"] = "geni" ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py ##One authority only ##elif type == "authority": @@ -1649,13 +1463,3 @@ class IotlabTestbedAPI(): -<<<<<<< HEAD:sfa/iotlab/iotlabapi.py -======= - - - - - - - ->>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py diff --git a/sfa/iotlab/iotlabdriver.py b/sfa/iotlab/iotlabdriver.py index 4f05e810..7fa49574 100644 --- a/sfa/iotlab/iotlabdriver.py +++ b/sfa/iotlab/iotlabdriver.py @@ -1,31 +1,18 @@ +""" +Implements what a driver should provide for SFA to work. +""" from sfa.util.faults import SliverDoesNotExist, UnknownSfaType from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession from sfa.storage.model import RegRecord - - from sfa.managers.driver import Driver from sfa.rspecs.version_manager import VersionManager from sfa.rspecs.rspec import RSpec from sfa.util.xrn import Xrn, hrn_to_urn, get_authority - -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py -from sfa.iotlab.iotlabpostgres import IotlabDB -======= -from sfa.senslab.slabpostgres import SlabDB - - -from sfa.senslab.slabaggregate import SlabAggregate, slab_xrn_to_hostname - -from sfa.senslab.slabslices import SlabSlices ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - - from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname - from sfa.iotlab.iotlabslices import IotlabSlices @@ -35,241 +22,189 @@ from sfa.iotlab.iotlabapi import IotlabTestbedAPI class IotlabDriver(Driver): """ Iotlab Driver class inherited from Driver generic class. -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py -======= -class SlabDriver(Driver): - """ Senslab Driver class inherited from Driver generic class. - ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py Contains methods compliant with the SFA standard and the testbed - infrastructure (calls to LDAP and OAR). + infrastructure (calls to LDAP and OAR). - ..seealso:: Driver class + .. seealso::: Driver class """ def __init__(self, config): """ -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - Sets the iotlab SFA config parameters , - instanciates the testbed api and the iotlab database. + Sets the iotlab SFA config parameters, + instanciates the testbed api and the iotlab database. :param config: iotlab SFA configuration object -======= - Sets the senslab SFA config parameters , - instanciates the testbed api and the senslab database. - - :param config: senslab SFA configuration object ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py :type config: Config object + """ - Driver.__init__ (self, config) + Driver.__init__(self, config) self.config = config - - self.db = IotlabDB(config, debug = False) self.iotlab_api = IotlabTestbedAPI(config) self.cache = None - def augment_records_with_testbed_info (self, record_list ): + def augment_records_with_testbed_info(self, record_list): """ Adds specific testbed info to the records. :param record_list: list of sfa dictionaries records :type record_list: list - :return: list of records with extended information in each record + :returns: list of records with extended information in each record :rtype: list + """ - return self.fill_record_info (record_list) + return self.fill_record_info(record_list) def fill_record_info(self, record_list): """ -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py + For each SFA record, fill in the iotlab specific and SFA specific -======= - For each SFA record, fill in the senslab specific and SFA specific ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - fields in the record. + fields in the record. :param record_list: list of sfa dictionaries records :type record_list: list - :return: list of records with extended information in each record + :returns: list of records with extended information in each record :rtype: list - .. warnings:: Should not be modifying record_list directly because modi - fication are kept outside the method's scope. Howerver, there is no - other way to do it given the way it's called in registry manager. + .. warning:: Should not be modifying record_list directly because modi + fication are kept outside the method's scope. Howerver, there is no + other way to do it given the way it's called in registry manager. + """ -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug("IOTLABDRIVER \tfill_record_info records %s " %(record_list)) -======= - logger.debug("SLABDRIVER \tfill_record_info records %s " %(record_list)) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py + logger.debug("IOTLABDRIVER \tfill_record_info records %s " + % (record_list)) if not isinstance(record_list, list): record_list = [record_list] - try: for record in record_list: + + if str(record['type']) == 'node': + # look for node info using GetNodes + # the record is about one node only + filter_dict = {'hrn': [record['hrn']]} + node_info = self.iotlab_api.GetNodes(filter_dict) + # the node_info is about one node only, but it is formatted + # as a list + record.update(node_info[0]) + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info NODE" % (record)) + #If the record is a SFA slice record, then add information #about the user of this slice. This kind of -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py #information is in the Iotlab's DB. -======= - #information is in the Senslab's DB. ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py if str(record['type']) == 'slice': - if 'reg_researchers' in record and \ - isinstance(record['reg_researchers'], list) : + if 'reg_researchers' in record and isinstance(record + ['reg_researchers'], + list): record['reg_researchers'] = \ record['reg_researchers'][0].__dict__ - record.update({'PI':[record['reg_researchers']['hrn']], - 'researcher': [record['reg_researchers']['hrn']], - 'name':record['hrn'], - 'oar_job_id':[], - 'node_ids': [], - 'person_ids':[record['reg_researchers']['record_id']], - 'geni_urn':'', #For client_helper.py compatibility - 'keys':'', #For client_helper.py compatibility - 'key_ids':''}) #For client_helper.py compatibility - - -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - #Get iotlab slice record and oar job id if any. - recslice_list = self.iotlab_api.GetSlices(slice_filter = \ -======= - #Get slab slice record and oar job id if any. - recslice_list = self.slab_api.GetSlices(slice_filter = \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - str(record['hrn']),\ - slice_filter_type = 'slice_hrn') + record.update( + {'PI': [record['reg_researchers']['hrn']], + 'researcher': [record['reg_researchers']['hrn']], + 'name': record['hrn'], + 'oar_job_id': [], + 'node_ids': [], + 'person_ids': [record['reg_researchers'] + ['record_id']], + # For client_helper.py compatibility + 'geni_urn': '', + # For client_helper.py compatibility + 'keys': '', + # For client_helper.py compatibility + 'key_ids': ''}) + #Get iotlab slice record and oar job id if any. + recslice_list = self.iotlab_api.GetSlices( + slice_filter=str(record['hrn']), + slice_filter_type='slice_hrn') -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py logger.debug("IOTLABDRIVER \tfill_record_info \ -======= - logger.debug("SLABDRIVER \tfill_record_info \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - TYPE SLICE RECUSER record['hrn'] %s ecord['oar_job_id']\ - %s " %(record['hrn'], record['oar_job_id'])) + TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\ + %s " % (record['hrn'], record['oar_job_id'])) del record['reg_researchers'] try: for rec in recslice_list: -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py logger.debug("IOTLABDRIVER\r\n \t \ - fill_record_info oar_job_id %s " \ - %(rec['oar_job_id'])) - - record['node_ids'] = [ self.iotlab_api.root_auth + \ -======= - logger.debug("SLABDRIVER\r\n \t \ - fill_record_info oar_job_id %s " \ - %(rec['oar_job_id'])) - - record['node_ids'] = [ self.slab_api.root_auth + \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - hostname for hostname in rec['node_ids']] + fill_record_info oar_job_id %s " + % (rec['oar_job_id'])) + + record['node_ids'] = [self.iotlab_api.root_auth + + '.' + hostname for hostname + in rec['node_ids']] except KeyError: pass - -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug( "IOTLABDRIVER.PY \t fill_record_info SLICE \ -======= - logger.debug( "SLABDRIVER.PY \t fill_record_info SLICE \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py + logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \ recslice_list %s \r\n \t RECORD %s \r\n \ - \r\n" %(recslice_list, record)) + \r\n" % (recslice_list, record)) if str(record['type']) == 'user': #The record is a SFA user record. #Get the information about his slice from Iotlab's DB #and add it to the user record. - recslice_list = self.iotlab_api.GetSlices(\ - slice_filter = record['record_id'],\ - slice_filter_type = 'record_id_user') - -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug( "IOTLABDRIVER.PY \t fill_record_info TYPE USER \ -======= - logger.debug( "SLABDRIVER.PY \t fill_record_info TYPE USER \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - recslice_list %s \r\n \t RECORD %s \r\n" \ - %(recslice_list , record)) + recslice_list = self.iotlab_api.GetSlices( + slice_filter=record['record_id'], + slice_filter_type='record_id_user') + + logger.debug("IOTLABDRIVER.PY \t fill_record_info \ + TYPE USER recslice_list %s \r\n \t RECORD %s \r\n" + % (recslice_list, record)) #Append slice record in records list, #therefore fetches user and slice info again(one more loop) #Will update PIs and researcher for the slice recuser = recslice_list[0]['reg_researchers'] -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug( "IOTLABDRIVER.PY \t fill_record_info USER \ -======= - logger.debug( "SLABDRIVER.PY \t fill_record_info USER \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - recuser %s \r\n \r\n" %(recuser)) + logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \ + recuser %s \r\n \r\n" % (recuser)) recslice = {} recslice = recslice_list[0] - recslice.update({'PI':[recuser['hrn']], - 'researcher': [recuser['hrn']], - 'name':record['hrn'], - 'node_ids': [], - 'oar_job_id': [], - 'person_ids':[recuser['record_id']]}) + recslice.update( + {'PI': [recuser['hrn']], + 'researcher': [recuser['hrn']], + 'name': record['hrn'], + 'node_ids': [], + 'oar_job_id': [], + 'person_ids': [recuser['record_id']]}) try: for rec in recslice_list: recslice['oar_job_id'].append(rec['oar_job_id']) except KeyError: pass - recslice.update({'type':'slice', \ - 'hrn':recslice_list[0]['hrn']}) - + recslice.update({'type': 'slice', + 'hrn': recslice_list[0]['hrn']}) #GetPersons takes [] as filters -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py user_iotlab = self.iotlab_api.GetPersons([record]) - record.update(user_iotlab[0]) -======= - user_slab = self.slab_api.GetPersons([record]) - - - record.update(user_slab[0]) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py #For client_helper.py compatibility - record.update( { 'geni_urn':'', - 'keys':'', - 'key_ids':'' }) + record.update( + {'geni_urn': '', + 'keys': '', + 'key_ids': ''}) record_list.append(recslice) -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug("IOTLABDRIVER.PY \tfill_record_info ADDING SLICE\ -======= - logger.debug("SLABDRIVER.PY \tfill_record_info ADDING SLICE\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - INFO TO USER records %s" %(record_list)) - + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info ADDING SLICE\ + INFO TO USER records %s" % (record_list)) except TypeError, error: - logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"\ - %(error)) + logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s" + % (error)) return record_list - def sliver_status(self, slice_urn, slice_hrn): """ Receive a status request for slice named urn/hrn -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice -======= - urn:publicid:IDN+senslab+nturro_slice hrn senslab.nturro_slice ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - shall return a structure as described in - http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus - NT : not sure if we should implement this or not, but used by sface. + urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice + shall return a structure as described in + http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus + NT : not sure if we should implement this or not, but used by sface. :param slice_urn: slice urn :type slice_urn: string @@ -278,60 +213,47 @@ class SlabDriver(Driver): """ - #First get the slice with the slice hrn - slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \ - slice_filter_type = 'slice_hrn') + slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn, + slice_filter_type='slice_hrn') - if len(slice_list) is 0: + if len(slice_list) == 0: raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn)) #Used for fetching the user info witch comes along the slice info one_slice = slice_list[0] - #Make a list of all the nodes hostnames in use for this slice slice_nodes_list = [] - #for single_slice in slice_list: - #for node in single_slice['node_ids']: - #slice_nodes_list.append(node['hostname']) - #for node in one_slice: - #slice_nodes_list.append(node['hostname']) slice_nodes_list = one_slice['node_ids'] #Get all the corresponding nodes details -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - nodes_all = self.iotlab_api.GetNodes({'hostname':slice_nodes_list}, -======= - nodes_all = self.slab_api.GetNodes({'hostname':slice_nodes_list}, ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - ['node_id', 'hostname','site','boot_state']) - nodeall_byhostname = dict([(one_node['hostname'], one_node) \ - for one_node in nodes_all]) - - + nodes_all = self.iotlab_api.GetNodes( + {'hostname': slice_nodes_list}, + ['node_id', 'hostname', 'site', 'boot_state']) + nodeall_byhostname = dict([(one_node['hostname'], one_node) + for one_node in nodes_all]) for single_slice in slice_list: - #For compatibility top_level_status = 'empty' result = {} - result.fromkeys(\ - ['geni_urn','geni_error', 'pl_login','geni_status','geni_resources'], None) - result['pl_login'] = one_slice['reg_researchers'][0].hrn + result.fromkeys( + ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status', + 'geni_resources'], None) + # result.fromkeys(\ + # ['geni_urn','geni_error', 'pl_login','geni_status', + # 'geni_resources'], None) + # result['pl_login'] = one_slice['reg_researchers'][0].hrn + result['iotlab_login'] = one_slice['user'] logger.debug("Slabdriver - sliver_status Sliver status \ - urn %s hrn %s single_slice %s \r\n " \ - %(slice_urn, slice_hrn, single_slice)) + urn %s hrn %s single_slice %s \r\n " + % (slice_urn, slice_hrn, single_slice)) if 'node_ids' not in single_slice: #No job in the slice result['geni_status'] = top_level_status result['geni_resources'] = [] return result -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py -======= - - top_level_status = 'ready' ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py top_level_status = 'ready' @@ -343,22 +265,18 @@ class SlabDriver(Driver): resources = [] for node_hostname in single_slice['node_ids']: res = {} -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py res['iotlab_hostname'] = node_hostname - res['iotlab_boot_state'] = nodeall_byhostname[node_hostname]['boot_state'] -======= - res['slab_hostname'] = node_hostname - res['slab_boot_state'] = nodeall_byhostname[node_hostname]['boot_state'] ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py + res['iotlab_boot_state'] = \ + nodeall_byhostname[node_hostname]['boot_state'] #res['pl_hostname'] = node['hostname'] #res['pl_boot_state'] = \ #nodeall_byhostname[node['hostname']]['boot_state'] #res['pl_last_contact'] = strftime(self.time_format, \ #gmtime(float(timestamp))) - sliver_id = Xrn(slice_urn, type='slice', \ - id=nodeall_byhostname[node_hostname]['node_id'], \ - authority=self.hrn).urn + sliver_id = Xrn( + slice_urn, type='slice', + id=nodeall_byhostname[node_hostname]['node_id']).urn res['geni_urn'] = sliver_id #node_name = node['hostname'] @@ -375,44 +293,43 @@ class SlabDriver(Driver): result['geni_status'] = top_level_status result['geni_resources'] = resources -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "\ -======= - logger.debug("SLABDRIVER \tsliver_statusresources %s res %s "\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - %(resources,res)) + logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s " + % (resources, res)) return result @staticmethod def get_user_record(hrn): """ + Returns the user record based on the hrn from the SFA DB . :param hrn: user's hrn :type hrn: string - :return : user record from SFA database + :returns: user record from SFA database :rtype: RegUser """ - return dbsession.query(RegRecord).filter_by(hrn = hrn).first() - + return dbsession.query(RegRecord).filter_by(hrn=hrn).first() - def testbed_name (self): + def testbed_name(self): """ - Returns testbed's name. + Returns testbed's name. + :returns: testbed authority name. :rtype: string + """ return self.hrn # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory - def aggregate_version (self): + def aggregate_version(self): """ - Returns the testbed's supported rspec advertisement and - request versions. - + Returns the testbed's supported rspec advertisement and request + versions. + :returns: rspec versions supported ad a dictionary. :rtype: dict + """ version_manager = VersionManager() ad_rspec_versions = [] @@ -423,57 +340,43 @@ class SlabDriver(Driver): if rspec_version.content_type in ['*', 'request']: request_rspec_versions.append(rspec_version.to_dict()) return { - 'testbed':self.testbed_name(), + 'testbed': self.testbed_name(), 'geni_request_rspec_versions': request_rspec_versions, - 'geni_ad_rspec_versions': ad_rspec_versions, - } - - + 'geni_ad_rspec_versions': ad_rspec_versions} def _get_requested_leases_list(self, rspec): """ Process leases in rspec depending on the rspec version (format) - type. Find the lease requests in the rspec and creates - a lease request list with the mandatory information ( nodes, - start time and duration) of the valid leases (duration above or equal -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - to the iotlab experiment minimum duration). -======= - to the senslab experiment minimum duration). ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py + type. Find the lease requests in the rspec and creates + a lease request list with the mandatory information ( nodes, + start time and duration) of the valid leases (duration above or + equal to the iotlab experiment minimum duration). :param rspec: rspec request received. :type rspec: RSpec - :return: list of lease requests found in the rspec + :returns: list of lease requests found in the rspec :rtype: list """ requested_lease_list = [] for lease in rspec.version.get_leases(): single_requested_lease = {} -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug("IOTLABDRIVER.PY \t_get_requested_leases_list lease %s " %(lease)) -======= - logger.debug("SLABDRIVER.PY \t_get_requested_leases_list lease %s " %(lease)) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py + logger.debug("IOTLABDRIVER.PY \t \ + _get_requested_leases_list lease %s " % (lease)) if not lease.get('lease_id'): if get_authority(lease['component_id']) == \ - self.iotlab_api.root_auth: + self.iotlab_api.root_auth: single_requested_lease['hostname'] = \ - iotlab_xrn_to_hostname(\ - lease.get('component_id').strip()) + iotlab_xrn_to_hostname(\ + lease.get('component_id').strip()) single_requested_lease['start_time'] = \ - lease.get('start_time') + lease.get('start_time') single_requested_lease['duration'] = lease.get('duration') #Check the experiment's duration is valid before adding #the lease to the requested leases list duration_in_seconds = \ - int(single_requested_lease['duration']) -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec() : -======= - if duration_in_seconds >= self.slab_api.GetMinExperimentDurationInSec() : ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py + int(single_requested_lease['duration']) + if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec(): requested_lease_list.append(single_requested_lease) return requested_lease_list @@ -482,14 +385,15 @@ class SlabDriver(Driver): def _group_leases_by_start_time(requested_lease_list): """ Create dict of leases by start_time, regrouping nodes reserved - at the same time, for the same amount of time so as to - define one job on OAR. + at the same time, for the same amount of time so as to + define one job on OAR. :param requested_lease_list: list of leases :type requested_lease_list: list - :return: Dictionary with key = start time, value = list of leases - with the same start time. + :returns: Dictionary with key = start time, value = list of leases + with the same start time. :rtype: dictionary + """ requested_job_dict = {} @@ -503,12 +407,11 @@ class SlabDriver(Driver): if isinstance(lease['hostname'], str): lease['hostname'] = [lease['hostname']] - requested_job_dict[lease['start_time']] = lease - else : + else: job_lease = requested_job_dict[lease['start_time']] - if lease['duration'] == job_lease['duration'] : + if lease['duration'] == job_lease['duration']: job_lease['hostname'].append(lease['hostname']) return requested_job_dict @@ -516,33 +419,30 @@ class SlabDriver(Driver): def _process_requested_jobs(self, rspec): """ Turns the requested leases and information into a dictionary - of requested jobs, grouped by starting time. + of requested jobs, grouped by starting time. :param rspec: RSpec received :type rspec : RSpec :rtype: dictionary + """ requested_lease_list = self._get_requested_leases_list(rspec) - logger.debug("IOTLABDRIVER _process_requested_jobs requested_lease_list \ - %s"%(requested_lease_list)) - job_dict = self._group_leases_by_start_time(requested_lease_list) -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py + logger.debug("IOTLABDRIVER _process_requested_jobs \ + requested_lease_list %s" % (requested_lease_list)) + job_dict = self._group_leases_by_start_time(requested_lease_list) logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\ -======= - logger.debug("SLABDRIVER _process_requested_jobs job_dict\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - %s"%(job_dict)) + %s" % (job_dict)) return job_dict - def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \ - users, options): - """ - Answer to CreateSliver. + def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string, + users, options): + """Answer to CreateSliver. + Creates the leases and slivers for the users from the information - found in the rspec string. - Launch experiment on OAR if the requested leases is valid. Delete - no longer requested leases. + found in the rspec string. + Launch experiment on OAR if the requested leases is valid. Delete + no longer requested leases. :param creds: user's credentials @@ -552,21 +452,15 @@ class SlabDriver(Driver): :param options: :type options: - :return: a valid Rspec for the slice which has just been - modified. + :returns: a valid Rspec for the slice which has just been + modified. :rtype: RSpec """ -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py aggregate = IotlabAggregate(self) slices = IotlabSlices(self) -======= - aggregate = SlabAggregate(self) - - slices = SlabSlices(self) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py peer = slices.get_peer(slice_hrn) sfa_peer = slices.get_sfa_peer(slice_hrn) slice_record = None @@ -576,50 +470,40 @@ class SlabDriver(Driver): if users: slice_record = users[0].get('slice_record', {}) -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\ -======= - logger.debug("SLABDRIVER.PY \t ===============create_sliver \t\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - creds %s \r\n \r\n users %s" \ - %(creds, users)) - slice_record['user'] = {'keys':users[0]['keys'], \ - 'email':users[0]['email'], \ - 'hrn':slice_record['reg-researchers'][0]} + creds %s \r\n \r\n users %s" + % (creds, users)) + slice_record['user'] = {'keys': users[0]['keys'], + 'email': users[0]['email'], + 'hrn': slice_record['reg-researchers'][0]} # parse rspec rspec = RSpec(rspec_string) logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \ - %s slice_record %s users %s" \ - %(rspec.version,slice_record, users)) - + %s slice_record %s users %s" + % (rspec.version, slice_record, users)) # ensure site record exists? # ensure slice record exists - #Removed options to verify_slice SA 14/08/12 - sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \ - sfa_peer) + #Removed options in verify_slice SA 14/08/12 + #Removed peer record in verify_slice SA 18/07/13 + sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer) # ensure person records exists - #verify_persons returns added persons but since the return value + #verify_persons returns added persons but the return value #is not used - slices.verify_persons(slice_hrn, sfa_slice, users, peer, \ - sfa_peer, options=options) + #Removed peer record and sfa_peer in verify_persons SA 18/07/13 + slices.verify_persons(slice_hrn, sfa_slice, users, options=options) #requested_attributes returned by rspec.version.get_slice_attributes() #unused, removed SA 13/08/12 #rspec.version.get_slice_attributes() - logger.debug("IOTLABDRIVER.PY create_sliver slice %s " %(sfa_slice)) + logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice)) # add/remove slice from nodes -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py -======= - # add/remove slice from nodes - ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py #requested_slivers = [node.get('component_id') \ - #for node in rspec.version.get_nodes_with_slivers()\ - #if node.get('authority_id') is self.iotlab_api.root_auth] + #for node in rspec.version.get_nodes_with_slivers()\ + #if node.get('authority_id') is self.iotlab_api.root_auth] #l = [ node for node in rspec.version.get_nodes_with_slivers() ] #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \ #requested_slivers %s listnodes %s" \ @@ -627,105 +511,86 @@ class SlabDriver(Driver): #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) - requested_job_dict = self._process_requested_jobs(rspec) - -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\ -======= - logger.debug("SLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - %(requested_job_dict)) + logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s " + % (requested_job_dict)) #verify_slice_leases returns the leases , but the return value is unused #here. Removed SA 13/08/12 - slices.verify_slice_leases(sfa_slice, \ - requested_job_dict, peer) - - return aggregate.get_rspec(slice_xrn=slice_urn, \ - login=sfa_slice['login'], version=rspec.version) + slices.verify_slice_leases(sfa_slice, + requested_job_dict, peer) + return aggregate.get_rspec(slice_xrn=slice_urn, + login=sfa_slice['login'], + version=rspec.version) - def delete_sliver (self, slice_urn, slice_hrn, creds, options): + def delete_sliver(self, slice_urn, slice_hrn, creds, options): """ Deletes the lease associated with the slice hrn and the credentials -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - if the slice belongs to iotlab. Answer to DeleteSliver. - - :return: 1 if the slice to delete was not found on iotlab, - True if the deletion was successful, False otherwise otherwise. + if the slice belongs to iotlab. Answer to DeleteSliver. - .. note:: Should really be named delete_leases because iotlab does -======= - if the slice belongs to senslab. Answer to DeleteSliver. + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused - :return: 1 if the slice to delete was not found on senslab, - True if the deletion was successful, False otherwise otherwise. + :returns: 1 if the slice to delete was not found on iotlab, + True if the deletion was successful, False otherwise otherwise. - .. note:: Should really be named delete_leases because senslab does ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - not have any slivers, but only deals with leases. However, SFA api only - have delete_sliver define so far. SA 13.05/2013 + .. note:: Should really be named delete_leases because iotlab does + not have any slivers, but only deals with leases. However, + SFA api only have delete_sliver define so far. SA 13/05/2013 + .. note:: creds are unused, and are not used either in the dummy driver + delete_sliver . """ -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - sfa_slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \ -======= - sfa_slice_list = self.slab_api.GetSlices(slice_filter = slice_hrn, \ ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - slice_filter_type = 'slice_hrn') + sfa_slice_list = self.iotlab_api.GetSlices( + slice_filter=slice_hrn, + slice_filter_type='slice_hrn') if not sfa_slice_list: return 1 #Delete all leases in the slice for sfa_slice in sfa_slice_list: - - -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" %(sfa_slice)) + logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice)) slices = IotlabSlices(self) # determine if this is a peer slice -======= - logger.debug("SLABDRIVER.PY delete_sliver slice %s" %(sfa_slice)) - slices = SlabSlices(self) - # determine if this is a peer slice - - peer = slices.get_peer(slice_hrn) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py peer = slices.get_peer(slice_hrn) logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \ - \r\n \t sfa_slice %s " %(peer, sfa_slice)) + \r\n \t sfa_slice %s " % (peer, sfa_slice)) try: - -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py self.iotlab_api.DeleteSliceFromNodes(sfa_slice) -======= - self.slab_api.DeleteSliceFromNodes(sfa_slice) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py return True - except : + except: return False - def list_resources (self, slice_urn, slice_hrn, creds, options): """ -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py + List resources from the iotlab aggregate and returns a Rspec -======= - List resources from the senslab aggregate and returns a Rspec ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - advertisement with resources found when slice_urn and slice_hrn are None - (in case of resource discovery). - If a slice hrn and urn are provided, list experiment's slice - nodes in a rspec format. Answer to ListResources. - Caching unused. + advertisement with resources found when slice_urn and slice_hrn are + None (in case of resource discovery). + If a slice hrn and urn are provided, list experiment's slice + nodes in a rspec format. Answer to ListResources. + Caching unused. + + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused :param options: options used when listing resources (list_leases, info, - geni_available) - :return: rspec string in xml + geni_available) + :returns: rspec string in xml :rtype: string + + .. note:: creds are unused """ #cached_requested = options.get('cached', True) @@ -733,13 +598,13 @@ class SlabDriver(Driver): version_manager = VersionManager() # get the rspec's return format from options rspec_version = \ - version_manager.get_version(options.get('geni_rspec_version')) + version_manager.get_version(options.get('geni_rspec_version')) version_string = "rspec_%s" % (rspec_version) #panos adding the info option to the caching key (can be improved) if options.get('info'): version_string = version_string + "_" + \ - options.get('info', 'default') + options.get('info', 'default') # Adding the list_leases option to the caching key if options.get('list_leases'): @@ -760,37 +625,30 @@ class SlabDriver(Driver): #return rspec #panos: passing user-defined options -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py aggregate = IotlabAggregate(self) -======= - aggregate = SlabAggregate(self) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - rspec = aggregate.get_rspec(slice_xrn=slice_urn, \ - version=rspec_version, options=options) + rspec = aggregate.get_rspec(slice_xrn=slice_urn, + version=rspec_version, options=options) # cache the result #if self.cache and not slice_hrn: - #logger.debug("Slab.ListResources: stores advertisement in cache") + #logger.debug("Iotlab.ListResources: stores advertisement in cache") #self.cache.add(version_string, rspec) return rspec - def list_slices (self, creds, options): - """ - Answer to ListSlices. -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py + def list_slices(self, creds, options): + """Answer to ListSlices. + List slices belonging to iotlab, returns slice urns list. -======= - List slices belonging to senslab, returns slice urns list. ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - No caching used. Options unused but are defined in the SFA method - api prototype. + No caching used. Options unused but are defined in the SFA method + api prototype. - :return: slice urns list + :returns: slice urns list :rtype: list + .. note:: creds are unused """ # look in cache first #if self.cache: @@ -801,78 +659,77 @@ class SlabDriver(Driver): # get data from db -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py slices = self.iotlab_api.GetSlices() - logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices)) + logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" + % (slices)) slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices] -======= - slices = self.slab_api.GetSlices() - logger.debug("SLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices)) - slice_hrns = [slab_slice['hrn'] for slab_slice in slices] ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - slice_urns = [hrn_to_urn(slice_hrn, 'slice') \ - for slice_hrn in slice_hrns] + slice_urns = [hrn_to_urn(slice_hrn, 'slice') + for slice_hrn in slice_hrns] # cache the result #if self.cache: -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py #logger.debug ("IotlabDriver.list_slices stores value in cache") -======= - #logger.debug ("SlabDriver.list_slices stores value in cache") ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py #self.cache.add('slices', slice_urns) return slice_urns - def register (self, sfa_record, hrn, pub_key): + def register(self, sfa_record, hrn, pub_key): """ Adding new user, slice, node or site should not be handled - by SFA. + by SFA. ..warnings:: should not be used. Different components are in charge of - doing this task. Adding nodes = OAR - Adding users = LDAP Iotlab - Adding slice = Import from LDAP users - Adding site = OAR + doing this task. Adding nodes = OAR + Adding users = LDAP Iotlab + Adding slice = Import from LDAP users + Adding site = OAR :param sfa_record: record provided by the client of the - Register API call. + Register API call. :type sfa_record: dict + :param pub_key: public key of the user + :type pub_key: string + + .. note:: DOES NOTHING. Returns -1. + """ return -1 - def update (self, old_sfa_record, new_sfa_record, hrn, new_key): -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py - """No site or node record update allowed in Iotlab. -======= - """No site or node record update allowed in Senslab. ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py - The only modifications authorized here are key deletion/addition - on an existing user and password change. - On an existing user, CAN NOT BE MODIFIED: - 'first_name', 'last_name', 'email' - DOES NOT EXIST IN SENSLAB: - 'phone', 'url', 'bio','title', 'accepted_aup', - A slice is bound to its user, so modifying the user's ssh key should - modify the slice's GID after an import procedure. + def update(self, old_sfa_record, new_sfa_record, hrn, new_key): + """ + No site or node record update allowed in Iotlab. + The only modifications authorized here are key deletion/addition + on an existing user and password change. + On an existing user, CAN NOT BE MODIFIED: + 'first_name', 'last_name', 'email' + DOES NOT EXIST IN SENSLAB: + 'phone', 'url', 'bio','title', 'accepted_aup', + A slice is bound to its user, so modifying the user's ssh key should + modify the slice's GID after an import procedure. :param old_sfa_record: what is in the db for this hrn :param new_sfa_record: what was passed to the Update call + :param new_key: the new user's public key + :param hrn: the user's sfa hrn + :type old_sfa_record: dictionary + :type new_sfa_record: dictionary + :type pub_key: string + :type hrn: string - ..seealso:: update in driver.py. - """ + TODO: needs review + .. seealso::: update in driver.py. + """ pointer = old_sfa_record['pointer'] old_sfa_record_type = old_sfa_record['type'] # new_key implemented for users only - if new_key and old_sfa_record_type not in [ 'user' ]: + if new_key and old_sfa_record_type not in ['user']: raise UnknownSfaType(old_sfa_record_type) - if old_sfa_record_type == "user": update_fields = {} all_fields = new_sfa_record @@ -880,18 +737,13 @@ class SlabDriver(Driver): if key in ['key', 'password']: update_fields[key] = all_fields[key] - if new_key: # must check this key against the previous one if it exists persons = self.iotlab_api.GetPersons([old_sfa_record]) person = persons[0] keys = [person['pkey']] #Get all the person's keys -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py keys_dict = self.iotlab_api.GetKeys(keys) -======= - keys_dict = self.slab_api.GetKeys(keys) ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py # Delete all stale keys, meaning the user has only one key #at a time @@ -905,27 +757,26 @@ class SlabDriver(Driver): #remove all the other keys for key in keys_dict: self.iotlab_api.DeleteKey(person, key) - self.iotlab_api.AddPersonKey(person, \ - {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} ) - #self.iotlab_api.AddPersonKey(person, {'key_type': 'ssh', \ - #'key': new_key}) + self.iotlab_api.AddPersonKey( + person, {'sshPublicKey': person['pkey']}, + {'sshPublicKey': new_key}) return True - - def remove (self, sfa_record): + def remove(self, sfa_record): """ - Removes users only. Mark the user as disabled in - LDAP. The user and his slice are then deleted from the db by running an - import on the registry. - + Removes users only. Mark the user as disabled in + LDAP. The user and his slice are then deleted from the + db by running an import on the registry. :param sfa_record: record is the existing sfa record in the db :type sfa_record: dict ..warning::As fas as the slice is concerned, here only the leases are - removed from the slice. The slice is record itself is not removed from - the db. + removed from the slice. The slice is record itself is not removed + from the db. + TODO: needs review + TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, TODO: return boolean for the slice part @@ -934,29 +785,17 @@ class SlabDriver(Driver): hrn = sfa_record['hrn'] if sfa_record_type == 'user': -<<<<<<< HEAD:sfa/iotlab/iotlabdriver.py #get user from iotlab ldap person = self.iotlab_api.GetPersons(sfa_record) #No registering at a given site in Iotlab. #Once registered to the LDAP, all iotlab sites are -======= - #get user from senslab ldap - person = self.slab_api.GetPersons(sfa_record) - #No registering at a given site in Senslab. - #Once registered to the LDAP, all senslab sites are ->>>>>>> 3fe7429... SA:sfa/senslab/slabdriver.py #accesible. - if person : + if person: #Mark account as disabled in ldap return self.iotlab_api.DeletePerson(sfa_record) elif sfa_record_type == 'slice': - if self.iotlab_api.GetSlices(slice_filter = hrn, \ - slice_filter_type = 'slice_hrn'): + if self.iotlab_api.GetSlices(slice_filter=hrn, + slice_filter_type='slice_hrn'): ret = self.iotlab_api.DeleteSlice(sfa_record) - - - return True - - diff --git a/sfa/iotlab/iotlabpostgres.py b/sfa/iotlab/iotlabpostgres.py index 4e3c6df4..70bc1510 100644 --- a/sfa/iotlab/iotlabpostgres.py +++ b/sfa/iotlab/iotlabpostgres.py @@ -1,14 +1,17 @@ +""" +File defining classes to handle the table in the iotlab dedicated database. +""" + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker - -from sfa.util.config import Config +# from sfa.util.config import Config from sfa.util.sfalogging import logger from sqlalchemy import Column, Integer, String from sqlalchemy import Table, MetaData from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.dialects import postgresql +# from sqlalchemy.dialects import postgresql from sqlalchemy.exc import NoSuchTableError @@ -16,8 +19,8 @@ from sqlalchemy.exc import NoSuchTableError #Dict holding the columns names of the table as keys #and their type, used for creation of the table slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \ -CASCADE ON UPDATE CASCADE','oar_job_id':'integer DEFAULT -1', \ -'record_id_slice':'integer', 'slice_hrn':'text NOT NULL'} + CASCADE ON UPDATE CASCADE', 'oar_job_id': 'integer DEFAULT -1', + 'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'} #Dict with all the specific iotlab tables tablenames_dict = {'iotlab_xp': slice_table} @@ -26,138 +29,225 @@ tablenames_dict = {'iotlab_xp': slice_table} IotlabBase = declarative_base() - class IotlabXP (IotlabBase): - """ SQL alchemy class to manipulate slice_iotlab table in - iotlab_sfa database. + """ SQL alchemy class to manipulate the rows of the slice_iotlab table in + iotlab_sfa database. Handles the records representation and creates the + table if it does not exist yet. """ __tablename__ = 'iotlab_xp' - slice_hrn = Column(String) - job_id = Column(Integer, primary_key = True) - end_time = Column(Integer, nullable = False) - - - #oar_job_id = Column( Integer,default = -1) - #node_list = Column(postgresql.ARRAY(String), nullable =True) + job_id = Column(Integer, primary_key=True) + end_time = Column(Integer, nullable=False) - def __init__ (self, slice_hrn =None, job_id=None, end_time=None): + def __init__(self, slice_hrn=None, job_id=None, end_time=None): """ Defines a row of the slice_iotlab table """ if slice_hrn: self.slice_hrn = slice_hrn - if job_id : + if job_id: self.job_id = job_id if end_time: self.end_time = end_time - def __repr__(self): """Prints the SQLAlchemy record to the format defined by the function. """ result = " 0: + self.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + self.iotlab_session.commit() + return + + def __init__(self, config, debug=False): + self.sl_base = IotlabBase - """ - self.iotlab_engine.execute ("select 1").scalar() + # Check whether we already have an instance + if IotlabDB._connection_singleton is None: + IotlabDB._connection_singleton = IotlabDB.Singleton(config, debug) + # Store instance reference as the only member in the handle + self._EventHandler_singleton = IotlabDB._connection_singleton - def session (self): + def __getattr__(self, aAttr): """ - Creates a SQLalchemy session. Once the session object is created - it should be used throughout the code for all the operations on - tables for this given database. + Delegate access to implementation. + :param aAttr: Attribute wanted. + :returns: Attribute """ - if self.iotlab_session is None: - Session = sessionmaker() - self.iotlab_session = Session(bind = self.iotlab_engine) - return self.iotlab_session + return getattr(self._connection_singleton, aAttr) - def close_session(self): - """ - Closes connection to database. - """ - if self.iotlab_session is None: return - self.iotlab_session.close() - self.iotlab_session = None + # def __setattr__(self, aAttr, aValue): + # """Delegate access to implementation. + + # :param attr: Attribute wanted. + # :param value: Vaule to be set. + # :return: Result of operation. + # """ + # return setattr(self._connection_singleton, aAttr, aValue) def exists(self, tablename): """ Checks if the table specified as tablename exists. + :param tablename: name of the table in the db that has to be checked. + :type tablename: string + :returns: True if the table exists, False otherwise. + :rtype: bool """ - + metadata = MetaData(bind=self.iotlab_engine) try: - metadata = MetaData (bind=self.iotlab_engine) - table = Table (tablename, metadata, autoload=True) + table = Table(tablename, metadata, autoload=True) return True except NoSuchTableError: - logger.log_exc("SLABPOSTGRES tablename %s does not exists" \ - %(tablename)) + logger.log_exc("SLABPOSTGRES tablename %s does not exist" + % (tablename)) return False - def createtable(self): """ Creates all the table sof the engine. @@ -165,13 +255,8 @@ class IotlabDB: """ - logger.debug("SLABPOSTGRES createtable IotlabBase.metadata.sorted_tables \ - %s \r\n engine %s" %(IotlabBase.metadata.sorted_tables , iotlab_engine)) - IotlabBase.metadata.create_all(iotlab_engine) + logger.debug("SLABPOSTGRES createtable \ + IotlabBase.metadata.sorted_tables %s \r\n engine %s" + % (IotlabBase.metadata.sorted_tables, self.iotlab_engine)) + IotlabBase.metadata.create_all(self.iotlab_engine) return - - - -iotlab_alchemy = IotlabDB(Config()) -iotlab_engine = iotlab_alchemy.iotlab_engine -iotlab_dbsession = iotlab_alchemy.session() diff --git a/sfa/iotlab/iotlabslices.py b/sfa/iotlab/iotlabslices.py index e449b213..f44575c9 100644 --- a/sfa/iotlab/iotlabslices.py +++ b/sfa/iotlab/iotlabslices.py @@ -1,13 +1,22 @@ +""" +This file defines the IotlabSlices class by which all the slice checkings +upon lease creation are done. +""" from sfa.util.xrn import get_authority, urn_to_hrn from sfa.util.sfalogging import logger +MAXINT = 2L**31-1 -MAXINT = 2L**31-1 class IotlabSlices: - - rspec_to_slice_tag = {'max_rate':'net_max_rate'} - + """ + This class is responsible for checking the slice when creating a + lease or a sliver. Those checks include verifying that the user is valid, + that the slice is known from the testbed or from our peers, that the list + of nodes involved has not changed (in this case the lease is modified + accordingly). + """ + rspec_to_slice_tag = {'max_rate': 'net_max_rate'} def __init__(self, driver): """ @@ -15,13 +24,18 @@ class IotlabSlices: """ self.driver = driver - def get_peer(self, xrn): """ - Find the authority of a resources based on its xrn. + Finds the authority of a resource based on its xrn. If the authority is Iotlab (local) return None, Otherwise, look up in the DB if Iotlab is federated with this site - authority and returns its DB record if it is the case, + authority and returns its DB record if it is the case. + + :param xrn: resource's xrn + :type xrn: string + :returns: peer record + :rtype: dict + """ hrn, hrn_type = urn_to_hrn(xrn) #Does this slice belong to a local site or a peer iotlab site? @@ -31,29 +45,36 @@ class IotlabSlices: slice_authority = get_authority(hrn) #Iotlab stuff #This slice belongs to the current site - if slice_authority == self.driver.iotlab_api.root_auth: + if slice_authority == self.driver.iotlab_api.root_auth: site_authority = slice_authority return None site_authority = get_authority(slice_authority).lower() # get this site's authority (sfa root authority or sub authority) - logger.debug("IOTLABSLICES \ get_peer slice_authority %s \ - site_authority %s hrn %s" %(slice_authority, \ - site_authority, hrn)) - + logger.debug("IOTLABSLICES \t get_peer slice_authority %s \ + site_authority %s hrn %s" + % (slice_authority, site_authority, hrn)) # check if we are already peered with this site_authority #if so find the peer record - peers = self.driver.iotlab_api.GetPeers(peer_filter = site_authority) + peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority) for peer_record in peers: - if site_authority == peer_record.hrn: peer = peer_record - logger.debug(" IOTLABSLICES \tget_peer peer %s " %(peer)) + logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer)) return peer def get_sfa_peer(self, xrn): + """Returns the authority name for the xrn or None if the local site + is the authority. + + :param xrn: the xrn of the resource we are looking the authority for. + :type xrn: string + :returns: the resources's authority name. + :rtype: string + + """ hrn, hrn_type = urn_to_hrn(xrn) # return the authority for this hrn or None if we are the authority @@ -66,7 +87,6 @@ class IotlabSlices: return sfa_peer - def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer): """ Compare requested leases with the leases already scheduled/ @@ -75,36 +95,35 @@ class IotlabSlices: :param sfa_slice: sfa slice record :param requested_jobs_dict: dictionary of requested leases - :param peer: sfa peer + :param peer: sfa peer record :type sfa_slice: dict :type requested_jobs_dict: dict - :type peer: - :return: leases list of dictionary + :type peer: dict + :returns: leases list of dictionary :rtype: list """ - logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s \ - "%( sfa_slice)) + logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s " + % (sfa_slice)) #First get the list of current leases from OAR - leases = self.driver.iotlab_api.GetLeases({'name':sfa_slice['hrn']}) + leases = self.driver.iotlab_api.GetLeases({'name': sfa_slice['hrn']}) logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \ - leases %s "%(requested_jobs_dict, leases )) + leases %s " % (requested_jobs_dict, leases)) current_nodes_reserved_by_start_time = {} requested_nodes_by_start_time = {} leases_by_start_time = {} reschedule_jobs_dict = {} - #Create reduced dictionary with key start_time and value # the list of nodes #-for the leases already registered by OAR first # then for the new leases requested by the user #Leases already scheduled/running in OAR - for lease in leases : + for lease in leases: current_nodes_reserved_by_start_time[lease['t_from']] = \ lease['reserved_nodes'] leases_by_start_time[lease['t_from']] = lease @@ -116,8 +135,8 @@ class IotlabSlices: #Requested jobs for start_time in requested_jobs_dict: - requested_nodes_by_start_time[int(start_time)] = \ - requested_jobs_dict[start_time]['hostname'] + requested_nodes_by_start_time[int(start_time)] = \ + requested_jobs_dict[start_time]['hostname'] #Check if there is any difference between the leases already #registered in OAR and the requested jobs. #Difference could be: @@ -127,7 +146,7 @@ class IotlabSlices: logger.debug("IOTLABSLICES verify_slice_leases \ requested_nodes_by_start_time %s \ - "%(requested_nodes_by_start_time )) + "% (requested_nodes_by_start_time)) #Find all deleted leases start_time_list = \ list(set(leases_by_start_time.keys()).\ @@ -136,7 +155,6 @@ class IotlabSlices: for start_time in start_time_list] - #Find added or removed nodes in exisiting leases for start_time in requested_nodes_by_start_time: logger.debug("IOTLABSLICES verify_slice_leases start_time %s \ @@ -184,91 +202,112 @@ class IotlabSlices: job = requested_jobs_dict[str(start_time)] logger.debug("IOTLABSLICES \ - NEWLEASE slice %s job %s"\ - %(sfa_slice, job)) - self.driver.iotlab_api.AddLeases(job['hostname'], \ - sfa_slice, int(job['start_time']), \ - int(job['duration'])) + NEWLEASE slice %s job %s" + % (sfa_slice, job)) + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) #Deleted leases are the ones with lease id not declared in the Rspec if deleted_leases: - self.driver.iotlab_api.DeleteLeases(deleted_leases, sfa_slice['hrn']) + self.driver.iotlab_api.DeleteLeases(deleted_leases, + sfa_slice['user']['uid']) logger.debug("IOTLABSLICES \ - verify_slice_leases slice %s deleted_leases %s"\ - %(sfa_slice, deleted_leases)) - + verify_slice_leases slice %s deleted_leases %s" + % (sfa_slice, deleted_leases)) - if reschedule_jobs_dict : - for start_time in reschedule_jobs_dict: + if reschedule_jobs_dict: + for start_time in reschedule_jobs_dict: job = reschedule_jobs_dict[start_time] - self.driver.iotlab_api.AddLeases(job['hostname'], \ - sfa_slice, int(job['start_time']), \ + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), int(job['duration'])) return leases def verify_slice_nodes(self, sfa_slice, requested_slivers, peer): + """Check for wanted and unwanted nodes in the slice. + + Removes nodes and associated leases that the user does not want anymore + by deleteing the associated job in OAR (DeleteSliceFromNodes). + Returns the nodes' hostnames that are going to be in the slice. + + :param sfa_slice: slice record. Must contain node_ids and list_node_ids. + + :param requested_slivers: list of requested nodes' hostnames. + :param peer: unused so far. + + :type sfa_slice: dict + :type requested_slivers: list + :type peer: string + + :returns: list requested nodes hostnames + :rtype: list + + .. warning:: UNUSED SQA 24/07/13 + .. seealso:: DeleteSliceFromNodes + .. todo:: check what to do with the peer? Can not remove peer nodes from + slice here. Anyway, in this case, the peer should have gotten the + remove request too. + + """ current_slivers = [] deleted_nodes = [] if 'node_ids' in sfa_slice: - nodes = self.driver.iotlab_api.GetNodes(sfa_slice['list_node_ids'], \ + nodes = self.driver.iotlab_api.GetNodes( + sfa_slice['list_node_ids'], ['hostname']) current_slivers = [node['hostname'] for node in nodes] # remove nodes not in rspec - deleted_nodes = list(set(current_slivers).\ - difference(requested_slivers)) - # add nodes from rspec - #added_nodes = list(set(requested_slivers).\ - #difference(current_slivers)) - + deleted_nodes = list(set(current_slivers). + difference(requested_slivers)) logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\ - \r\n \r\n deleted_nodes %s"\ - %(sfa_slice, deleted_nodes)) + \r\n \r\n deleted_nodes %s" + % (sfa_slice, deleted_nodes)) if deleted_nodes: #Delete the entire experience self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice) - #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \ - #deleted_nodes) return nodes + def verify_slice(self, slice_hrn, slice_record, sfa_peer): + """Ensures slice record exists. + The slice record must exist either in Iotlab or in the other + federated testbed (sfa_peer). If the slice does not belong to Iotlab, + check if the user already exists in LDAP. In this case, adds the slice + to the sfa DB and associates its LDAP user. - def free_egre_key(self): - used = set() - for tag in self.driver.iotlab_api.GetSliceTags({'tagname': 'egre_key'}): - used.add(int(tag['value'])) - - for i in range(1, 256): - if i not in used: - key = i - break - else: - raise KeyError("No more EGRE keys available") - - return str(key) - - + :param slice_hrn: slice's name + :param slice_record: sfa record of the slice + :param sfa_peer: name of the peer authority if any.(not Iotlab). + :type slice_hrn: string + :type slice_record: dictionary + :type sfa_peer: string + .. seealso:: AddSlice - def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer): + """ - #login_base = slice_hrn.split(".")[0] slicename = slice_hrn - slices_list = self.driver.iotlab_api.GetSlices(slice_filter = slicename, \ - slice_filter_type = 'slice_hrn') + # check if slice belongs to Iotlab + slices_list = self.driver.iotlab_api.GetSlices( + slice_filter=slicename, slice_filter_type='slice_hrn') + sfa_slice = None + if slices_list: for sl in slices_list: - logger.debug("SLABSLICE \tverify_slice slicename %s \ - slices_list %s sl %s \ slice_record %s"\ - %(slicename, slices_list,sl, \ - slice_record)) + logger.debug("SLABSLICE \t verify_slice slicename %s \ + slices_list %s sl %s \r slice_record %s" + % (slicename, slices_list, sl, slice_record)) sfa_slice = sl sfa_slice.update(slice_record) @@ -277,83 +316,67 @@ class IotlabSlices: ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\ slice_record['user']) logger.debug(" IOTLABSLICES \tverify_slice Oups \ - slice_record %s sfa_peer %s ldap_user %s"\ - %(slice_record, sfa_peer, ldap_user )) + slice_record %s sfa_peer %s ldap_user %s" + % (slice_record, sfa_peer, ldap_user)) #User already registered in ldap, meaning user should be in SFA db #and hrn = sfa_auth+ uid sfa_slice = {'hrn': slicename, - #'url': slice_record.get('url', slice_hrn), - #'description': slice_record.get('description', slice_hrn) - 'node_list' : [], - 'authority' : slice_record['authority'], - 'gid':slice_record['gid'], - #'record_id_user' : user.record_id, - 'slice_id' : slice_record['record_id'], - 'reg-researchers':slice_record['reg-researchers'], - #'record_id_slice': slice_record['record_id'], - 'peer_authority':str(sfa_peer) - - } - if ldap_user : - hrn = self.driver.iotlab_api.root_auth +'.'+ ldap_user['uid'] - + 'node_list': [], + 'authority': slice_record['authority'], + 'gid': slice_record['gid'], + 'slice_id': slice_record['record_id'], + 'reg-researchers': slice_record['reg-researchers'], + 'peer_authority': str(sfa_peer) + } + + if ldap_user: + hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid'] user = self.driver.get_user_record(hrn) - logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" \ - %(hrn, user)) - #sfa_slice = {'slice_hrn': slicename, - ##'url': slice_record.get('url', slice_hrn), - ##'description': slice_record.get('description', slice_hrn) - #'node_list' : [], - #'authority' : slice_record['authority'], - #'gid':slice_record['gid'], - ##'record_id_user' : user.record_id, - #'slice_id' : slice_record['record_id'], - #'reg-researchers':slice_record['reg-researchers'], - ##'record_id_slice': slice_record['record_id'], - #'peer_authority':str(peer.hrn) - - #} - # add the slice - if sfa_slice : - self.driver.iotlab_api.AddSlice(sfa_slice, user) + logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" + % (hrn, user)) - if peer: - sfa_slice['slice_id'] = slice_record['record_id'] + # add the external slice to the local SFA iotlab DB + if sfa_slice: + self.driver.iotlab_api.AddSlice(sfa_slice, user) - #slice['slice_id'] = self.driver.iotlab_api.AddSlice(slice) logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK") - #slice['node_ids']=[] - #slice['person_ids'] = [] - #if peer: - #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None) - # mark this slice as an sfa peer record - #if sfa_peer: - #peer_dict = {'type': 'slice', 'hrn': slice_hrn, - #'peer_authority': sfa_peer, 'pointer': \ - #slice['slice_id']} - #self.registry.register_peer_object(self.credential, peer_dict) + return sfa_slice + def verify_persons(self, slice_hrn, slice_record, users, options={}): + """Ensures the users in users list exist and are enabled in LDAP. Adds + person if needed. - return sfa_slice + Checking that a user exist is based on the user's email. If the user is + still not found in the LDAP, it means that the user comes from another + federated. In this case an account has to be created in LDAP + so as to enable the user to use the testbed, since we trust the testbed + he comes from. This is done by calling AddPerson. + + :param slice_hrn: slice name + :param slice_record: record of the slice_hrn + :param users: users is a record list. Records can either be + local records or users records from known and trusted federated + sites.If the user is from another site that iotlab doesn't trust yet, + then Resolve will raise an error before getting to create_sliver. + + :type slice_hrn: string + :type slice_record: string + :type users: list + + .. seealso:: AddPerson + .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13. - def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \ - options={}): - """ - users is a record list. Records can either be local records - or users records from known and trusted federated sites. - If the user is from another site that iotlab doesn't trust yet, - then Resolve will raise an error before getting to create_sliver. """ #TODO SA 21/08/12 verify_persons Needs review logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \ - \t slice_record %s\r\n users %s \t peer %s "\ - %( slice_hrn, slice_record, users, peer)) + \t slice_record %s\r\n users %s \t " + % (slice_hrn, slice_record, users)) users_by_id = {} - #users_by_hrn = {} + users_by_email = {} #users_dict : dict whose keys can either be the user's hrn or its id. #Values contains only id and hrn @@ -361,8 +384,7 @@ class IotlabSlices: #First create dicts by hrn and id for each user in the user record list: for info in users: - - if 'slice_record' in info : + if 'slice_record' in info: slice_rec = info['slice_record'] user = slice_rec['user'] @@ -370,44 +392,36 @@ class IotlabSlices: users_by_email[user['email']] = user users_dict[user['email']] = user - #if 'hrn' in user: - #users_by_hrn[user['hrn']] = user - #users_dict[user['hrn']] = user - - logger.debug( "SLABSLICE.PY \t verify_person \ + logger.debug("SLABSLICE.PY \t verify_person \ users_dict %s \r\n user_by_email %s \r\n \ - \tusers_by_id %s " \ - %(users_dict,users_by_email, users_by_id)) + \tusers_by_id %s " + % (users_dict, users_by_email, users_by_id)) existing_user_ids = [] - #existing_user_hrns = [] existing_user_emails = [] existing_users = [] # Check if user is in Iotlab LDAP using its hrn. # Assuming Iotlab is centralised : one LDAP for all sites, - # user'as record_id unknown from LDAP - # LDAP does not provide users id, therefore we rely on hrns containing - # the login of the user. - # If the hrn is not a iotlab hrn, the user may not be in LDAP. + # user's record_id unknown from LDAP + # LDAP does not provide users id, therefore we rely on email to find the + # user in LDAP - if users_by_email : + if users_by_email: #Construct the list of filters (list of dicts) for GetPersons - filter_user = [] - for email in users_by_email : - filter_user.append (users_by_email[email]) - #Check user's in LDAP with GetPersons + filter_user = [users_by_email[email] for email in users_by_email] + #Check user i in LDAP with GetPersons #Needed because what if the user has been deleted in LDAP but #is still in SFA? existing_users = self.driver.iotlab_api.GetPersons(filter_user) logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user \ - %s existing_users %s " \ - %(filter_user, existing_users)) - #User's in iotlab LDAP + %s existing_users %s " + % (filter_user, existing_users)) + #User is in iotlab LDAP if existing_users: - for user in existing_users : + for user in existing_users: users_dict[user['email']].update(user) - existing_user_emails.append(\ - users_dict[user['email']]['email']) + existing_user_emails.append( + users_dict[user['email']]['email']) # User from another known trusted federated site. Check @@ -415,52 +429,39 @@ class IotlabSlices: else: req = 'mail=' if isinstance(users, list): - req += users[0]['email'] else: req += users['email'] - ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req) if ldap_reslt: logger.debug(" SLABSLICE.PY \tverify_person users \ USER already in Iotlab \t ldap_reslt %s \ - "%( ldap_reslt)) + " % (ldap_reslt)) existing_users.append(ldap_reslt[1]) else: #User not existing in LDAP - #TODO SA 21/08/12 raise smthg to add user or add it auto ? - #new_record = {} - #new_record['pkey'] = users[0]['keys'][0] - #new_record['mail'] = users[0]['email'] - - logger.debug(" SLABSLICE.PY \tverify_person users \ + logger.debug("SLABSLICE.PY \tverify_person users \ not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \ - ldap_reslt %s " %(users, ldap_reslt)) + ldap_reslt %s " % (users, ldap_reslt)) requested_user_emails = users_by_email.keys() requested_user_hrns = \ - [users_by_email[user]['hrn'] for user in users_by_email] + [users_by_email[user]['hrn'] for user in users_by_email] logger.debug("SLABSLICE.PY \tverify_person \ - users_by_email %s " %( users_by_email)) - #logger.debug("SLABSLICE.PY \tverify_person \ - #user_by_hrn %s " %( users_by_hrn)) - + users_by_email %s " % (users_by_email)) #Check that the user of the slice in the slice record #matches one of the existing users try: if slice_record['PI'][0] in requested_user_hrns: - #if slice_record['record_id_user'] in requested_user_ids and \ - #slice_record['PI'][0] in requested_user_hrns: logger.debug(" SLABSLICE \tverify_person ['PI']\ - slice_record %s" %(slice_record)) + slice_record %s" % (slice_record)) except KeyError: pass - # users to be added, removed or updated #One user in one iotlab slice : there should be no need #to remove/ add any user from/to a slice. @@ -475,23 +476,19 @@ class IotlabSlices: added_persons = [] # add new users - #requested_user_email is in existing_user_emails if len(added_user_emails) == 0: - slice_record['login'] = users_dict[requested_user_emails[0]]['uid'] - logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" \ - %(slice_record)) - + logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" + % (slice_record)) for added_user_email in added_user_emails: - #hrn, type = urn_to_hrn(added_user['urn']) added_user = users_dict[added_user_email] logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person \ - added_user %s" %(added_user)) + added_user %s" % (added_user)) person = {} - person['peer_person_id'] = None - k_list = ['first_name', 'last_name','person_id'] + person['peer_person_id'] = None + k_list = ['first_name', 'last_name', 'person_id'] for k in k_list: if k in added_user: person[k] = added_user[k] @@ -499,26 +496,25 @@ class IotlabSlices: person['pkey'] = added_user['keys'][0] person['mail'] = added_user['email'] person['email'] = added_user['email'] - person['key_ids'] = added_user.get('key_ids', []) - #person['urn'] = added_user['urn'] + person['key_ids'] = added_user.get('key_ids', []) - #person['person_id'] = self.driver.iotlab_api.AddPerson(person) ret = self.driver.iotlab_api.AddPerson(person) - if type(ret) == int : + if type(ret) == int: person['uid'] = ret logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person\ - personne %s" %(person)) + person %s" % (person)) #Update slice_Record with the id now known to LDAP slice_record['login'] = person['uid'] added_persons.append(person) - - return added_persons - #Unused + def verify_keys(self, persons, users, peer, options={}): + """ + .. warning:: unused + """ # existing keys key_ids = [] for person in persons: @@ -549,10 +545,11 @@ class IotlabSlices: #try: ##if peer: #person = persondict[user['email']] - #self.driver.iotlab_api.UnBindObjectFromPeer('person', - #person['person_id'], peer['shortname']) - ret = self.driver.iotlab_api.AddPersonKey(\ - user['email'], key) + #self.driver.iotlab_api.UnBindObjectFromPeer( + # 'person',person['person_id'], + # peer['shortname']) + ret = self.driver.iotlab_api.AddPersonKey( + user['email'], key) #if peer: #key_index = user_keys.index(key['key']) #remote_key_id = user['key_ids'][key_index] @@ -568,7 +565,7 @@ class IotlabSlices: # remove old keys (only if we are not appending) append = options.get('append', True) - if append == False: + if append is False: removed_keys = set(existing_keys).difference(requested_keys) for key in removed_keys: #if peer: diff --git a/sfa/rspecs/elements/versions/iotlabv1Lease.py b/sfa/rspecs/elements/versions/iotlabv1Lease.py index 7840f083..601297a3 100644 --- a/sfa/rspecs/elements/versions/iotlabv1Lease.py +++ b/sfa/rspecs/elements/versions/iotlabv1Lease.py @@ -1,8 +1,6 @@ from sfa.util.sfalogging import logger from sfa.util.xml import XpathFilter from sfa.util.xrn import Xrn -from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch - #from sfa.rspecs.elements.versions.sfav1PLTag import SFAv1PLTag #from sfa.rspecs.elements.versions.pgv2Services import PGv2Services @@ -26,7 +24,6 @@ class Iotlabv1Lease: lease_elems = [] for lease in leases: - lease['start_time'] = datetime_to_string(utcparse(lease['start_time'])) lease_fields = ['lease_id', 'component_id', 'slice_id', 'start_time', 'duration'] lease_elem = network_elem.add_instance('lease', lease, lease_fields) @@ -48,7 +45,7 @@ class Iotlabv1Lease: for node_elem in node_elems: lease = Lease(lease_elem.attrib, lease_elem) lease['slice_id'] = lease_elem.attrib['slice_id'] - lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + lease['start_time'] = lease_elem.attrib['start_time'] lease['duration'] = lease_elem.attrib['duration'] lease['component_id'] = node_elem.attrib['component_id'] leases.append(lease) diff --git a/sfa/rspecs/elements/versions/iotlabv1Node.py b/sfa/rspecs/elements/versions/iotlabv1Node.py index 2535c4e1..50f8e48e 100644 --- a/sfa/rspecs/elements/versions/iotlabv1Node.py +++ b/sfa/rspecs/elements/versions/iotlabv1Node.py @@ -1,7 +1,7 @@ from sfa.util.xrn import Xrn from sfa.util.xml import XpathFilter -from sfa.rspecs.elements.node import NodeElement +from sfa.rspecs.elements.node import Node from sfa.rspecs.elements.sliver import Sliver from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.hardware_type import HardwareType @@ -10,17 +10,10 @@ from sfa.rspecs.elements.interface import Interface from sfa.rspecs.elements.versions.iotlabv1Sliver import Iotlabv1Sliver from sfa.util.sfalogging import logger -<<<<<<< HEAD:sfa/rspecs/elements/versions/slabv1Node.py -class SlabNode(NodeElement): - #First get the fields already defined in the class Node - fields = list(NodeElement.fields) - #Extend it with senslab's specific fields -======= class IotlabNode(Node): #First get the fields already defined in the class Node fields = list(Node.fields) #Extend it with iotlab's specific fields ->>>>>>> 7cb1e78... Renaming Senslab into Iotlab.:sfa/rspecs/elements/versions/iotlabv1Node.py fields.extend (['archi', 'radio', 'mobile','position']) @@ -166,13 +159,8 @@ class Iotlabv1Node: def get_node_objs(node_elems): nodes = [] for node_elem in node_elems: -<<<<<<< HEAD:sfa/rspecs/elements/versions/slabv1Node.py - node = NodeElement(node_elem.attrib, node_elem) - nodes.append(node) -======= node = Node(node_elem.attrib, node_elem) nodes.append(node) ->>>>>>> 7cb1e78... Renaming Senslab into Iotlab.:sfa/rspecs/elements/versions/iotlabv1Node.py if 'component_id' in node_elem.attrib: node['authority_id'] = \ Xrn(node_elem.attrib['component_id']).get_authority_urn() diff --git a/sfa/rspecs/versions/iotlabv1.py b/sfa/rspecs/versions/iotlabv1.py index a4a83004..cee1f901 100644 --- a/sfa/rspecs/versions/iotlabv1.py +++ b/sfa/rspecs/versions/iotlabv1.py @@ -19,7 +19,7 @@ class Iotlabv1(RSpecVersion): valid Iotlab XML Rspec. """ #enabled = True - type = 'Slab' + type = 'Iotlab' content_type = 'ad' version = '1' #template = '' % type -- 2.43.0