based on the iotlab driver.
So far, all the files in the cortexlab are copies of
iotlab's, with a very few modifications. First draft.
--- /dev/null
+"""
+This API is adapted for OpenLDAP. The file contains all LDAP classes and methods
+needed to:
+- Load the LDAP connection configuration file (login, address..) with LdapConfig
+- Connect to LDAP with ldap_co
+- Create a unique LDAP login and password for a user based on his email or last
+name and first name with LoginPassword.
+- Manage entries in LDAP using SFA records with LDAPapi (Search, Add, Delete,
+Modify)
+
+"""
+import random
+from passlib.hash import ldap_salted_sha1 as lssha
+
+from sfa.util.xrn import get_authority
+from sfa.util.sfalogging import logger
+from sfa.util.config import Config
+
+import ldap
+import ldap.modlist as modlist
+
+import os.path
+
+
+class LdapConfig():
+ """
+ Ldap configuration class loads the configuration file and sets the
+ ldap IP address, password, people dn, web dn, group dn. All these settings
+ were defined in a separate file ldap_config.py to avoid sharing them in
+ the SFA git as it contains sensible information.
+
+ """
+ def __init__(self, config_file='/etc/sfa/ldap_config.py'):
+ """Loads configuration from file /etc/sfa/ldap_config.py and set the
+ parameters for connection to LDAP.
+
+ """
+
+ try:
+ execfile(config_file, self.__dict__)
+
+ self.config_file = config_file
+ # path to configuration data
+ self.config_path = os.path.dirname(config_file)
+ except IOError:
+ raise IOError, "Could not find or load the configuration file: %s" \
+ % config_file
+
+
+class ldap_co:
+ """ Set admin login and server configuration variables."""
+
+ def __init__(self):
+ """Fetch LdapConfig attributes (Ldap server connection parameters and
+ defines port , version and subtree scope.
+
+ """
+ #Iotlab PROD LDAP parameters
+ self.ldapserv = None
+ ldap_config = LdapConfig()
+ self.config = ldap_config
+ self.ldapHost = ldap_config.LDAP_IP_ADDRESS
+ self.ldapPeopleDN = ldap_config.LDAP_PEOPLE_DN
+ self.ldapGroupDN = ldap_config.LDAP_GROUP_DN
+ self.ldapAdminDN = ldap_config.LDAP_WEB_DN
+ self.ldapAdminPassword = ldap_config.LDAP_WEB_PASSWORD
+ self.ldapPort = ldap.PORT
+ self.ldapVersion = ldap.VERSION3
+ self.ldapSearchScope = ldap.SCOPE_SUBTREE
+
+ def connect(self, bind=True):
+ """Enables connection to the LDAP server.
+
+ :param bind: Set the bind parameter to True if a bind is needed
+ (for add/modify/delete operations). Set to False otherwise.
+ :type bind: boolean
+ :returns: dictionary with status of the connection. True if Successful,
+ False if not and in this case the error
+ message( {'bool', 'message'} ).
+ :rtype: dict
+
+ """
+ try:
+ self.ldapserv = ldap.open(self.ldapHost)
+ except ldap.LDAPError, error:
+ return {'bool': False, 'message': error}
+
+ # Bind with authentification
+ if(bind):
+ return self.bind()
+
+ else:
+ return {'bool': True}
+
+ def bind(self):
+ """ Binding method.
+
+ :returns: dictionary with the bind status. True if Successful,
+ False if not and in this case the error message({'bool','message'})
+ :rtype: dict
+
+ """
+ try:
+ # Opens a connection after a call to ldap.open in connect:
+ self.ldapserv = ldap.initialize("ldap://" + self.ldapHost)
+
+ # Bind/authenticate with a user with apropriate
+ #rights to add objects
+ self.ldapserv.simple_bind_s(self.ldapAdminDN,
+ self.ldapAdminPassword)
+
+ except ldap.LDAPError, error:
+ return {'bool': False, 'message': error}
+
+ return {'bool': True}
+
+ def close(self):
+ """Close the LDAP connection.
+
+ Can throw an exception if the unbinding fails.
+
+ :returns: dictionary with the bind status if the unbinding failed and
+ in this case the dict contains an error message. The dictionary keys
+ are : ({'bool','message'})
+ :rtype: dict or None
+
+ """
+ try:
+ self.ldapserv.unbind_s()
+ except ldap.LDAPError, error:
+ return {'bool': False, 'message': error}
+
+
+class LoginPassword():
+ """
+
+ Class to handle login and password generation, using custom login generation
+ algorithm.
+
+ """
+ def __init__(self):
+ """
+
+ Sets password and login maximum length, and defines the characters that
+ can be found in a random generated password.
+
+ """
+ self.login_max_length = 8
+ self.length_password = 8
+ self.chars_password = ['!', '$', '(',')', '*', '+', ',', '-', '.',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8',
+ '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+ 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q',
+ 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
+ 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q',
+ 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '\'']
+
+ @staticmethod
+ def clean_user_names(record):
+ """
+
+ Removes special characters such as '-', '_' , '[', ']' and ' ' from the
+ first name and last name.
+
+ :param record: user's record
+ :type record: dict
+ :returns: lower_first_name and lower_last_name if they were found
+ in the user's record. Return None, none otherwise.
+ :rtype: string, string or None, None.
+
+ """
+ if 'first_name' in record and 'last_name' in record:
+ #Remove all special characters from first_name/last name
+ lower_first_name = record['first_name'].replace('-', '')\
+ .replace('_', '').replace('[', '')\
+ .replace(']', '').replace(' ', '')\
+ .lower()
+ lower_last_name = record['last_name'].replace('-', '')\
+ .replace('_', '').replace('[', '')\
+ .replace(']', '').replace(' ', '')\
+ .lower()
+ return lower_first_name, lower_last_name
+ else:
+ return None, None
+
+ @staticmethod
+ def extract_name_from_email(record):
+ """
+
+ When there is no valid first name and last name in the record,
+ the email is used to generate the login. Here, we assume the email
+ is firstname.lastname@something.smthg. The first name and last names
+ are extracted from the email, special charcaters are removed and
+ they are changed into lower case.
+
+ :param record: user's data
+ :type record: dict
+ :returns: the first name and last name taken from the user's email.
+ lower_first_name, lower_last_name.
+ :rtype: string, string
+
+ """
+
+ email = record['email']
+ email = email.split('@')[0].lower()
+ lower_first_name = None
+ lower_last_name = None
+ #Assume there is first name and last name in email
+ #if there is a separator
+ separator_list = ['.', '_', '-']
+ for sep in separator_list:
+ if sep in email:
+ mail = email.split(sep)
+ lower_first_name = mail[0]
+ lower_last_name = mail[1]
+ break
+
+ #Otherwise just take the part before the @ as the
+ #lower_first_name and lower_last_name
+ if lower_first_name is None:
+ lower_first_name = email
+ lower_last_name = email
+
+ return lower_first_name, lower_last_name
+
+ def get_user_firstname_lastname(self, record):
+ """
+
+ Get the user first name and last name from the information we have in
+ the record.
+
+ :param record: user's information
+ :type record: dict
+ :returns: the user's first name and last name.
+
+ .. seealso:: clean_user_names
+ .. seealso:: extract_name_from_email
+
+ """
+ lower_first_name, lower_last_name = self.clean_user_names(record)
+
+ #No first name and last name check email
+ if lower_first_name is None and lower_last_name is None:
+
+ lower_first_name, lower_last_name = \
+ self.extract_name_from_email(record)
+
+ return lower_first_name, lower_last_name
+
+ def choose_sets_chars_for_login(self, lower_first_name, lower_last_name):
+ """
+
+ Algorithm to select sets of characters from the first name and last
+ name, depending on the lenght of the last name and the maximum login
+ length which in our case is set to 8 characters.
+
+ :param lower_first_name: user's first name in lower case.
+ :param lower_last_name: usr's last name in lower case.
+ :returns: user's login
+ :rtype: string
+
+ """
+ length_last_name = len(lower_last_name)
+ self.login_max_length = 8
+
+ #Try generating a unique login based on first name and last name
+
+ if length_last_name >= self.login_max_length:
+ login = lower_last_name[0:self.login_max_length]
+ index = 0
+ logger.debug("login : %s index : %s" % (login, index))
+ elif length_last_name >= 4:
+ login = lower_last_name
+ index = 0
+ logger.debug("login : %s index : %s" % (login, index))
+ elif length_last_name == 3:
+ login = lower_first_name[0:1] + lower_last_name
+ index = 1
+ logger.debug("login : %s index : %s" % (login, index))
+ elif length_last_name == 2:
+ if len(lower_first_name) >= 2:
+ login = lower_first_name[0:2] + lower_last_name
+ index = 2
+ logger.debug("login : %s index : %s" % (login, index))
+ else:
+ logger.error("LoginException : \
+ Generation login error with \
+ minimum four characters")
+
+ else:
+ logger.error("LDAP LdapGenerateUniqueLogin failed : \
+ impossible to generate unique login for %s %s"
+ % (lower_first_name, lower_last_name))
+ return index, login
+
+ def generate_password(self):
+ """
+
+ Generate a password upon adding a new user in LDAP Directory
+ (8 characters length). The generated password is composed of characters
+ from the chars_password list.
+
+ :returns: the randomly generated password
+ :rtype: string
+
+ """
+ password = str()
+
+ length = len(self.chars_password)
+ for index in range(self.length_password):
+ char_index = random.randint(0, length - 1)
+ password += self.chars_password[char_index]
+
+ return password
+
+ @staticmethod
+ def encrypt_password(password):
+ """
+
+ Use passlib library to make a RFC2307 LDAP encrypted password salt size
+ is 8, use sha-1 algorithm.
+
+ :param password: password not encrypted.
+ :type password: string
+ :returns: Returns encrypted password.
+ :rtype: string
+
+ """
+ #Keep consistency with Java Iotlab's LDAP API
+ #RFC2307SSHAPasswordEncryptor so set the salt size to 8 bytes
+ return lssha.encrypt(password, salt_size=8)
+
+
+class LDAPapi:
+ """Defines functions to insert and search entries in the LDAP.
+
+ .. note:: class supposes the unix schema is used
+
+ """
+ def __init__(self):
+ logger.setLevelDebug()
+
+ #SFA related config
+
+ config = Config()
+ self.login_pwd = LoginPassword()
+ self.authname = config.SFA_REGISTRY_ROOT_AUTH
+ self.conn = ldap_co()
+ self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS
+ self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN
+ self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER
+ self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH
+ self.baseDN = self.conn.ldapPeopleDN
+ self.ldapShell = '/bin/bash'
+
+
+ def LdapGenerateUniqueLogin(self, record):
+ """
+
+ Generate login for adding a new user in LDAP Directory
+ (four characters minimum length). Get proper last name and
+ first name so that the user's login can be generated.
+
+ :param record: Record must contain first_name and last_name.
+ :type record: dict
+ :returns: the generated login for the user described with record if the
+ login generation is successful, None if it fails.
+ :rtype: string or None
+
+ """
+ #For compatibility with other ldap func
+ if 'mail' in record and 'email' not in record:
+ record['email'] = record['mail']
+
+ lower_first_name, lower_last_name = \
+ self.login_pwd.get_user_firstname_lastname(record)
+
+ index, login = self.login_pwd.choose_sets_chars_for_login(
+ lower_first_name, lower_last_name)
+
+ login_filter = '(uid=' + login + ')'
+ get_attrs = ['uid']
+ try:
+ #Check if login already in use
+
+ while (len(self.LdapSearch(login_filter, get_attrs)) is not 0):
+
+ index += 1
+ if index >= 9:
+ logger.error("LoginException : Generation login error \
+ with minimum four characters")
+ else:
+ try:
+ login = \
+ lower_first_name[0:index] + \
+ lower_last_name[0:
+ self.login_pwd.login_max_length
+ - index]
+ login_filter = '(uid=' + login + ')'
+ except KeyError:
+ print "lower_first_name - lower_last_name too short"
+
+ logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s"
+ % (login))
+ return login
+
+ except ldap.LDAPError, error:
+ logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error))
+ return None
+
+ def find_max_uidNumber(self):
+ """Find the LDAP max uidNumber (POSIX uid attribute).
+
+ Used when adding a new user in LDAP Directory
+
+ :returns: max uidNumber + 1
+ :rtype: string
+
+ """
+ #First, get all the users in the LDAP
+ get_attrs = "(uidNumber=*)"
+ login_filter = ['uidNumber']
+
+ result_data = self.LdapSearch(get_attrs, login_filter)
+ #It there is no user in LDAP yet, First LDAP user
+ if result_data == []:
+ max_uidnumber = self.ldapUserUidNumberMin
+ #Otherwise, get the highest uidNumber
+ else:
+ uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data]
+ logger.debug("LDAPapi.py \tfind_max_uidNumber \
+ uidNumberList %s " % (uidNumberList))
+ max_uidnumber = max(uidNumberList) + 1
+
+ return str(max_uidnumber)
+
+
+ def get_ssh_pkey(self, record):
+ """TODO ; Get ssh public key from sfa record
+ To be filled by N. Turro ? or using GID pl way?
+
+ """
+ return 'A REMPLIR '
+
+ @staticmethod
+ #TODO Handle OR filtering in the ldap query when
+ #dealing with a list of records instead of doing a for loop in GetPersons
+ def make_ldap_filters_from_record(record=None):
+ """Helper function to make LDAP filter requests out of SFA records.
+
+ :param record: user's sfa record. Should contain first_name,last_name,
+ email or mail, and if the record is enabled or not. If the dict
+ record does not have all of these, must at least contain the user's
+ email.
+ :type record: dict
+ :returns: LDAP request
+ :rtype: string
+
+ """
+ req_ldap = ''
+ req_ldapdict = {}
+ if record :
+ if 'first_name' in record and 'last_name' in record:
+ if record['first_name'] != record['last_name']:
+ req_ldapdict['cn'] = str(record['first_name'])+" "\
+ + str(record['last_name'])
+ if 'email' in record:
+ req_ldapdict['mail'] = record['email']
+ if 'mail' in record:
+ req_ldapdict['mail'] = record['mail']
+ if 'enabled' in record:
+ if record['enabled'] is True:
+ req_ldapdict['shadowExpire'] = '-1'
+ else:
+ req_ldapdict['shadowExpire'] = '0'
+
+ #Hrn should not be part of the filter because the hrn
+ #presented by a certificate of a SFA user not imported in
+ #Iotlab does not include the iotlab login in it
+ #Plus, the SFA user may already have an account with iotlab
+ #using another login.
+
+ logger.debug("\r\n \t LDAP.PY make_ldap_filters_from_record \
+ record %s req_ldapdict %s"
+ % (record, req_ldapdict))
+
+ for k in req_ldapdict:
+ req_ldap += '(' + str(k) + '=' + str(req_ldapdict[k]) + ')'
+ if len(req_ldapdict.keys()) >1 :
+ req_ldap = req_ldap[:0]+"(&"+req_ldap[0:]
+ size = len(req_ldap)
+ req_ldap = req_ldap[:(size-1)] + ')' + req_ldap[(size-1):]
+ else:
+ req_ldap = "(cn=*)"
+
+ return req_ldap
+
+ def make_ldap_attributes_from_record(self, record):
+ """
+
+ When adding a new user to Iotlab's LDAP, creates an attributes
+ dictionnary from the SFA record understandable by LDAP. Generates the
+ user's LDAP login.User is automatically validated (account enabled)
+ and described as a SFA USER FROM OUTSIDE IOTLAB.
+
+ :param record: must contain the following keys and values:
+ first_name, last_name, mail, pkey (ssh key).
+ :type record: dict
+ :returns: dictionary of attributes using LDAP data structure model.
+ :rtype: dict
+
+ """
+
+ attrs = {}
+ attrs['objectClass'] = ["top", "person", "inetOrgPerson",
+ "organizationalPerson", "posixAccount",
+ "shadowAccount", "systemQuotas",
+ "ldapPublicKey"]
+
+ attrs['uid'] = self.LdapGenerateUniqueLogin(record)
+ try:
+ attrs['givenName'] = str(record['first_name']).lower().capitalize()
+ attrs['sn'] = str(record['last_name']).lower().capitalize()
+ attrs['cn'] = attrs['givenName'] + ' ' + attrs['sn']
+ attrs['gecos'] = attrs['givenName'] + ' ' + attrs['sn']
+
+ except KeyError:
+ attrs['givenName'] = attrs['uid']
+ attrs['sn'] = attrs['uid']
+ attrs['cn'] = attrs['uid']
+ attrs['gecos'] = attrs['uid']
+
+ attrs['quota'] = self.ldapUserQuotaNFS
+ attrs['homeDirectory'] = self.ldapUserHomePath + attrs['uid']
+ attrs['loginShell'] = self.ldapShell
+ attrs['gidNumber'] = self.ldapUserGidNumber
+ attrs['uidNumber'] = self.find_max_uidNumber()
+ attrs['mail'] = record['mail'].lower()
+ try:
+ attrs['sshPublicKey'] = record['pkey']
+ except KeyError:
+ attrs['sshPublicKey'] = self.get_ssh_pkey(record)
+
+
+ #Password is automatically generated because SFA user don't go
+ #through the Iotlab website used to register new users,
+ #There is no place in SFA where users can enter such information
+ #yet.
+ #If the user wants to set his own password , he must go to the Iotlab
+ #website.
+ password = self.login_pwd.generate_password()
+ attrs['userPassword'] = self.login_pwd.encrypt_password(password)
+
+ #Account automatically validated (no mail request to admins)
+ #Set to 0 to disable the account, -1 to enable it,
+ attrs['shadowExpire'] = '-1'
+
+ #Motivation field in Iotlab
+ attrs['description'] = 'SFA USER FROM OUTSIDE SENSLAB'
+
+ attrs['ou'] = 'SFA' #Optional: organizational unit
+ #No info about those here:
+ attrs['l'] = 'To be defined'#Optional: Locality.
+ attrs['st'] = 'To be defined' #Optional: state or province (country).
+
+ return attrs
+
+
+
+ def LdapAddUser(self, record) :
+ """Add SFA user to LDAP if it is not in LDAP yet.
+
+ :param record: dictionnary with the user's data.
+ :returns: a dictionary with the status (Fail= False, Success= True)
+ and the uid of the newly added user if successful, or the error
+ message it is not. Dict has keys bool and message in case of
+ failure, and bool uid in case of success.
+ :rtype: dict
+
+ .. seealso:: make_ldap_filters_from_record
+
+ """
+ logger.debug(" \r\n \t LDAP LdapAddUser \r\n\r\n ================\r\n ")
+ user_ldap_attrs = self.make_ldap_attributes_from_record(record)
+
+ #Check if user already in LDAP wih email, first name and last name
+ filter_by = self.make_ldap_filters_from_record(user_ldap_attrs)
+ user_exist = self.LdapSearch(filter_by)
+ if user_exist:
+ logger.warning(" \r\n \t LDAP LdapAddUser user %s %s \
+ already exists" % (user_ldap_attrs['sn'],
+ user_ldap_attrs['mail']))
+ return {'bool': False}
+
+ #Bind to the server
+ result = self.conn.connect()
+
+ if(result['bool']):
+
+ # A dict to help build the "body" of the object
+ logger.debug(" \r\n \t LDAP LdapAddUser attrs %s "
+ % user_ldap_attrs)
+
+ # The dn of our new entry/object
+ dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN
+
+ try:
+ ldif = modlist.addModlist(user_ldap_attrs)
+ logger.debug("LDAPapi.py add attrs %s \r\n ldif %s"
+ % (user_ldap_attrs, ldif))
+ self.conn.ldapserv.add_s(dn, ldif)
+
+ logger.info("Adding user %s login %s in LDAP"
+ % (user_ldap_attrs['cn'], user_ldap_attrs['uid']))
+ except ldap.LDAPError, error:
+ logger.log_exc("LDAP Add Error %s" % error)
+ return {'bool': False, 'message': error}
+
+ self.conn.close()
+ return {'bool': True, 'uid': user_ldap_attrs['uid']}
+ else:
+ return result
+
+ def LdapDelete(self, person_dn):
+ """Deletes a person in LDAP. Uses the dn of the user.
+
+ :param person_dn: user's ldap dn.
+ :type person_dn: string
+ :returns: dictionary with bool True if successful, bool False
+ and the error if not.
+ :rtype: dict
+
+ """
+ #Connect and bind
+ result = self.conn.connect()
+ if(result['bool']):
+ try:
+ self.conn.ldapserv.delete_s(person_dn)
+ self.conn.close()
+ return {'bool': True}
+
+ except ldap.LDAPError, error:
+ logger.log_exc("LDAP Delete Error %s" % error)
+ return {'bool': False, 'message': error}
+
+ def LdapDeleteUser(self, record_filter):
+ """Deletes a SFA person in LDAP, based on the user's hrn.
+
+ :param record_filter: Filter to find the user to be deleted. Must
+ contain at least the user's email.
+ :type record_filter: dict
+ :returns: dict with bool True if successful, bool False and error
+ message otherwise.
+ :rtype: dict
+
+ .. seealso:: LdapFindUser docstring for more info on record filter.
+ .. seealso:: LdapDelete for user deletion
+
+ """
+ #Find uid of the person
+ person = self.LdapFindUser(record_filter, [])
+ logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s"
+ % (record_filter, person))
+
+ if person:
+ dn = 'uid=' + person['uid'] + "," + self.baseDN
+ else:
+ return {'bool': False}
+
+ result = self.LdapDelete(dn)
+ return result
+
+ def LdapModify(self, dn, old_attributes_dict, new_attributes_dict):
+ """ Modifies a LDAP entry, replaces user's old attributes with
+ the new ones given.
+
+ :param dn: user's absolute name in the LDAP hierarchy.
+ :param old_attributes_dict: old user's attributes. Keys must match
+ the ones used in the LDAP model.
+ :param new_attributes_dict: new user's attributes. Keys must match
+ the ones used in the LDAP model.
+ :type dn: string
+ :type old_attributes_dict: dict
+ :type new_attributes_dict: dict
+ :returns: dict bool True if Successful, bool False if not.
+ :rtype: dict
+
+ """
+
+ ldif = modlist.modifyModlist(old_attributes_dict, new_attributes_dict)
+ # Connect and bind/authenticate
+ result = self.conn.connect()
+ if (result['bool']):
+ try:
+ self.conn.ldapserv.modify_s(dn, ldif)
+ self.conn.close()
+ return {'bool': True}
+ except ldap.LDAPError, error:
+ logger.log_exc("LDAP LdapModify Error %s" % error)
+ return {'bool': False}
+
+
+ def LdapModifyUser(self, user_record, new_attributes_dict):
+ """
+
+ Gets the record from one user based on the user sfa recordand changes
+ the attributes according to the specified new_attributes. Do not use
+ this if we need to modify the uid. Use a ModRDN operation instead
+ ( modify relative DN ).
+
+ :param user_record: sfa user record.
+ :param new_attributes_dict: new user attributes, keys must be the
+ same as the LDAP model.
+ :type user_record: dict
+ :type new_attributes_dict: dict
+ :returns: bool True if successful, bool False if not.
+ :rtype: dict
+
+ .. seealso:: make_ldap_filters_from_record for info on what is mandatory
+ in the user_record.
+ .. seealso:: make_ldap_attributes_from_record for the LDAP objectclass.
+
+ """
+ if user_record is None:
+ logger.error("LDAP \t LdapModifyUser Need user record ")
+ return {'bool': False}
+
+ #Get all the attributes of the user_uid_login
+ #person = self.LdapFindUser(record_filter,[])
+ req_ldap = self.make_ldap_filters_from_record(user_record)
+ person_list = self.LdapSearch(req_ldap, [])
+ logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s"
+ % (person_list))
+
+ if person_list and len(person_list) > 1:
+ logger.error("LDAP \t LdapModifyUser Too many users returned")
+ return {'bool': False}
+ if person_list is None:
+ logger.error("LDAP \t LdapModifyUser User %s doesn't exist "
+ % (user_record))
+ return {'bool': False}
+
+ # The dn of our existing entry/object
+ #One result only from ldapSearch
+ person = person_list[0][1]
+ dn = 'uid=' + person['uid'][0] + "," + self.baseDN
+
+ if new_attributes_dict:
+ old = {}
+ for k in new_attributes_dict:
+ if k not in person:
+ old[k] = ''
+ else:
+ old[k] = person[k]
+ logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s"
+ % (new_attributes_dict))
+ result = self.LdapModify(dn, old, new_attributes_dict)
+ return result
+ else:
+ logger.error("LDAP \t LdapModifyUser No new attributes given. ")
+ return {'bool': False}
+
+
+ def LdapMarkUserAsDeleted(self, record):
+ """
+
+ Sets shadowExpire to 0, disabling the user in LDAP. Calls LdapModifyUser
+ to change the shadowExpire of the user.
+
+ :param record: the record of the user who has to be disabled.
+ Should contain first_name,last_name, email or mail, and if the
+ record is enabled or not. If the dict record does not have all of
+ these, must at least contain the user's email.
+ :type record: dict
+ :returns: {bool: True} if successful or {bool: False} if not
+ :rtype: dict
+
+ .. seealso:: LdapModifyUser, make_ldap_attributes_from_record
+ """
+
+ new_attrs = {}
+ #Disable account
+ new_attrs['shadowExpire'] = '0'
+ logger.debug(" LDAPapi.py \t LdapMarkUserAsDeleted ")
+ ret = self.LdapModifyUser(record, new_attrs)
+ return ret
+
+ def LdapResetPassword(self, record):
+ """Resets password for the user whose record is the parameter and
+ changes the corresponding entry in the LDAP.
+
+ :param record: user's sfa record whose Ldap password must be reset.
+ Should contain first_name,last_name,
+ email or mail, and if the record is enabled or not. If the dict
+ record does not have all of these, must at least contain the user's
+ email.
+ :type record: dict
+ :returns: return value of LdapModifyUser. True if successful, False
+ otherwise.
+
+ .. seealso:: LdapModifyUser, make_ldap_attributes_from_record
+
+ """
+ password = self.login_pwd.generate_password()
+ attrs = {}
+ attrs['userPassword'] = self.login_pwd.encrypt_password(password)
+ logger.debug("LDAP LdapResetPassword encrypt_password %s"
+ % (attrs['userPassword']))
+ result = self.LdapModifyUser(record, attrs)
+ return result
+
+
+ def LdapSearch(self, req_ldap=None, expected_fields=None):
+ """
+ Used to search directly in LDAP, by using ldap filters and return
+ fields. When req_ldap is None, returns all the entries in the LDAP.
+
+ :param req_ldap: ldap style request, with appropriate filters,
+ example: (cn=*).
+ :param expected_fields: Fields in the user ldap entry that has to be
+ returned. If None is provided, will return 'mail', 'givenName',
+ 'sn', 'uid', 'sshPublicKey', 'shadowExpire'.
+ :type req_ldap: string
+ :type expected_fields: list
+
+ .. seealso:: make_ldap_filters_from_record for req_ldap format.
+
+ """
+ result = self.conn.connect(bind=False)
+ if (result['bool']):
+
+ return_fields_list = []
+ if expected_fields is None:
+ return_fields_list = ['mail', 'givenName', 'sn', 'uid',
+ 'sshPublicKey', 'shadowExpire']
+ else:
+ return_fields_list = expected_fields
+ #No specifc request specified, get the whole LDAP
+ if req_ldap is None:
+ req_ldap = '(cn=*)'
+
+ logger.debug("LDAP.PY \t LdapSearch req_ldap %s \
+ return_fields_list %s" \
+ %(req_ldap, return_fields_list))
+
+ try:
+ msg_id = self.conn.ldapserv.search(
+ self.baseDN, ldap.SCOPE_SUBTREE,
+ req_ldap, return_fields_list)
+ #Get all the results matching the search from ldap in one
+ #shot (1 value)
+ result_type, result_data = \
+ self.conn.ldapserv.result(msg_id, 1)
+
+ self.conn.close()
+
+ logger.debug("LDAP.PY \t LdapSearch result_data %s"
+ % (result_data))
+
+ return result_data
+
+ except ldap.LDAPError, error:
+ logger.log_exc("LDAP LdapSearch Error %s" % error)
+ return []
+
+ else:
+ logger.error("LDAP.PY \t Connection Failed")
+ return
+
+ def _process_ldap_info_for_all_users(self, result_data):
+ """Process the data of all enabled users in LDAP.
+
+ :param result_data: Contains information of all enabled users in LDAP
+ and is coming from LdapSearch.
+ :param result_data: list
+
+ .. seealso:: LdapSearch
+
+ """
+ results = []
+ logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s "
+ % (result_data))
+ for ldapentry in result_data:
+ logger.debug(" LDAP.py _process_ldap_info_for_all_users \
+ ldapentry name : %s " % (ldapentry[1]['uid'][0]))
+ tmpname = ldapentry[1]['uid'][0]
+ hrn = self.authname + "." + tmpname
+
+ tmpemail = ldapentry[1]['mail'][0]
+ if ldapentry[1]['mail'][0] == "unknown":
+ tmpemail = None
+
+ try:
+ results.append({
+ 'type': 'user',
+ 'pkey': ldapentry[1]['sshPublicKey'][0],
+ #'uid': ldapentry[1]['uid'][0],
+ 'uid': tmpname ,
+ 'email':tmpemail,
+ #'email': ldapentry[1]['mail'][0],
+ 'first_name': ldapentry[1]['givenName'][0],
+ 'last_name': ldapentry[1]['sn'][0],
+ #'phone': 'none',
+ 'serial': 'none',
+ 'authority': self.authname,
+ 'peer_authority': '',
+ 'pointer': -1,
+ 'hrn': hrn,
+ })
+ except KeyError, error:
+ logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s"
+ % (error))
+ return
+
+ return results
+
+ def _process_ldap_info_for_one_user(self, record, result_data):
+ """
+
+ Put the user's ldap data into shape. Only deals with one user
+ record and one user data from ldap.
+
+ :param record: user record
+ :param result_data: Raw ldap data coming from LdapSearch
+ :returns: user's data dict with 'type','pkey','uid', 'email',
+ 'first_name' 'last_name''serial''authority''peer_authority'
+ 'pointer''hrn'
+ :type record: dict
+ :type result_data: list
+ :rtype :dict
+
+ """
+ #One entry only in the ldap data because we used a filter
+ #to find one user only
+ ldapentry = result_data[0][1]
+ logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" % (ldapentry))
+ tmpname = ldapentry['uid'][0]
+
+ tmpemail = ldapentry['mail'][0]
+ if ldapentry['mail'][0] == "unknown":
+ tmpemail = None
+
+ parent_hrn = None
+ peer_authority = None
+ if 'hrn' in record:
+ hrn = record['hrn']
+ parent_hrn = get_authority(hrn)
+ if parent_hrn != self.authname:
+ peer_authority = parent_hrn
+ #In case the user was not imported from Iotlab LDAP
+ #but from another federated site, has an account in
+ #iotlab but currently using his hrn from federated site
+ #then the login is different from the one found in its hrn
+ if tmpname != hrn.split('.')[1]:
+ hrn = None
+ else:
+ hrn = None
+
+ results = {
+ 'type': 'user',
+ 'pkey': ldapentry['sshPublicKey'],
+ #'uid': ldapentry[1]['uid'][0],
+ 'uid': tmpname,
+ 'email': tmpemail,
+ #'email': ldapentry[1]['mail'][0],
+ 'first_name': ldapentry['givenName'][0],
+ 'last_name': ldapentry['sn'][0],
+ #'phone': 'none',
+ 'serial': 'none',
+ 'authority': parent_hrn,
+ 'peer_authority': peer_authority,
+ 'pointer': -1,
+ 'hrn': hrn,
+ }
+ return results
+
+ def LdapFindUser(self, record=None, is_user_enabled=None,
+ expected_fields=None):
+ """
+
+ Search a SFA user with a hrn. User should be already registered
+ in Iotlab LDAP.
+
+ :param record: sfa user's record. Should contain first_name,last_name,
+ email or mail. If no record is provided, returns all the users found
+ in LDAP.
+ :type record: dict
+ :param is_user_enabled: is the user's iotlab account already valid.
+ :type is_user_enabled: Boolean.
+ :returns: LDAP entries from ldap matching the filter provided. Returns
+ a single entry if one filter has been given and a list of
+ entries otherwise.
+ :rtype: dict or list
+
+ """
+ custom_record = {}
+ if is_user_enabled:
+ custom_record['enabled'] = is_user_enabled
+ if record:
+ custom_record.update(record)
+
+ req_ldap = self.make_ldap_filters_from_record(custom_record)
+ return_fields_list = []
+ if expected_fields is None:
+ return_fields_list = ['mail', 'givenName', 'sn', 'uid',
+ 'sshPublicKey']
+ else:
+ return_fields_list = expected_fields
+
+ result_data = self.LdapSearch(req_ldap, return_fields_list)
+ logger.debug("LDAP.PY \t LdapFindUser result_data %s" % (result_data))
+
+ if len(result_data) == 0:
+ return None
+ #Asked for a specific user
+ if record is not None:
+ results = self._process_ldap_info_for_one_user(record, result_data)
+
+ else:
+ #Asked for all users in ldap
+ results = self._process_ldap_info_for_all_users(result_data)
+ return results
\ No newline at end of file
--- /dev/null
+"""
+File providing methods to generate valid RSpecs for the Iotlab testbed.
+Contains methods to get information on slice, slivers, nodes and leases,
+formatting them and turn it into a RSpec.
+"""
+from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority
+
+from sfa.rspecs.rspec import RSpec
+#from sfa.rspecs.elements.location import Location
+from sfa.rspecs.elements.hardware_type import HardwareType
+from sfa.rspecs.elements.login import Login
+from sfa.rspecs.elements.services import Services
+from sfa.rspecs.elements.sliver import Sliver
+from sfa.rspecs.elements.lease import Lease
+from sfa.rspecs.elements.granularity import Granularity
+from sfa.rspecs.version_manager import VersionManager
+
+from sfa.rspecs.elements.versions.cortexlabv1Node import IotlabPosition, \
+ IotlabNode, IotlabLocation, IotlabMobility
+
+from sfa.util.sfalogging import logger
+from sfa.util.xrn import Xrn
+
+
+def cortexlab_xrn_to_hostname(xrn):
+ """Returns a node's hostname from its xrn.
+ :param xrn: The nodes xrn identifier.
+ :type xrn: Xrn (from sfa.util.xrn)
+
+ :returns: node's hostname.
+ :rtype: string
+
+ """
+ return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf())
+
+
+def cortexlab_xrn_object(root_auth, hostname):
+ """Creates a valid xrn object from the node's hostname and the authority
+ of the SFA server.
+
+ :param hostname: the node's hostname.
+ :param root_auth: the SFA root authority.
+ :type hostname: string
+ :type root_auth: string
+
+ :returns: the cortexlab node's xrn
+ :rtype: Xrn
+
+ """
+ return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node')
+
+
+class CortexlabAggregate:
+ """Aggregate manager class for Iotlab. """
+
+ sites = {}
+ nodes = {}
+ api = None
+ interfaces = {}
+ links = {}
+ node_tags = {}
+
+ prepared = False
+
+ user_options = {}
+
+ def __init__(self, driver):
+ self.driver = driver
+
+ def get_slice_and_slivers(self, slice_xrn, login=None):
+ """
+ Get the slices and the associated leases if any, from the cortexlab
+ testbed. One slice can have mutliple leases.
+ For each slice, get the nodes in the associated lease
+ and create a sliver with the necessary info and insert it into the
+ sliver dictionary, keyed on the node hostnames.
+ Returns a dict of slivers based on the sliver's node_id.
+ Called by get_rspec.
+
+
+ :param slice_xrn: xrn of the slice
+ :param login: user's login on cortexlab ldap
+
+ :type slice_xrn: string
+ :type login: string
+ :returns: a list of slices dict and a list of Sliver object
+ :rtype: (list, list)
+
+ .. note: There is no real slivers in cortexlab, only leases. The goal
+ is to be consistent with the SFA standard.
+
+ """
+ slivers = {}
+ sfa_slice = None
+ if slice_xrn is None:
+ return (sfa_slice, slivers)
+ slice_urn = hrn_to_urn(slice_xrn, 'slice')
+ slice_hrn, _ = urn_to_hrn(slice_xrn)
+ slice_name = slice_hrn
+
+ slices = self.driver.cortexlab_api.GetSlices(slice_filter=str(slice_name),
+ slice_filter_type='slice_hrn',
+ login=login)
+
+ logger.debug("CortexlabAggregate api \tget_slice_and_slivers \
+ slice_hrn %s \r\n slices %s self.driver.hrn %s"
+ % (slice_hrn, slices, self.driver.hrn))
+
+ if slices == []:
+ return (sfa_slice, slivers)
+
+ # sort slivers by node id , if there is a job
+ #and therefore, node allocated to this slice
+ for sfa_slice in slices:
+ try:
+ node_ids_list = sfa_slice['node_ids']
+ except KeyError:
+ logger.log_exc("CortexlabAggregate \t \
+ get_slice_and_slivers No nodes in the slice \
+ - KeyError ")
+ node_ids_list = []
+ continue
+
+ for node in node_ids_list:
+ sliver_xrn = Xrn(slice_urn, type='sliver', id=node)
+ sliver_xrn.set_authority(self.driver.hrn)
+ sliver = Sliver({'sliver_id': sliver_xrn.urn,
+ 'name': sfa_slice['hrn'],
+ 'type': 'cortexlab-node',
+ 'tags': []})
+
+ slivers[node] = sliver
+
+ #Add default sliver attribute :
+ #connection information for cortexlab
+ if get_authority(sfa_slice['hrn']) == self.driver.cortexlab_api.root_auth:
+ tmp = sfa_slice['hrn'].split('.')
+ ldap_username = tmp[1].split('_')[0]
+ ssh_access = None
+ slivers['default_sliver'] = {'ssh': ssh_access,
+ 'login': ldap_username}
+
+ #TODO get_slice_and_slivers Find the login of the external user
+
+ logger.debug("CortexlabAggregate api get_slice_and_slivers slivers %s "
+ % (slivers))
+ return (slices, slivers)
+
+
+ def get_nodes(self, slices=None, slivers=[], options=None):
+ """Returns the nodes in the slice using the rspec format, with all the
+ nodes' properties.
+
+ Fetch the nodes ids in the slices dictionary and get all the nodes
+ properties from OAR. Makes a rspec dicitonary out of this and returns
+ it. If the slice does not have any job running or scheduled, that is
+ it has no reserved nodes, then returns an empty list.
+
+ :param slices: list of slices (record dictionaries)
+ :param slivers: the list of slivers in all the slices
+ :type slices: list of dicts
+ :type slivers: list of Sliver object (dictionaries)
+ :returns: An empty list if the slice has no reserved nodes, a rspec
+ list with all the nodes and their properties (a dict per node)
+ otherwise.
+ :rtype: list
+
+ .. seealso:: get_slice_and_slivers
+
+ """
+ # NT: the semantic of this function is not clear to me :
+ # if slice is not defined, then all the nodes should be returned
+ # if slice is defined, we should return only the nodes that
+ # are part of this slice
+ # but what is the role of the slivers parameter ?
+ # So i assume that slice['node_ids'] will be the same as slivers for us
+ slice_nodes_list = []
+ if slices is not None:
+ for one_slice in slices:
+ try:
+ slice_nodes_list = one_slice['node_ids']
+ # if we are dealing with a slice that has no node just
+ # return an empty list. In cortexlab a slice can have multiple
+ # jobs scheduled, so it either has at least one lease or
+ # not at all.
+ except KeyError:
+ return []
+
+ # get the granularity in second for the reservation system
+ grain = self.driver.cortexlab_api.GetLeaseGranularity()
+
+ nodes = self.driver.cortexlab_api.GetNodes()
+
+ nodes_dict = {}
+
+ #if slices, this means we got to list all the nodes given to this slice
+ # Make a list of all the nodes in the slice before getting their
+ #attributes
+ rspec_nodes = []
+
+ logger.debug("CortexlabAggregate api get_nodes slices %s "
+ % (slices))
+
+ reserved_nodes = self.driver.cortexlab_api.GetNodesCurrentlyInUse()
+ logger.debug("CortexlabAggregate api get_nodes slice_nodes_list %s "
+ % (slice_nodes_list))
+ for node in nodes:
+ nodes_dict[node['node_id']] = node
+ if slice_nodes_list == [] or node['hostname'] in slice_nodes_list:
+
+ rspec_node = IotlabNode()
+ # xxx how to retrieve site['login_base']
+ #site_id=node['site_id']
+ #site=sites_dict[site_id]
+
+ # rspec_node['mobile'] = node['mobile']
+ rspec_node['archi'] = node['archi']
+ rspec_node['radio'] = node['radio']
+
+ cortexlab_xrn = cortexlab_xrn_object(self.driver.cortexlab_api.root_auth,
+ node['hostname'])
+ rspec_node['component_id'] = cortexlab_xrn.urn
+ rspec_node['component_name'] = node['hostname']
+ rspec_node['component_manager_id'] = \
+ hrn_to_urn(self.driver.cortexlab_api.root_auth,
+ 'authority+sa')
+
+ # Iotlab's nodes are federated : there is only one authority
+ # for all Iotlab sites, registered in SFA.
+ # Removing the part including the site
+ # in authority_id SA 27/07/12
+ rspec_node['authority_id'] = rspec_node['component_manager_id']
+
+ # do not include boot state (<available> element)
+ #in the manifest rspec
+
+ rspec_node['boot_state'] = node['boot_state']
+ if node['hostname'] in reserved_nodes:
+ rspec_node['boot_state'] = "Reserved"
+ rspec_node['exclusive'] = 'true'
+ rspec_node['hardware_types'] = [HardwareType({'name':
+ 'cortexlab-node'})]
+
+
+ location = IotlabLocation({'country':'France', 'site':
+ node['site']})
+ rspec_node['location'] = location
+
+ # Adding mobility of the node in the rspec
+ mobility = IotlabMobility()
+ for field in mobility:
+ try:
+ mobility[field] = node[field]
+ except KeyError, error:
+ logger.log_exc("CortexlabAggregate\t get_nodes \
+ mobility %s " % (error))
+ rspec_node['mobility'] = mobility
+
+ position = IotlabPosition()
+ for field in position:
+ try:
+ position[field] = node[field]
+ except KeyError, error:
+ logger.log_exc("CortexlabAggregate\t get_nodes \
+ position %s " % (error))
+
+ rspec_node['position'] = position
+ #rspec_node['interfaces'] = []
+
+ # Granularity
+ granularity = Granularity({'grain': grain})
+ rspec_node['granularity'] = granularity
+ rspec_node['tags'] = []
+ if node['hostname'] in slivers:
+ # add sliver info
+ sliver = slivers[node['hostname']]
+ rspec_node['sliver_id'] = sliver['sliver_id']
+ rspec_node['client_id'] = node['hostname']
+ rspec_node['slivers'] = [sliver]
+
+ # slivers always provide the ssh service
+ login = Login({'authentication': 'ssh-keys',
+ 'hostname': node['hostname'], 'port': '22',
+ 'username': sliver['name']})
+ service = Services({'login': login})
+ rspec_node['services'] = [service]
+ rspec_nodes.append(rspec_node)
+
+ return (rspec_nodes)
+
+ def get_all_leases(self, ldap_username):
+ """
+
+ Get list of lease dictionaries which all have the mandatory keys
+ ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration').
+ All the leases running or scheduled are returned.
+
+ :param ldap_username: if ldap uid is not None, looks for the leases
+ belonging to this user.
+ :type ldap_username: string
+ :returns: rspec lease dictionary with keys lease_id, component_id,
+ slice_id, start_time, duration.
+ :rtype: dict
+
+ .. note::There is no filtering of leases within a given time frame.
+ All the running or scheduled leases are returned. options
+ removed SA 15/05/2013
+
+
+ """
+
+ #now = int(time.time())
+ #lease_filter = {'clip': now }
+
+ #if slice_record:
+ #lease_filter.update({'name': slice_record['name']})
+
+ #leases = self.driver.cortexlab_api.GetLeases(lease_filter)
+
+ logger.debug("CortexlabAggregate get_all_leases ldap_username %s "
+ % (ldap_username))
+ leases = self.driver.cortexlab_api.GetLeases(login=ldap_username)
+ grain = self.driver.cortexlab_api.GetLeaseGranularity()
+ # site_ids = []
+ rspec_leases = []
+ for lease in leases:
+ #as many leases as there are nodes in the job
+ for node in lease['reserved_nodes']:
+ rspec_lease = Lease()
+ rspec_lease['lease_id'] = lease['lease_id']
+ #site = node['site_id']
+ cortexlab_xrn = cortexlab_xrn_object(self.driver.cortexlab_api.root_auth,
+ node)
+ rspec_lease['component_id'] = cortexlab_xrn.urn
+ #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\
+ #site, node['hostname'])
+ try:
+ rspec_lease['slice_id'] = lease['slice_id']
+ except KeyError:
+ #No info on the slice used in cortexlab_xp table
+ pass
+ rspec_lease['start_time'] = lease['t_from']
+ rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \
+ / grain
+ rspec_leases.append(rspec_lease)
+ return rspec_leases
+
+ def get_rspec(self, slice_xrn=None, login=None, version=None,
+ options=None):
+ """
+ Returns xml rspec:
+ - a full advertisement rspec with the testbed resources if slice_xrn is
+ not specified.If a lease option is given, also returns the leases
+ scheduled on the testbed.
+ - a manifest Rspec with the leases and nodes in slice's leases if
+ slice_xrn is not None.
+
+ :param slice_xrn: srn of the slice
+ :type slice_xrn: string
+ :param login: user'uid (ldap login) on cortexlab
+ :type login: string
+ :param version: can be set to sfa or cortexlab
+ :type version: RSpecVersion
+ :param options: used to specify if the leases should also be included in
+ the returned rspec.
+ :type options: dict
+
+ :returns: Xml Rspec.
+ :rtype: XML
+
+
+ """
+
+ ldap_username = None
+ rspec = None
+ version_manager = VersionManager()
+ version = version_manager.get_version(version)
+ logger.debug("CortexlabAggregate \t get_rspec ***version %s \
+ version.type %s version.version %s options %s \r\n"
+ % (version, version.type, version.version, options))
+
+ if slice_xrn is None:
+ rspec_version = version_manager._get_version(version.type,
+ version.version, 'ad')
+
+ else:
+ rspec_version = version_manager._get_version(
+ version.type, version.version, 'manifest')
+
+ slices, slivers = self.get_slice_and_slivers(slice_xrn, login)
+ if slice_xrn and slices is not None:
+ #Get user associated with this slice
+ #for one_slice in slices :
+ ldap_username = slices[0]['reg_researchers'][0].__dict__['hrn']
+ # ldap_username = slices[0]['user']
+ tmp = ldap_username.split('.')
+ ldap_username = tmp[1]
+ logger.debug("CortexlabAggregate \tget_rspec **** \
+ LDAP USERNAME %s \r\n" \
+ % (ldap_username))
+ #at this point sliver may be empty if no cortexlab job
+ #is running for this user/slice.
+ rspec = RSpec(version=rspec_version, user_options=options)
+
+ logger.debug("\r\n \r\n CortexlabAggregate \tget_rspec *** \
+ slice_xrn %s slices %s\r\n \r\n"
+ % (slice_xrn, slices))
+
+ if options is not None and 'list_leases' in options:
+ lease_option = options['list_leases']
+ else:
+ #If no options are specified, at least print the resources
+ lease_option = 'all'
+ #if slice_xrn :
+ #lease_option = 'all'
+
+ if lease_option in ['all', 'resources']:
+ #if not options.get('list_leases') or options.get('list_leases')
+ #and options['list_leases'] != 'leases':
+ nodes = self.get_nodes(slices, slivers)
+ if slice_xrn and slices is None:
+ nodes = []
+ logger.debug("\r\n")
+ logger.debug("CortexlabAggregate \t lease_option %s \
+ get rspec ******* nodes %s"
+ % (lease_option, nodes))
+
+ sites_set = set([node['location']['site'] for node in nodes])
+
+ #In case creating a job, slice_xrn is not set to None
+ rspec.version.add_nodes(nodes)
+ if slice_xrn and slices is not None:
+ # #Get user associated with this slice
+ # #for one_slice in slices :
+ # ldap_username = slices[0]['reg_researchers']
+ # # ldap_username = slices[0]['user']
+ # tmp = ldap_username.split('.')
+ # ldap_username = tmp[1]
+ # # ldap_username = tmp[1].split('_')[0]
+
+ logger.debug("CortexlabAggregate \tget_rspec **** \
+ version type %s ldap_ user %s \r\n" \
+ % (version.type, ldap_username))
+ if version.type == "Iotlab":
+ rspec.version.add_connection_information(
+ ldap_username, sites_set)
+
+ default_sliver = slivers.get('default_sliver', [])
+ if default_sliver and len(nodes) is not 0:
+ #default_sliver_attribs = default_sliver.get('tags', [])
+ logger.debug("CortexlabAggregate \tget_rspec **** \
+ default_sliver%s \r\n" % (default_sliver))
+ for attrib in default_sliver:
+ rspec.version.add_default_sliver_attribute(
+ attrib, default_sliver[attrib])
+
+ if lease_option in ['all','leases']:
+ leases = self.get_all_leases(ldap_username)
+ rspec.version.add_leases(leases)
+ logger.debug("CortexlabAggregate \tget_rspec **** \
+ FINAL RSPEC %s \r\n" % (rspec.toxml()))
+ return rspec.toxml()
--- /dev/null
+"""
+File containing the IotlabTestbedAPI, used to interact with nodes, users,
+slices, leases and keys, as well as the dedicated iotlab database and table,
+holding information about which slice is running which job.
+
+"""
+from datetime import datetime
+
+from sfa.util.sfalogging import logger
+
+from sfa.storage.alchemy import dbsession
+from sqlalchemy.orm import joinedload
+from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
+from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP
+from sfa.iotlab.OARrestapi import OARrestapi
+from sfa.iotlab.LDAPapi import LDAPapi
+
+from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
+
+from sfa.trust.certificate import Keypair, convert_public_key
+from sfa.trust.gid import create_uuid
+from sfa.trust.hierarchy import Hierarchy
+
+from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
+
+class CortexlabTestbedAPI():
+ """ Class enabled to use LDAP and OAR api calls. """
+
+ _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins
+
+ def __init__(self, config):
+ """Creates an instance of OARrestapi and LDAPapi which will be used to
+ issue calls to OAR or LDAP methods.
+ Set the time format and the testbed granularity used for OAR
+ reservation and leases.
+
+ :param config: configuration object from sfa.util.config
+ :type config: Config object
+ """
+ self.iotlab_db = IotlabDB(config)
+ self.oar = OARrestapi()
+ self.ldap = LDAPapi()
+ self.time_format = "%Y-%m-%d %H:%M:%S"
+ self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
+ self.grain = 60 # 10 mins lease minimum, 60 sec granularity
+ #import logging, logging.handlers
+ #from sfa.util.sfalogging import _SfaLogger
+ #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \
+ #level=logging.DEBUG)
+ return
+
+ @staticmethod
+ def GetMinExperimentDurationInGranularity():
+ """ Returns the minimum allowed duration for an experiment on the
+ testbed. In seconds.
+
+ """
+ return IotlabTestbedAPI._MINIMUM_DURATION
+
+ @staticmethod
+ def GetPeers(peer_filter=None ):
+ """ Gathers registered authorities in SFA DB and looks for specific peer
+ if peer_filter is specified.
+ :param peer_filter: name of the site authority looked for.
+ :type peer_filter: string
+ :returns: list of records.
+
+ """
+
+ existing_records = {}
+ existing_hrns_by_types = {}
+ logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter))
+ all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
+
+ for record in all_records:
+ existing_records[(record.hrn, record.type)] = record
+ if record.type not in existing_hrns_by_types:
+ existing_hrns_by_types[record.type] = [record.hrn]
+ else:
+ existing_hrns_by_types[record.type].append(record.hrn)
+
+ logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s "
+ % (existing_hrns_by_types))
+ records_list = []
+
+ try:
+ if peer_filter:
+ records_list.append(existing_records[(peer_filter,
+ 'authority')])
+ else:
+ for hrn in existing_hrns_by_types['authority']:
+ records_list.append(existing_records[(hrn, 'authority')])
+
+ logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s "
+ % (records_list))
+
+ except KeyError:
+ pass
+
+ return_records = records_list
+ logger.debug("CORTEXLAB_API \tGetPeer return_records %s "
+ % (return_records))
+ return return_records
+
+ #TODO : Handling OR request in make_ldap_filters_from_records
+ #instead of the for loop
+ #over the records' list
+ def GetPersons(self, person_filter=None):
+ """
+ Get the enabled users and their properties from Iotlab LDAP.
+ If a filter is specified, looks for the user whose properties match
+ the filter, otherwise returns the whole enabled users'list.
+
+ :param person_filter: Must be a list of dictionnaries with users
+ properties when not set to None.
+ :type person_filter: list of dict
+
+ :returns: Returns a list of users whose accounts are enabled
+ found in ldap.
+ :rtype: list of dicts
+
+ """
+ logger.debug("CORTEXLAB_API \tGetPersons person_filter %s"
+ % (person_filter))
+ person_list = []
+ if person_filter and isinstance(person_filter, list):
+ #If we are looking for a list of users (list of dict records)
+ #Usually the list contains only one user record
+ for searched_attributes in person_filter:
+
+ #Get only enabled user accounts in iotlab LDAP :
+ #add a filter for make_ldap_filters_from_record
+ person = self.ldap.LdapFindUser(searched_attributes,
+ is_user_enabled=True)
+ #If a person was found, append it to the list
+ if person:
+ person_list.append(person)
+
+ #If the list is empty, return None
+ if len(person_list) is 0:
+ person_list = None
+
+ else:
+ #Get only enabled user accounts in iotlab LDAP :
+ #add a filter for make_ldap_filters_from_record
+ person_list = self.ldap.LdapFindUser(is_user_enabled=True)
+
+ return person_list
+
+
+ #def GetTimezone(self):
+ #""" Returns the OAR server time and timezone.
+ #Unused SA 30/05/13"""
+ #server_timestamp, server_tz = self.oar.parser.\
+ #SendRequest("GET_timezone")
+ #return server_timestamp, server_tz
+
+ def DeleteLeases(self, lease_id, username):
+ """
+
+ Deletes the lease with the specified lease_id and username on OAR by
+ posting a delete request to OAR.
+
+ :param lease_id: Reservation identifier.
+ :param username: user's iotlab login in LDAP.
+ :type lease_id: Depends on what tou are using, could be integer or
+ string
+ :type username: string
+
+ :returns: dictionary with the lease id and if delete has been successful
+ (True) or no (False)
+ :rtype: dict
+
+ """
+
+ # Here delete the lease specified
+
+ # If the username is not necessary to delete the lease, then you can
+ # remove it from the parameters, given that you propagate the changes
+
+
+ # Return delete status so that you know if the delete has been
+ # successuf or not
+ if answer['status'] == 'Delete request registered':
+ ret = {lease_id: True}
+ else:
+ ret = {lease_id: False}
+ logger.debug("CORTEXLAB_API \DeleteLeases lease_id %s \r\n answer %s \
+ username %s" % (lease_id, answer, username))
+ return ret
+
+
+
+
+
+
+ def GetJobsResources(self, job_id, username = None):
+ """ Gets the list of nodes associated with the job_id and username
+ if provided.
+ Transforms the iotlab hostnames to the corresponding
+ SFA nodes hrns.
+ Rertuns dict key :'node_ids' , value : hostnames list
+ :param username: user's LDAP login
+ :paran job_id: job's OAR identifier.
+ :type username: string
+ :type job_id: integer
+
+ :returns: dicionary with nodes' hostnames belonging to the job.
+ :rtype: dict
+ """
+
+ req = "GET_jobs_id_resources"
+
+
+ #Get job resources list from OAR
+ node_id_list = self.oar.parser.SendRequest(req, job_id, username)
+ logger.debug("CORTEXLAB_API \t GetJobsResources %s " %(node_id_list))
+
+ hostname_list = \
+ self.__get_hostnames_from_oar_node_ids(node_id_list)
+
+
+ #Replaces the previous entry "assigned_network_address" /
+ #"reserved_resources" with "node_ids"
+ job_info = {'node_ids': hostname_list}
+
+ return job_info
+
+
+
+ def GetNodesCurrentlyInUse(self):
+ """Returns a list of all the nodes already involved in an oar running
+ job.
+ :rtype: list of nodes hostnames.
+ """
+ return self.oar.parser.SendRequest("GET_running_jobs")
+
+ def __get_hostnames_from_oar_node_ids(self, resource_id_list ):
+ """Get the hostnames of the nodes from their OAR identifiers.
+ Get the list of nodes dict using GetNodes and find the hostname
+ associated with the identifier.
+ :param resource_id_list: list of nodes identifiers
+ :returns: list of node hostnames.
+ """
+ full_nodes_dict_list = self.GetNodes()
+ #Put the full node list into a dictionary keyed by oar node id
+ oar_id_node_dict = {}
+ for node in full_nodes_dict_list:
+ oar_id_node_dict[node['oar_id']] = node
+
+ hostname_list = []
+ for resource_id in resource_id_list:
+ #Because jobs requested "asap" do not have defined resources
+ if resource_id is not "Undefined":
+ hostname_list.append(\
+ oar_id_node_dict[resource_id]['hostname'])
+
+ #hostname_list.append(oar_id_node_dict[resource_id]['hostname'])
+ return hostname_list
+
+ def GetReservedNodes(self, username=None):
+ """ Get list of leases. Get the leases for the username if specified,
+ otherwise get all the leases. Finds the nodes hostnames for each
+ OAR node identifier.
+ :param username: user's LDAP login
+ :type username: string
+ :returns: list of reservations dict
+ :rtype: dict list
+ """
+
+ #Get the nodes in use and the reserved nodes
+ reservation_dict_list = \
+ self.oar.parser.SendRequest("GET_reserved_nodes", \
+ username = username)
+
+
+ for resa in reservation_dict_list:
+ logger.debug ("GetReservedNodes resa %s"%(resa))
+ #dict list of hostnames and their site
+ resa['reserved_nodes'] = \
+ self.__get_hostnames_from_oar_node_ids(resa['resource_ids'])
+
+ #del resa['resource_ids']
+ return reservation_dict_list
+
+ def GetNodes(self, node_filter_dict=None, return_fields_list=None):
+ """
+
+ Make a list of iotlab nodes and their properties from information
+ given by OAR. Search for specific nodes if some filters are
+ specified. Nodes properties returned if no return_fields_list given:
+ 'hrn','archi','mobile','hostname','site','boot_state','node_id',
+ 'radio','posx','posy','oar_id','posz'.
+
+ :param node_filter_dict: dictionnary of lists with node properties. For
+ instance, if you want to look for a specific node with its hrn,
+ the node_filter_dict should be {'hrn': [hrn_of_the_node]}
+ :type node_filter_dict: dict
+ :param return_fields_list: list of specific fields the user wants to be
+ returned.
+ :type return_fields_list: list
+ :returns: list of dictionaries with node properties
+ :rtype: list
+
+ """
+ node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
+ node_dict_list = node_dict_by_id.values()
+ logger.debug (" CORTEXLAB_API GetNodes node_filter_dict %s \
+ return_fields_list %s " % (node_filter_dict, return_fields_list))
+ #No filtering needed return the list directly
+ if not (node_filter_dict or return_fields_list):
+ return node_dict_list
+
+ return_node_list = []
+ if node_filter_dict:
+ for filter_key in node_filter_dict:
+ try:
+ #Filter the node_dict_list by each value contained in the
+ #list node_filter_dict[filter_key]
+ for value in node_filter_dict[filter_key]:
+ for node in node_dict_list:
+ if node[filter_key] == value:
+ if return_fields_list:
+ tmp = {}
+ for k in return_fields_list:
+ tmp[k] = node[k]
+ return_node_list.append(tmp)
+ else:
+ return_node_list.append(node)
+ except KeyError:
+ logger.log_exc("GetNodes KeyError")
+ return
+
+
+ return return_node_list
+
+
+
+ @staticmethod
+ def AddSlice(slice_record, user_record):
+ """
+
+ Add slice to the local iotlab sfa tables if the slice comes
+ from a federated site and is not yet in the iotlab sfa DB,
+ although the user has already a LDAP login.
+ Called by verify_slice during lease/sliver creation.
+
+ :param slice_record: record of slice, must contain hrn, gid, slice_id
+ and authority of the slice.
+ :type slice_record: dictionary
+ :param user_record: record of the user
+ :type user_record: RegUser
+
+ """
+
+ sfa_record = RegSlice(hrn=slice_record['hrn'],
+ gid=slice_record['gid'],
+ pointer=slice_record['slice_id'],
+ authority=slice_record['authority'])
+ logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s"
+ % (sfa_record, user_record))
+ sfa_record.just_created()
+ dbsession.add(sfa_record)
+ dbsession.commit()
+ #Update the reg-researcher dependance table
+ sfa_record.reg_researchers = [user_record]
+ dbsession.commit()
+
+ return
+
+
+ def GetSites(self, site_filter_name_list=None, return_fields_list=None):
+ """Returns the list of Iotlab's sites with the associated nodes and
+ their properties as dictionaries.
+
+ Uses the OAR request GET_sites to find the Iotlab's sites.
+
+ :param site_filter_name_list: used to specify specific sites
+ :param return_fields_list: field that has to be returned
+ :type site_filter_name_list: list
+ :type return_fields_list: list
+
+ .. warning:: unused
+ """
+ site_dict = self.oar.parser.SendRequest("GET_sites")
+ #site_dict : dict where the key is the sit ename
+ return_site_list = []
+ if not (site_filter_name_list or return_fields_list):
+ return_site_list = site_dict.values()
+ return return_site_list
+
+ for site_filter_name in site_filter_name_list:
+ if site_filter_name in site_dict:
+ if return_fields_list:
+ for field in return_fields_list:
+ tmp = {}
+ try:
+ tmp[field] = site_dict[site_filter_name][field]
+ except KeyError:
+ logger.error("GetSites KeyError %s " % (field))
+ return None
+ return_site_list.append(tmp)
+ else:
+ return_site_list.append(site_dict[site_filter_name])
+
+ return return_site_list
+
+
+ #TODO : Check rights to delete person
+ def DeletePerson(self, person_record):
+ """Disable an existing account in iotlab LDAP.
+
+ Users and techs can only delete themselves. PIs can only
+ delete themselves and other non-PIs at their sites.
+ ins can delete anyone.
+
+ :param person_record: user's record
+ :type person_record: dict
+ :returns: True if successful, False otherwise.
+ :rtype: boolean
+
+ .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF.
+ """
+ #Disable user account in iotlab LDAP
+ ret = self.ldap.LdapMarkUserAsDeleted(person_record)
+ logger.warning("CORTEXLAB_API DeletePerson %s " % (person_record))
+ return ret['bool']
+
+ def DeleteSlice(self, slice_record):
+ """Deletes the specified slice and kills the jobs associated with
+ the slice if any, using DeleteSliceFromNodes.
+
+ :param slice_record: record of the slice, must contain oar_job_id, user
+ :type slice_record: dict
+ :returns: True if all the jobs in the slice have been deleted,
+ or the list of jobs that could not be deleted otherwise.
+ :rtype: list or boolean
+
+ .. seealso:: DeleteSliceFromNodes
+
+ """
+ ret = self.DeleteSliceFromNodes(slice_record)
+ delete_failed = None
+ for job_id in ret:
+ if False in ret[job_id]:
+ if delete_failed is None:
+ delete_failed = []
+ delete_failed.append(job_id)
+
+ logger.info("CORTEXLAB_API DeleteSlice %s answer %s"%(slice_record, \
+ delete_failed))
+ return delete_failed or True
+
+ @staticmethod
+ def __add_person_to_db(user_dict):
+ """
+ Add a federated user straight to db when the user issues a lease
+ request with iotlab nodes and that he has not registered with iotlab
+ yet (that is he does not have a LDAP entry yet).
+ Uses parts of the routines in SlabImport when importing user from LDAP.
+ Called by AddPerson, right after LdapAddUser.
+ :param user_dict: Must contain email, hrn and pkey to get a GID
+ and be added to the SFA db.
+ :type user_dict: dict
+
+ """
+ check_if_exists = \
+ dbsession.query(RegUser).filter_by(email = user_dict['email']).first()
+ #user doesn't exists
+ if not check_if_exists:
+ logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
+ " %(user_dict))
+ hrn = user_dict['hrn']
+ person_urn = hrn_to_urn(hrn, 'user')
+ pubkey = user_dict['pkey']
+ try:
+ pkey = convert_public_key(pubkey)
+ except TypeError:
+ #key not good. create another pkey
+ logger.warn('__add_person_to_db: unable to convert public \
+ key for %s' %(hrn ))
+ pkey = Keypair(create=True)
+
+
+ if pubkey is not None and pkey is not None :
+ hierarchy = Hierarchy()
+ person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
+ pkey)
+ if user_dict['email']:
+ logger.debug("__add_person_to_db \r\n \r\n \
+ IOTLAB IMPORTER PERSON EMAIL OK email %s "\
+ %(user_dict['email']))
+ person_gid.set_email(user_dict['email'])
+
+ user_record = RegUser(hrn=hrn , pointer= '-1', \
+ authority=get_authority(hrn), \
+ email=user_dict['email'], gid = person_gid)
+ user_record.reg_keys = [RegKey(user_dict['pkey'])]
+ user_record.just_created()
+ dbsession.add (user_record)
+ dbsession.commit()
+ return
+
+
+ def AddPerson(self, record):
+ """
+
+ Adds a new account. Any fields specified in records are used,
+ otherwise defaults are used. Creates an appropriate login by calling
+ LdapAddUser.
+
+ :param record: dictionary with the sfa user's properties.
+ :returns: a dicitonary with the status. If successful, the dictionary
+ boolean is set to True and there is a 'uid' key with the new login
+ added to LDAP, otherwise the bool is set to False and a key
+ 'message' is in the dictionary, with the error message.
+ :rtype: dict
+
+ """
+ ret = self.ldap.LdapAddUser(record)
+
+ if ret['bool'] is True:
+ record['hrn'] = self.root_auth + '.' + ret['uid']
+ logger.debug("CORTEXLAB_API AddPerson return code %s record %s "
+ % (ret, record))
+ self.__add_person_to_db(record)
+ return ret
+
+
+
+
+
+ #TODO AddPersonKey 04/07/2012 SA
+ def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
+ """Adds a new key to the specified account. Adds the key to the
+ iotlab ldap, provided that the person_uid is valid.
+
+ Non-admins can only modify their own keys.
+
+ :param person_uid: user's iotlab login in LDAP
+ :param old_attributes_dict: dict with the user's old sshPublicKey
+ :param new_key_dict: dict with the user's new sshPublicKey
+ :type person_uid: string
+
+
+ :rtype: Boolean
+ :returns: True if the key has been modified, False otherwise.
+
+ """
+ ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \
+ new_key_dict)
+ logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ")
+ return ret['bool']
+
+ def DeleteLeases(self, leases_id_list, slice_hrn):
+ """
+
+ Deletes several leases, based on their job ids and the slice
+ they are associated with. Uses DeleteJobs to delete the jobs
+ on OAR. Note that one slice can contain multiple jobs, and in this
+ case all the jobs in the leases_id_list MUST belong to ONE slice,
+ since there is only one slice hrn provided here.
+
+ :param leases_id_list: list of job ids that belong to the slice whose
+ slice hrn is provided.
+ :param slice_hrn: the slice hrn.
+ :type slice_hrn: string
+
+ .. warning:: Does not have a return value since there was no easy
+ way to handle failure when dealing with multiple job delete. Plus,
+ there was no easy way to report it to the user.
+
+ """
+ logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
+ \r\n " %(leases_id_list, slice_hrn))
+ for job_id in leases_id_list:
+ self.DeleteJobs(job_id, slice_hrn)
+
+ return
+
+ @staticmethod
+ def _process_walltime(duration):
+ """ Calculates the walltime in seconds from the duration in H:M:S
+ specified in the RSpec.
+
+ """
+ if duration:
+ # Fixing the walltime by adding a few delays.
+ # First put the walltime in seconds oarAdditionalDelay = 20;
+ # additional delay for /bin/sleep command to
+ # take in account prologue and epilogue scripts execution
+ # int walltimeAdditionalDelay = 240; additional delay
+ #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
+ #in oar.conf
+ # Put the duration in seconds first
+ #desired_walltime = duration * 60
+ desired_walltime = duration
+ total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
+ sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
+ walltime = []
+ #Put the walltime back in str form
+ #First get the hours
+ walltime.append(str(total_walltime / 3600))
+ total_walltime = total_walltime - 3600 * int(walltime[0])
+ #Get the remaining minutes
+ walltime.append(str(total_walltime / 60))
+ total_walltime = total_walltime - 60 * int(walltime[1])
+ #Get the seconds
+ walltime.append(str(total_walltime))
+
+ else:
+ logger.log_exc(" __process_walltime duration null")
+
+ return walltime, sleep_walltime
+
+ @staticmethod
+ def _create_job_structure_request_for_OAR(lease_dict):
+ """ Creates the structure needed for a correct POST on OAR.
+ Makes the timestamp transformation into the appropriate format.
+ Sends the POST request to create the job with the resources in
+ added_nodes.
+
+ """
+
+ nodeid_list = []
+ reqdict = {}
+
+
+ reqdict['workdir'] = '/tmp'
+ reqdict['resource'] = "{network_address in ("
+
+ for node in lease_dict['added_nodes']:
+ logger.debug("\r\n \r\n OARrestapi \t \
+ __create_job_structure_request_for_OAR node %s" %(node))
+
+ # Get the ID of the node
+ nodeid = node
+ reqdict['resource'] += "'" + nodeid + "', "
+ nodeid_list.append(nodeid)
+
+ custom_length = len(reqdict['resource'])- 2
+ reqdict['resource'] = reqdict['resource'][0:custom_length] + \
+ ")}/nodes=" + str(len(nodeid_list))
+
+
+ walltime, sleep_walltime = \
+ IotlabTestbedAPI._process_walltime(\
+ int(lease_dict['lease_duration']))
+
+
+ reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
+ ":" + str(walltime[1]) + ":" + str(walltime[2])
+ reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
+
+ #In case of a scheduled experiment (not immediate)
+ #To run an XP immediately, don't specify date and time in RSpec
+ #They will be set to None.
+ if lease_dict['lease_start_time'] is not '0':
+ #Readable time accepted by OAR
+ start_time = datetime.fromtimestamp( \
+ int(lease_dict['lease_start_time'])).\
+ strftime(lease_dict['time_format'])
+ reqdict['reservation'] = start_time
+ #If there is not start time, Immediate XP. No need to add special
+ # OAR parameters
+
+
+ reqdict['type'] = "deploy"
+ reqdict['directory'] = ""
+ reqdict['name'] = "SFA_" + lease_dict['slice_user']
+
+ return reqdict
+
+
+ def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
+ lease_start_time, lease_duration, slice_user=None):
+
+ """
+ Create a job request structure based on the information provided
+ and post the job on OAR.
+ :param added_nodes: list of nodes that belong to the described lease.
+ :param slice_name: the slice hrn associated to the lease.
+ :param lease_start_time: timestamp of the lease startting time.
+ :param lease_duration: lease durationin minutes
+
+ """
+ lease_dict = {}
+ lease_dict['lease_start_time'] = lease_start_time
+ lease_dict['lease_duration'] = lease_duration
+ lease_dict['added_nodes'] = added_nodes
+ lease_dict['slice_name'] = slice_name
+ lease_dict['slice_user'] = slice_user
+ lease_dict['grain'] = self.GetLeaseGranularity()
+ lease_dict['time_format'] = self.time_format
+
+
+ logger.debug("CORTEXLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\
+ \r\n " %(slice_user))
+ #Create the request for OAR
+ reqdict = self._create_job_structure_request_for_OAR(lease_dict)
+ # first step : start the OAR job and update the job
+ logger.debug("CORTEXLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\
+ \r\n " %(reqdict))
+
+ answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
+ reqdict, slice_user)
+ logger.debug("CORTEXLAB_API \tLaunchExperimentOnOAR jobid %s " %(answer))
+ try:
+ jobid = answer['id']
+ except KeyError:
+ logger.log_exc("CORTEXLAB_API \tLaunchExperimentOnOAR \
+ Impossible to create job %s " %(answer))
+ return None
+
+
+
+
+ if jobid :
+ logger.debug("CORTEXLAB_API \tLaunchExperimentOnOAR jobid %s \
+ added_nodes %s slice_user %s" %(jobid, added_nodes, \
+ slice_user))
+
+
+ return jobid
+
+
+ def AddLeases(self, hostname_list, slice_record,
+ lease_start_time, lease_duration):
+
+ """Creates a job in OAR corresponding to the information provided
+ as parameters. Adds the job id and the slice hrn in the iotlab
+ database so that we are able to know which slice has which nodes.
+
+ :param hostname_list: list of nodes' OAR hostnames.
+ :param slice_record: sfa slice record, must contain login and hrn.
+ :param lease_start_time: starting time , unix timestamp format
+ :param lease_duration: duration in minutes
+
+ :type hostname_list: list
+ :type slice_record: dict
+ :type lease_start_time: integer
+ :type lease_duration: integer
+
+ """
+ logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \
+ slice_record %s lease_start_time %s lease_duration %s "\
+ %( hostname_list, slice_record , lease_start_time, \
+ lease_duration))
+
+ #tmp = slice_record['reg-researchers'][0].split(".")
+ username = slice_record['login']
+ #username = tmp[(len(tmp)-1)]
+ job_id = self.LaunchExperimentOnOAR(hostname_list, \
+ slice_record['hrn'], \
+ lease_start_time, lease_duration, \
+ username)
+ start_time = \
+ datetime.fromtimestamp(int(lease_start_time)).\
+ strftime(self.time_format)
+ end_time = lease_start_time + lease_duration
+
+
+ logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
+ %s %s %s "%(slice_record['hrn'], job_id, end_time))
+
+
+ logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \
+ %(type(slice_record['hrn']), type(job_id), type(end_time)))
+
+ iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id,
+ end_time= end_time)
+
+ logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \
+ %(iotlab_ex_row))
+ self.iotlab_db.iotlab_session.add(iotlab_ex_row)
+ self.iotlab_db.iotlab_session.commit()
+
+ logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \
+ %(start_time))
+
+ return
+
+
+ #Delete the jobs from job_iotlab table
+ def DeleteSliceFromNodes(self, slice_record):
+ """
+
+ Deletes all the running or scheduled jobs of a given slice
+ given its record.
+
+ :param slice_record: record of the slice, must contain oar_job_id, user
+ :type slice_record: dict
+
+ :returns: dict of the jobs'deletion status. Success= True, Failure=
+ False, for each job id.
+ :rtype: dict
+
+ """
+ logger.debug("CORTEXLAB_API \t DeleteSliceFromNodes %s "
+ % (slice_record))
+
+ if isinstance(slice_record['oar_job_id'], list):
+ oar_bool_answer = {}
+ for job_id in slice_record['oar_job_id']:
+ ret = self.DeleteJobs(job_id, slice_record['user'])
+
+ oar_bool_answer.update(ret)
+
+ else:
+ oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'],
+ slice_record['user'])]
+
+ return oar_bool_answer
+
+
+
+ def GetLeaseGranularity(self):
+ """ Returns the granularity of an experiment in the Iotlab testbed.
+ OAR uses seconds for experiments duration , the granulaity is also
+ defined in seconds.
+ Experiments which last less than 10 min (600 sec) are invalid"""
+ return self.grain
+
+
+ # @staticmethod
+ # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql):
+ # """ Cleans the iotlab db by deleting expired and cancelled jobs.
+ # Compares the list of job ids given by OAR with the job ids that
+ # are already in the database, deletes the jobs that are no longer in
+ # the OAR job id list.
+ # :param job_oar_list: list of job ids coming from OAR
+ # :type job_oar_list: list
+ # :param job_psql: list of job ids cfrom the database.
+ # type job_psql: list
+ # """
+ # #Turn the list into a set
+ # set_jobs_psql = set(jobs_psql)
+
+ # kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
+ # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \
+ # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
+ # deleted_jobs = set_jobs_psql.difference(kept_jobs)
+ # deleted_jobs = list(deleted_jobs)
+ # if len(deleted_jobs) > 0:
+ # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
+ # self.iotlab_db.iotlab_session.commit()
+
+ # return
+
+ @staticmethod
+ def filter_lease_name(reservation_list, filter_value):
+ filtered_reservation_list = list(reservation_list)
+ logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \
+ % (reservation_list))
+ for reservation in reservation_list:
+ if 'slice_hrn' in reservation and \
+ reservation['slice_hrn'] != filter_value:
+ filtered_reservation_list.remove(reservation)
+
+ logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \
+ % (filtered_reservation_list))
+ return filtered_reservation_list
+
+ @staticmethod
+ def filter_lease_start_time(reservation_list, filter_value):
+ filtered_reservation_list = list(reservation_list)
+
+ for reservation in reservation_list:
+ if 't_from' in reservation and \
+ reservation['t_from'] > filter_value:
+ filtered_reservation_list.remove(reservation)
+
+ return filtered_reservation_list
+
+ def GetLeases(self, lease_filter_dict=None, login=None):
+ """
+
+ Get the list of leases from OAR with complete information
+ about which slice owns which jobs and nodes.
+ Two purposes:
+ -Fetch all the jobs from OAR (running, waiting..)
+ complete the reservation information with slice hrn
+ found in iotlab_xp table. If not available in the table,
+ assume it is a iotlab slice.
+ -Updates the iotlab table, deleting jobs when necessary.
+
+ :returns: reservation_list, list of dictionaries with 'lease_id',
+ 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list',
+ 'slice_hrn', 'resource_ids', 't_from', 't_until'
+ :rtype: list
+
+ """
+
+ unfiltered_reservation_list = self.GetReservedNodes(login)
+
+ reservation_list = []
+ #Find the slice associated with this user iotlab ldap uid
+ logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\
+ unfiltered_reservation_list %s "
+ % (login, unfiltered_reservation_list))
+ #Create user dict first to avoid looking several times for
+ #the same user in LDAP SA 27/07/12
+ job_oar_list = []
+
+ jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all()
+ jobs_psql_dict = dict([(row.job_id, row.__dict__)
+ for row in jobs_psql_query])
+ #jobs_psql_dict = jobs_psql_dict)
+ logger.debug("CORTEXLAB_API \tGetLeases jobs_psql_dict %s"
+ % (jobs_psql_dict))
+ jobs_psql_id_list = [row.job_id for row in jobs_psql_query]
+
+ for resa in unfiltered_reservation_list:
+ logger.debug("CORTEXLAB_API \tGetLeases USER %s"
+ % (resa['user']))
+ #Construct list of jobs (runing, waiting..) in oar
+ job_oar_list.append(resa['lease_id'])
+ #If there is information on the job in IOTLAB DB ]
+ #(slice used and job id)
+ if resa['lease_id'] in jobs_psql_dict:
+ job_info = jobs_psql_dict[resa['lease_id']]
+ logger.debug("CORTEXLAB_API \tGetLeases job_info %s"
+ % (job_info))
+ resa['slice_hrn'] = job_info['slice_hrn']
+ resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
+
+ #otherwise, assume it is a iotlab slice:
+ else:
+ resa['slice_id'] = hrn_to_urn(self.root_auth + '.' +
+ resa['user'] + "_slice", 'slice')
+ resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
+
+ resa['component_id_list'] = []
+ #Transform the hostnames into urns (component ids)
+ for node in resa['reserved_nodes']:
+
+ iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
+ resa['component_id_list'].append(iotlab_xrn.urn)
+
+ if lease_filter_dict:
+ logger.debug("CORTEXLAB_API \tGetLeases \
+ \r\n leasefilter %s" % ( lease_filter_dict))
+
+ filter_dict_functions = {
+ 'slice_hrn' : IotlabTestbedAPI.filter_lease_name,
+ 't_from' : IotlabTestbedAPI.filter_lease_start_time
+ }
+ reservation_list = list(unfiltered_reservation_list)
+ for filter_type in lease_filter_dict:
+ logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \
+ % (reservation_list))
+ reservation_list = filter_dict_functions[filter_type](\
+ reservation_list,lease_filter_dict[filter_type] )
+
+ # Filter the reservation list with a maximum timespan so that the
+ # leases and jobs running after this timestamp do not appear
+ # in the result leases.
+ # if 'start_time' in :
+ # if resa['start_time'] < lease_filter_dict['start_time']:
+ # reservation_list.append(resa)
+
+
+ # if 'name' in lease_filter_dict and \
+ # lease_filter_dict['name'] == resa['slice_hrn']:
+ # reservation_list.append(resa)
+
+
+ if lease_filter_dict is None:
+ reservation_list = unfiltered_reservation_list
+
+ self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list)
+
+ logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s"
+ % (reservation_list))
+ return reservation_list
+
+
+
+
+#TODO FUNCTIONS SECTION 04/07/2012 SA
+
+ ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
+ ##04/07/2012 SA
+ #@staticmethod
+ #def UnBindObjectFromPeer( auth, object_type, object_id, shortname):
+ #""" This method is a hopefully temporary hack to let the sfa correctly
+ #detach the objects it creates from a remote peer object. This is
+ #needed so that the sfa federation link can work in parallel with
+ #RefreshPeer, as RefreshPeer depends on remote objects being correctly
+ #marked.
+ #Parameters:
+ #auth : struct, API authentication structure
+ #AuthMethod : string, Authentication method to use
+ #object_type : string, Object type, among 'site','person','slice',
+ #'node','key'
+ #object_id : int, object_id
+ #shortname : string, peer shortname
+ #FROM PLC DOC
+
+ #"""
+ #logger.warning("CORTEXLAB_API \tUnBindObjectFromPeer EMPTY-\
+ #DO NOTHING \r\n ")
+ #return
+
+ ##TODO Is BindObjectToPeer still necessary ? Currently does nothing
+ ##04/07/2012 SA
+ #|| Commented out 28/05/13 SA
+ #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
+ #remote_object_id=None):
+ #"""This method is a hopefully temporary hack to let the sfa correctly
+ #attach the objects it creates to a remote peer object. This is needed
+ #so that the sfa federation link can work in parallel with RefreshPeer,
+ #as RefreshPeer depends on remote objects being correctly marked.
+ #Parameters:
+ #shortname : string, peer shortname
+ #remote_object_id : int, remote object_id, set to 0 if unknown
+ #FROM PLC API DOC
+
+ #"""
+ #logger.warning("CORTEXLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
+ #return
+
+ ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA
+ ##Funciton should delete and create another job since oin iotlab slice=job
+ #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
+ #"""Updates the parameters of an existing slice with the values in
+ #slice_fields.
+ #Users may only update slices of which they are members.
+ #PIs may update any of the slices at their sites, or any slices of
+ #which they are members. Admins may update any slice.
+ #Only PIs and admins may update max_nodes. Slices cannot be renewed
+ #(by updating the expires parameter) more than 8 weeks into the future.
+ #Returns 1 if successful, faults otherwise.
+ #FROM PLC API DOC
+
+ #"""
+ #logger.warning("CORTEXLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ")
+ #return
+
+ #Unused SA 30/05/13, we only update the user's key or we delete it.
+ ##TODO UpdatePerson 04/07/2012 SA
+ #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None):
+ #"""Updates a person. Only the fields specified in person_fields
+ #are updated, all other fields are left untouched.
+ #Users and techs can only update themselves. PIs can only update
+ #themselves and other non-PIs at their sites.
+ #Returns 1 if successful, faults otherwise.
+ #FROM PLC API DOC
+
+ #"""
+ ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
+ ##self.iotlab_db.iotlab_session.add(new_row)
+ ##self.iotlab_db.iotlab_session.commit()
+
+ #logger.debug("CORTEXLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
+ #return
+
+ @staticmethod
+ def GetKeys(key_filter=None):
+ """Returns a dict of dict based on the key string. Each dict entry
+ contains the key id, the ssh key, the user's email and the
+ user's hrn.
+ If key_filter is specified and is an array of key identifiers,
+ only keys matching the filter will be returned.
+
+ Admin may query all keys. Non-admins may only query their own keys.
+ FROM PLC API DOC
+
+ :returns: dict with ssh key as key and dicts as value.
+ :rtype: dict
+ """
+ if key_filter is None:
+ keys = dbsession.query(RegKey).options(joinedload('reg_user')).all()
+ else:
+ keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all()
+
+ key_dict = {}
+ for key in keys:
+ key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
+ 'email': key.reg_user.email,
+ 'hrn': key.reg_user.hrn}
+
+ #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
+ #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
+ #for user in ldap_rslt)
+
+ logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
+ return key_dict
+
+ #TODO : test
+ def DeleteKey(self, user_record, key_string):
+ """Deletes a key in the LDAP entry of the specified user.
+
+ Removes the key_string from the user's key list and updates the LDAP
+ user's entry with the new key attributes.
+
+ :param key_string: The ssh key to remove
+ :param user_record: User's record
+ :type key_string: string
+ :type user_record: dict
+ :returns: True if sucessful, False if not.
+ :rtype: Boolean
+
+ """
+ all_user_keys = user_record['keys']
+ all_user_keys.remove(key_string)
+ new_attributes = {'sshPublicKey':all_user_keys}
+ ret = self.ldap.LdapModifyUser(user_record, new_attributes)
+ logger.debug("CORTEXLAB_API DeleteKey %s- " % (ret))
+ return ret['bool']
+
+
+
+
+ @staticmethod
+ def _sql_get_slice_info(slice_filter):
+ """
+ Get the slice record based on the slice hrn. Fetch the record of the
+ user associated with the slice by using joinedload based on the
+ reg_researcher relationship.
+
+ :param slice_filter: the slice hrn we are looking for
+ :type slice_filter: string
+ :returns: the slice record enhanced with the user's information if the
+ slice was found, None it wasn't.
+
+ :rtype: dict or None.
+ """
+ #DO NOT USE RegSlice - reg_researchers to get the hrn
+ #of the user otherwise will mess up the RegRecord in
+ #Resolve, don't know why - SA 08/08/2012
+
+ #Only one entry for one user = one slice in iotlab_xp table
+ #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
+ raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
+ #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
+ if raw_slicerec:
+ #load_reg_researcher
+ #raw_slicerec.reg_researchers
+ raw_slicerec = raw_slicerec.__dict__
+ logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \
+ raw_slicerec %s" % (slice_filter, raw_slicerec))
+ slicerec = raw_slicerec
+ #only one researcher per slice so take the first one
+ #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
+ #del slicerec['reg_researchers']['_sa_instance_state']
+ return slicerec
+
+ else:
+ return None
+
+ @staticmethod
+ def _sql_get_slice_info_from_user(slice_filter):
+ """
+ Get the slice record based on the user recordid by using a joinedload
+ on the relationship reg_slices_as_researcher. Format the sql record
+ into a dict with the mandatory fields for user and slice.
+ :returns: dict with slice record and user record if the record was found
+ based on the user's id, None if not..
+ :rtype:dict or None..
+ """
+ #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
+ raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
+ #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
+ #Put it in correct order
+ user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
+ 'classtype', 'authority', 'gid', 'record_id',
+ 'date_created', 'type', 'email', 'pointer']
+ slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
+ 'classtype', 'authority', 'gid', 'record_id',
+ 'date_created', 'type', 'pointer']
+ if raw_slicerec:
+ #raw_slicerec.reg_slices_as_researcher
+ raw_slicerec = raw_slicerec.__dict__
+ slicerec = {}
+ slicerec = \
+ dict([(k, raw_slicerec[
+ 'reg_slices_as_researcher'][0].__dict__[k])
+ for k in slice_needed_fields])
+ slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
+ for k in user_needed_fields])
+ #TODO Handle multiple slices for one user SA 10/12/12
+ #for now only take the first slice record associated to the rec user
+ ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
+ #del raw_slicerec['reg_slices_as_researcher']
+ #slicerec['reg_researchers'] = raw_slicerec
+ ##del slicerec['_sa_instance_state']
+
+ return slicerec
+
+ else:
+ return None
+
+ def _get_slice_records(self, slice_filter=None,
+ slice_filter_type=None):
+ """
+ Get the slice record depending on the slice filter and its type.
+ :param slice_filter: Can be either the slice hrn or the user's record
+ id.
+ :type slice_filter: string
+ :param slice_filter_type: describes the slice filter type used, can be
+ slice_hrn or record_id_user
+ :type: string
+ :returns: the slice record
+ :rtype:dict
+ .. seealso::_sql_get_slice_info_from_user
+ .. seealso:: _sql_get_slice_info
+ """
+
+ #Get list of slices based on the slice hrn
+ if slice_filter_type == 'slice_hrn':
+
+ #if get_authority(slice_filter) == self.root_auth:
+ #login = slice_filter.split(".")[1].split("_")[0]
+
+ slicerec = self._sql_get_slice_info(slice_filter)
+
+ if slicerec is None:
+ return None
+ #return login, None
+
+ #Get slice based on user id
+ if slice_filter_type == 'record_id_user':
+
+ slicerec = self._sql_get_slice_info_from_user(slice_filter)
+
+ if slicerec:
+ fixed_slicerec_dict = slicerec
+ #At this point if there is no login it means
+ #record_id_user filter has been used for filtering
+ #if login is None :
+ ##If theslice record is from iotlab
+ #if fixed_slicerec_dict['peer_authority'] is None:
+ #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
+ #return login, fixed_slicerec_dict
+ return fixed_slicerec_dict
+ else:
+ return None
+
+
+ def GetSlices(self, slice_filter=None, slice_filter_type=None,
+ login=None):
+ """Get the slice records from the iotlab db and add lease information
+ if any.
+
+ :param slice_filter: can be the slice hrn or slice record id in the db
+ depending on the slice_filter_type.
+ :param slice_filter_type: defines the type of the filtering used, Can be
+ either 'slice_hrn' or "record_id'.
+ :type slice_filter: string
+ :type slice_filter_type: string
+ :returns: a slice dict if slice_filter and slice_filter_type
+ are specified and a matching entry is found in the db. The result
+ is put into a list.Or a list of slice dictionnaries if no filters
+ arespecified.
+
+ :rtype: list
+
+ """
+ #login = None
+ authorized_filter_types_list = ['slice_hrn', 'record_id_user']
+ return_slicerec_dictlist = []
+
+ #First try to get information on the slice based on the filter provided
+ if slice_filter_type in authorized_filter_types_list:
+ fixed_slicerec_dict = self._get_slice_records(slice_filter,
+ slice_filter_type)
+ # if the slice was not found in the sfa db
+ if fixed_slicerec_dict is None:
+ return return_slicerec_dictlist
+
+ slice_hrn = fixed_slicerec_dict['hrn']
+
+ logger.debug(" CORTEXLAB_API \tGetSlices login %s \
+ slice record %s slice_filter %s \
+ slice_filter_type %s " % (login,
+ fixed_slicerec_dict, slice_filter,
+ slice_filter_type))
+
+
+ #Now we have the slice record fixed_slicerec_dict, get the
+ #jobs associated to this slice
+ leases_list = []
+
+ leases_list = self.GetLeases(login=login)
+ #If no job is running or no job scheduled
+ #return only the slice record
+ if leases_list == [] and fixed_slicerec_dict:
+ return_slicerec_dictlist.append(fixed_slicerec_dict)
+
+ # if the jobs running don't belong to the user/slice we are looking
+ # for
+ leases_hrn = [lease['slice_hrn'] for lease in leases_list]
+ if slice_hrn not in leases_hrn:
+ return_slicerec_dictlist.append(fixed_slicerec_dict)
+ #If several jobs for one slice , put the slice record into
+ # each lease information dict
+ for lease in leases_list:
+ slicerec_dict = {}
+ logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \
+ \t lease['slice_hrn'] %s"
+ % (slice_filter, lease['slice_hrn']))
+ if lease['slice_hrn'] == slice_hrn:
+ slicerec_dict['oar_job_id'] = lease['lease_id']
+ #Update lease dict with the slice record
+ if fixed_slicerec_dict:
+ fixed_slicerec_dict['oar_job_id'] = []
+ fixed_slicerec_dict['oar_job_id'].append(
+ slicerec_dict['oar_job_id'])
+ slicerec_dict.update(fixed_slicerec_dict)
+ #slicerec_dict.update({'hrn':\
+ #str(fixed_slicerec_dict['slice_hrn'])})
+ slicerec_dict['slice_hrn'] = lease['slice_hrn']
+ slicerec_dict['hrn'] = lease['slice_hrn']
+ slicerec_dict['user'] = lease['user']
+ slicerec_dict.update(
+ {'list_node_ids':
+ {'hostname': lease['reserved_nodes']}})
+ slicerec_dict.update({'node_ids': lease['reserved_nodes']})
+
+
+
+ return_slicerec_dictlist.append(slicerec_dict)
+ logger.debug("CORTEXLAB_API.PY \tGetSlices \
+ OHOHOHOH %s" %(return_slicerec_dictlist))
+
+ logger.debug("CORTEXLAB_API.PY \tGetSlices \
+ slicerec_dict %s return_slicerec_dictlist %s \
+ lease['reserved_nodes'] \
+ %s" % (slicerec_dict, return_slicerec_dictlist,
+ lease['reserved_nodes']))
+
+ logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
+ return_slicerec_dictlist %s"
+ % (return_slicerec_dictlist))
+
+ return return_slicerec_dictlist
+
+
+ else:
+ #Get all slices from the iotlab sfa database ,
+ #put them in dict format
+ #query_slice_list = dbsession.query(RegRecord).all()
+ query_slice_list = \
+ dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
+
+ for record in query_slice_list:
+ tmp = record.__dict__
+ tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
+ #del tmp['reg_researchers']['_sa_instance_state']
+ return_slicerec_dictlist.append(tmp)
+ #return_slicerec_dictlist.append(record.__dict__)
+
+ #Get all the jobs reserved nodes
+ leases_list = self.GetReservedNodes()
+
+ for fixed_slicerec_dict in return_slicerec_dictlist:
+ slicerec_dict = {}
+ #Check if the slice belongs to a iotlab user
+ if fixed_slicerec_dict['peer_authority'] is None:
+ owner = fixed_slicerec_dict['hrn'].split(
+ ".")[1].split("_")[0]
+ else:
+ owner = None
+ for lease in leases_list:
+ if owner == lease['user']:
+ slicerec_dict['oar_job_id'] = lease['lease_id']
+
+ #for reserved_node in lease['reserved_nodes']:
+ logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s "
+ % (lease))
+ slicerec_dict.update(fixed_slicerec_dict)
+ slicerec_dict.update({'node_ids':
+ lease['reserved_nodes']})
+ slicerec_dict.update({'list_node_ids':
+ {'hostname':
+ lease['reserved_nodes']}})
+
+ #slicerec_dict.update({'hrn':\
+ #str(fixed_slicerec_dict['slice_hrn'])})
+ #return_slicerec_dictlist.append(slicerec_dict)
+ fixed_slicerec_dict.update(slicerec_dict)
+
+ logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
+ return_slicerec_dictlist %s \slice_filter %s " \
+ %(return_slicerec_dictlist, slice_filter))
+
+ return return_slicerec_dictlist
+
+
+
+ #Update slice unused, therefore sfa_fields_to_iotlab_fields unused
+ #SA 30/05/13
+ #@staticmethod
+ #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record):
+ #"""
+ #"""
+
+ #iotlab_record = {}
+ ##for field in record:
+ ## iotlab_record[field] = record[field]
+
+ #if sfa_type == "slice":
+ ##instantion used in get_slivers ?
+ #if not "instantiation" in iotlab_record:
+ #iotlab_record["instantiation"] = "iotlab-instantiated"
+ ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn)
+ ##Unused hrn_to_pl_slicename because Iotlab's hrn already
+ ##in the appropriate form SA 23/07/12
+ #iotlab_record["hrn"] = hrn
+ #logger.debug("CORTEXLAB_API.PY sfa_fields_to_iotlab_fields \
+ #iotlab_record %s " %(iotlab_record['hrn']))
+ #if "url" in record:
+ #iotlab_record["url"] = record["url"]
+ #if "description" in record:
+ #iotlab_record["description"] = record["description"]
+ #if "expires" in record:
+ #iotlab_record["expires"] = int(record["expires"])
+
+ ##nodes added by OAR only and then imported to SFA
+ ##elif type == "node":
+ ##if not "hostname" in iotlab_record:
+ ##if not "hostname" in record:
+ ##raise MissingSfaInfo("hostname")
+ ##iotlab_record["hostname"] = record["hostname"]
+ ##if not "model" in iotlab_record:
+ ##iotlab_record["model"] = "geni"
+
+ ##One authority only
+ ##elif type == "authority":
+ ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn)
+
+ ##if not "name" in iotlab_record:
+ ##iotlab_record["name"] = hrn
+
+ ##if not "abbreviated_name" in iotlab_record:
+ ##iotlab_record["abbreviated_name"] = hrn
+
+ ##if not "enabled" in iotlab_record:
+ ##iotlab_record["enabled"] = True
+
+ ##if not "is_public" in iotlab_record:
+ ##iotlab_record["is_public"] = True
+
+ #return iotlab_record
+
+
+
+
+
+
+
+
+
+
--- /dev/null
+"""
+Implements what a driver should provide for SFA to work.
+"""
+from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
+from sfa.util.sfalogging import logger
+from sfa.storage.alchemy import dbsession
+from sfa.storage.model import RegRecord
+
+from sfa.managers.driver import Driver
+from sfa.rspecs.version_manager import VersionManager
+from sfa.rspecs.rspec import RSpec
+
+from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
+
+from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname
+from sfa.iotlab.iotlabslices import IotlabSlices
+
+
+from sfa.iotlab.iotlabapi import IotlabTestbedAPI
+
+
+class CortexlabDriver(Driver):
+ """ Cortexlab Driver class inherited from Driver generic class.
+
+ Contains methods compliant with the SFA standard and the testbed
+ infrastructure (calls to LDAP and scheduler to book the nodes).
+
+ .. seealso::: Driver class
+
+ """
+ def __init__(self, config):
+ """
+
+ Sets the iotlab SFA config parameters,
+ instanciates the testbed api and the iotlab database.
+
+ :param config: iotlab SFA configuration object
+ :type config: Config object
+
+ """
+ Driver.__init__(self, config)
+ self.config = config
+ self.iotlab_api = IotlabTestbedAPI(config)
+ self.cache = None
+
+ def augment_records_with_testbed_info(self, record_list):
+ """
+
+ Adds specific testbed info to the records.
+
+ :param record_list: list of sfa dictionaries records
+ :type record_list: list
+ :returns: list of records with extended information in each record
+ :rtype: list
+
+ """
+ return self.fill_record_info(record_list)
+
+ def fill_record_info(self, record_list):
+ """
+
+ For each SFA record, fill in the iotlab specific and SFA specific
+ fields in the record.
+
+ :param record_list: list of sfa dictionaries records
+ :type record_list: list
+ :returns: list of records with extended information in each record
+ :rtype: list
+
+ .. warning:: Should not be modifying record_list directly because modi
+ fication are kept outside the method's scope. Howerver, there is no
+ other way to do it given the way it's called in registry manager.
+
+ """
+
+ logger.debug("IOTLABDRIVER \tfill_record_info records %s "
+ % (record_list))
+ if not isinstance(record_list, list):
+ record_list = [record_list]
+
+ try:
+ for record in record_list:
+
+ if str(record['type']) == 'node':
+ # look for node info using GetNodes
+ # the record is about one node only
+ filter_dict = {'hrn': [record['hrn']]}
+ node_info = self.iotlab_api.GetNodes(filter_dict)
+ # the node_info is about one node only, but it is formatted
+ # as a list
+ record.update(node_info[0])
+ logger.debug("IOTLABDRIVER.PY \t \
+ fill_record_info NODE" % (record))
+
+ #If the record is a SFA slice record, then add information
+ #about the user of this slice. This kind of
+ #information is in the Iotlab's DB.
+ if str(record['type']) == 'slice':
+ if 'reg_researchers' in record and isinstance(record
+ ['reg_researchers'],
+ list):
+ record['reg_researchers'] = \
+ record['reg_researchers'][0].__dict__
+ record.update(
+ {'PI': [record['reg_researchers']['hrn']],
+ 'researcher': [record['reg_researchers']['hrn']],
+ 'name': record['hrn'],
+ 'oar_job_id': [],
+ 'node_ids': [],
+ 'person_ids': [record['reg_researchers']
+ ['record_id']],
+ # For client_helper.py compatibility
+ 'geni_urn': '',
+ # For client_helper.py compatibility
+ 'keys': '',
+ # For client_helper.py compatibility
+ 'key_ids': ''})
+
+ #Get iotlab slice record and oar job id if any.
+ recslice_list = self.iotlab_api.GetSlices(
+ slice_filter=str(record['hrn']),
+ slice_filter_type='slice_hrn')
+
+ logger.debug("IOTLABDRIVER \tfill_record_info \
+ TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\
+ %s " % (record['hrn'], record['oar_job_id']))
+ del record['reg_researchers']
+ try:
+ for rec in recslice_list:
+ logger.debug("IOTLABDRIVER\r\n \t \
+ fill_record_info oar_job_id %s "
+ % (rec['oar_job_id']))
+
+ record['node_ids'] = [self.iotlab_api.root_auth +
+ '.' + hostname for hostname
+ in rec['node_ids']]
+ except KeyError:
+ pass
+
+ logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \
+ recslice_list %s \r\n \t RECORD %s \r\n \
+ \r\n" % (recslice_list, record))
+
+ if str(record['type']) == 'user':
+ #The record is a SFA user record.
+ #Get the information about his slice from Iotlab's DB
+ #and add it to the user record.
+ recslice_list = self.iotlab_api.GetSlices(
+ slice_filter=record['record_id'],
+ slice_filter_type='record_id_user')
+
+ logger.debug("IOTLABDRIVER.PY \t fill_record_info \
+ TYPE USER recslice_list %s \r\n \t RECORD %s \r\n"
+ % (recslice_list, record))
+ #Append slice record in records list,
+ #therefore fetches user and slice info again(one more loop)
+ #Will update PIs and researcher for the slice
+
+ recuser = recslice_list[0]['reg_researchers']
+ logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \
+ recuser %s \r\n \r\n" % (recuser))
+ recslice = {}
+ recslice = recslice_list[0]
+ recslice.update(
+ {'PI': [recuser['hrn']],
+ 'researcher': [recuser['hrn']],
+ 'name': record['hrn'],
+ 'node_ids': [],
+ 'oar_job_id': [],
+ 'person_ids': [recuser['record_id']]})
+ try:
+ for rec in recslice_list:
+ recslice['oar_job_id'].append(rec['oar_job_id'])
+ except KeyError:
+ pass
+
+ recslice.update({'type': 'slice',
+ 'hrn': recslice_list[0]['hrn']})
+
+ #GetPersons takes [] as filters
+ user_iotlab = self.iotlab_api.GetPersons([record])
+
+ record.update(user_iotlab[0])
+ #For client_helper.py compatibility
+ record.update(
+ {'geni_urn': '',
+ 'keys': '',
+ 'key_ids': ''})
+ record_list.append(recslice)
+
+ logger.debug("IOTLABDRIVER.PY \t \
+ fill_record_info ADDING SLICE\
+ INFO TO USER records %s" % (record_list))
+
+ except TypeError, error:
+ logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"
+ % (error))
+
+ return record_list
+
+ def sliver_status(self, slice_urn, slice_hrn):
+ """
+ Receive a status request for slice named urn/hrn
+ urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
+ shall return a structure as described in
+ http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
+ NT : not sure if we should implement this or not, but used by sface.
+
+ :param slice_urn: slice urn
+ :type slice_urn: string
+ :param slice_hrn: slice hrn
+ :type slice_hrn: string
+
+ """
+
+ #First get the slice with the slice hrn
+ slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn,
+ slice_filter_type='slice_hrn')
+
+ if len(slice_list) == 0:
+ raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
+
+ #Used for fetching the user info witch comes along the slice info
+ one_slice = slice_list[0]
+
+ #Make a list of all the nodes hostnames in use for this slice
+ slice_nodes_list = []
+ slice_nodes_list = one_slice['node_ids']
+ #Get all the corresponding nodes details
+ nodes_all = self.iotlab_api.GetNodes(
+ {'hostname': slice_nodes_list},
+ ['node_id', 'hostname', 'site', 'boot_state'])
+ nodeall_byhostname = dict([(one_node['hostname'], one_node)
+ for one_node in nodes_all])
+
+ for single_slice in slice_list:
+ #For compatibility
+ top_level_status = 'empty'
+ result = {}
+ result.fromkeys(
+ ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status',
+ 'geni_resources'], None)
+ # result.fromkeys(\
+ # ['geni_urn','geni_error', 'pl_login','geni_status',
+ # 'geni_resources'], None)
+ # result['pl_login'] = one_slice['reg_researchers'][0].hrn
+ result['iotlab_login'] = one_slice['user']
+ logger.debug("Slabdriver - sliver_status Sliver status \
+ urn %s hrn %s single_slice %s \r\n "
+ % (slice_urn, slice_hrn, single_slice))
+
+ if 'node_ids' not in single_slice:
+ #No job in the slice
+ result['geni_status'] = top_level_status
+ result['geni_resources'] = []
+ return result
+
+ top_level_status = 'ready'
+
+ #A job is running on Iotlab for this slice
+ # report about the local nodes that are in the slice only
+
+ result['geni_urn'] = slice_urn
+
+ resources = []
+ for node_hostname in single_slice['node_ids']:
+ res = {}
+ res['iotlab_hostname'] = node_hostname
+ res['iotlab_boot_state'] = \
+ nodeall_byhostname[node_hostname]['boot_state']
+
+ #res['pl_hostname'] = node['hostname']
+ #res['pl_boot_state'] = \
+ #nodeall_byhostname[node['hostname']]['boot_state']
+ #res['pl_last_contact'] = strftime(self.time_format, \
+ #gmtime(float(timestamp)))
+ sliver_id = Xrn(
+ slice_urn, type='slice',
+ id=nodeall_byhostname[node_hostname]['node_id']).urn
+
+ res['geni_urn'] = sliver_id
+ #node_name = node['hostname']
+ if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
+
+ res['geni_status'] = 'ready'
+ else:
+ res['geni_status'] = 'failed'
+ top_level_status = 'failed'
+
+ res['geni_error'] = ''
+
+ resources.append(res)
+
+ result['geni_status'] = top_level_status
+ result['geni_resources'] = resources
+ logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "
+ % (resources, res))
+ return result
+
+ @staticmethod
+ def get_user_record(hrn):
+ """
+
+ Returns the user record based on the hrn from the SFA DB .
+
+ :param hrn: user's hrn
+ :type hrn: string
+ :returns: user record from SFA database
+ :rtype: RegUser
+
+ """
+ return dbsession.query(RegRecord).filter_by(hrn=hrn).first()
+
+ def testbed_name(self):
+ """
+
+ Returns testbed's name.
+ :returns: testbed authority name.
+ :rtype: string
+
+ """
+ return self.hrn
+
+ # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
+ def aggregate_version(self):
+ """
+
+ Returns the testbed's supported rspec advertisement and request
+ versions.
+ :returns: rspec versions supported ad a dictionary.
+ :rtype: dict
+
+ """
+ version_manager = VersionManager()
+ ad_rspec_versions = []
+ request_rspec_versions = []
+ for rspec_version in version_manager.versions:
+ if rspec_version.content_type in ['*', 'ad']:
+ ad_rspec_versions.append(rspec_version.to_dict())
+ if rspec_version.content_type in ['*', 'request']:
+ request_rspec_versions.append(rspec_version.to_dict())
+ return {
+ 'testbed': self.testbed_name(),
+ 'geni_request_rspec_versions': request_rspec_versions,
+ 'geni_ad_rspec_versions': ad_rspec_versions}
+
+ def _get_requested_leases_list(self, rspec):
+ """
+ Process leases in rspec depending on the rspec version (format)
+ type. Find the lease requests in the rspec and creates
+ a lease request list with the mandatory information ( nodes,
+ start time and duration) of the valid leases (duration above or
+ equal to the iotlab experiment minimum duration).
+
+ :param rspec: rspec request received.
+ :type rspec: RSpec
+ :returns: list of lease requests found in the rspec
+ :rtype: list
+ """
+ requested_lease_list = []
+ for lease in rspec.version.get_leases():
+ single_requested_lease = {}
+ logger.debug("IOTLABDRIVER.PY \t \
+ _get_requested_leases_list lease %s " % (lease))
+
+ if not lease.get('lease_id'):
+ if get_authority(lease['component_id']) == \
+ self.iotlab_api.root_auth:
+ single_requested_lease['hostname'] = \
+ iotlab_xrn_to_hostname(\
+ lease.get('component_id').strip())
+ single_requested_lease['start_time'] = \
+ lease.get('start_time')
+ single_requested_lease['duration'] = lease.get('duration')
+ #Check the experiment's duration is valid before adding
+ #the lease to the requested leases list
+ duration_in_seconds = \
+ int(single_requested_lease['duration'])
+ if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInGranularity():
+ requested_lease_list.append(single_requested_lease)
+
+ return requested_lease_list
+
+ @staticmethod
+ def _group_leases_by_start_time(requested_lease_list):
+ """
+ Create dict of leases by start_time, regrouping nodes reserved
+ at the same time, for the same amount of time so as to
+ define one job on OAR.
+
+ :param requested_lease_list: list of leases
+ :type requested_lease_list: list
+ :returns: Dictionary with key = start time, value = list of leases
+ with the same start time.
+ :rtype: dictionary
+
+ """
+
+ requested_job_dict = {}
+ for lease in requested_lease_list:
+
+ #In case it is an asap experiment start_time is empty
+ if lease['start_time'] == '':
+ lease['start_time'] = '0'
+
+ if lease['start_time'] not in requested_job_dict:
+ if isinstance(lease['hostname'], str):
+ lease['hostname'] = [lease['hostname']]
+
+ requested_job_dict[lease['start_time']] = lease
+
+ else:
+ job_lease = requested_job_dict[lease['start_time']]
+ if lease['duration'] == job_lease['duration']:
+ job_lease['hostname'].append(lease['hostname'])
+
+ return requested_job_dict
+
+ def _process_requested_jobs(self, rspec):
+ """
+ Turns the requested leases and information into a dictionary
+ of requested jobs, grouped by starting time.
+
+ :param rspec: RSpec received
+ :type rspec : RSpec
+ :rtype: dictionary
+
+ """
+ requested_lease_list = self._get_requested_leases_list(rspec)
+ logger.debug("IOTLABDRIVER _process_requested_jobs \
+ requested_lease_list %s" % (requested_lease_list))
+ job_dict = self._group_leases_by_start_time(requested_lease_list)
+ logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\
+ %s" % (job_dict))
+
+ return job_dict
+
+ def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string,
+ users, options):
+ """Answer to CreateSliver.
+
+ Creates the leases and slivers for the users from the information
+ found in the rspec string.
+ Launch experiment on OAR if the requested leases is valid. Delete
+ no longer requested leases.
+
+
+ :param creds: user's credentials
+ :type creds: string
+ :param users: user record list
+ :type users: list
+ :param options:
+ :type options:
+
+ :returns: a valid Rspec for the slice which has just been
+ modified.
+ :rtype: RSpec
+
+
+ """
+ aggregate = IotlabAggregate(self)
+
+ slices = IotlabSlices(self)
+ peer = slices.get_peer(slice_hrn)
+ sfa_peer = slices.get_sfa_peer(slice_hrn)
+ slice_record = None
+
+ if not isinstance(creds, list):
+ creds = [creds]
+
+ if users:
+ slice_record = users[0].get('slice_record', {})
+ logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\
+ creds %s \r\n \r\n users %s"
+ % (creds, users))
+ slice_record['user'] = {'keys': users[0]['keys'],
+ 'email': users[0]['email'],
+ 'hrn': slice_record['reg-researchers'][0]}
+ # parse rspec
+ rspec = RSpec(rspec_string)
+ logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \
+ %s slice_record %s users %s"
+ % (rspec.version, slice_record, users))
+
+ # ensure site record exists?
+ # ensure slice record exists
+ #Removed options in verify_slice SA 14/08/12
+ #Removed peer record in verify_slice SA 18/07/13
+ sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer)
+
+ # ensure person records exists
+ #verify_persons returns added persons but the return value
+ #is not used
+ #Removed peer record and sfa_peer in verify_persons SA 18/07/13
+ slices.verify_persons(slice_hrn, sfa_slice, users, options=options)
+ #requested_attributes returned by rspec.version.get_slice_attributes()
+ #unused, removed SA 13/08/12
+ #rspec.version.get_slice_attributes()
+
+ logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice))
+
+ # add/remove slice from nodes
+
+ #requested_slivers = [node.get('component_id') \
+ #for node in rspec.version.get_nodes_with_slivers()\
+ #if node.get('authority_id') is self.iotlab_api.root_auth]
+ #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
+ #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
+ #requested_slivers %s listnodes %s" \
+ #%(requested_slivers,l))
+ #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
+ #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
+
+ requested_job_dict = self._process_requested_jobs(rspec)
+
+ logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "
+ % (requested_job_dict))
+ #verify_slice_leases returns the leases , but the return value is unused
+ #here. Removed SA 13/08/12
+ slices.verify_slice_leases(sfa_slice,
+ requested_job_dict, peer)
+
+ return aggregate.get_rspec(slice_xrn=slice_urn,
+ login=sfa_slice['login'],
+ version=rspec.version)
+
+ def delete_sliver(self, slice_urn, slice_hrn, creds, options):
+ """
+ Deletes the lease associated with the slice hrn and the credentials
+ if the slice belongs to iotlab. Answer to DeleteSliver.
+
+ :param slice_urn: urn of the slice
+ :param slice_hrn: name of the slice
+ :param creds: slice credenials
+ :type slice_urn: string
+ :type slice_hrn: string
+ :type creds: ? unused
+
+ :returns: 1 if the slice to delete was not found on iotlab,
+ True if the deletion was successful, False otherwise otherwise.
+
+ .. note:: Should really be named delete_leases because iotlab does
+ not have any slivers, but only deals with leases. However,
+ SFA api only have delete_sliver define so far. SA 13/05/2013
+ .. note:: creds are unused, and are not used either in the dummy driver
+ delete_sliver .
+ """
+
+ sfa_slice_list = self.iotlab_api.GetSlices(
+ slice_filter=slice_hrn,
+ slice_filter_type='slice_hrn')
+
+ if not sfa_slice_list:
+ return 1
+
+ #Delete all leases in the slice
+ for sfa_slice in sfa_slice_list:
+ logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice))
+ slices = IotlabSlices(self)
+ # determine if this is a peer slice
+
+ peer = slices.get_peer(slice_hrn)
+
+ logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
+ \r\n \t sfa_slice %s " % (peer, sfa_slice))
+ try:
+ self.iotlab_api.DeleteSliceFromNodes(sfa_slice)
+ return True
+ except:
+ return False
+
+ def list_resources (self, slice_urn, slice_hrn, creds, options):
+ """
+
+ List resources from the iotlab aggregate and returns a Rspec
+ advertisement with resources found when slice_urn and slice_hrn are
+ None (in case of resource discovery).
+ If a slice hrn and urn are provided, list experiment's slice
+ nodes in a rspec format. Answer to ListResources.
+ Caching unused.
+
+ :param slice_urn: urn of the slice
+ :param slice_hrn: name of the slice
+ :param creds: slice credenials
+ :type slice_urn: string
+ :type slice_hrn: string
+ :type creds: ? unused
+ :param options: options used when listing resources (list_leases, info,
+ geni_available)
+ :returns: rspec string in xml
+ :rtype: string
+
+ .. note:: creds are unused
+ """
+
+ #cached_requested = options.get('cached', True)
+
+ version_manager = VersionManager()
+ # get the rspec's return format from options
+ rspec_version = \
+ version_manager.get_version(options.get('geni_rspec_version'))
+ version_string = "rspec_%s" % (rspec_version)
+
+ #panos adding the info option to the caching key (can be improved)
+ if options.get('info'):
+ version_string = version_string + "_" + \
+ options.get('info', 'default')
+
+ # Adding the list_leases option to the caching key
+ if options.get('list_leases'):
+ version_string = version_string + "_" + \
+ options.get('list_leases', 'default')
+
+ # Adding geni_available to caching key
+ if options.get('geni_available'):
+ version_string = version_string + "_" + \
+ str(options.get('geni_available'))
+
+ # look in cache first
+ #if cached_requested and self.cache and not slice_hrn:
+ #rspec = self.cache.get(version_string)
+ #if rspec:
+ #logger.debug("IotlabDriver.ListResources: \
+ #returning cached advertisement")
+ #return rspec
+
+ #panos: passing user-defined options
+ aggregate = IotlabAggregate(self)
+
+ rspec = aggregate.get_rspec(slice_xrn=slice_urn,
+ version=rspec_version, options=options)
+
+ # cache the result
+ #if self.cache and not slice_hrn:
+ #logger.debug("Iotlab.ListResources: stores advertisement in cache")
+ #self.cache.add(version_string, rspec)
+
+ return rspec
+
+
+ def list_slices(self, creds, options):
+ """Answer to ListSlices.
+
+ List slices belonging to iotlab, returns slice urns list.
+ No caching used. Options unused but are defined in the SFA method
+ api prototype.
+
+ :returns: slice urns list
+ :rtype: list
+
+ .. note:: creds are unused
+ """
+ # look in cache first
+ #if self.cache:
+ #slices = self.cache.get('slices')
+ #if slices:
+ #logger.debug("PlDriver.list_slices returns from cache")
+ #return slices
+
+ # get data from db
+
+ slices = self.iotlab_api.GetSlices()
+ logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n"
+ % (slices))
+ slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
+
+ slice_urns = [hrn_to_urn(slice_hrn, 'slice')
+ for slice_hrn in slice_hrns]
+
+ # cache the result
+ #if self.cache:
+ #logger.debug ("IotlabDriver.list_slices stores value in cache")
+ #self.cache.add('slices', slice_urns)
+
+ return slice_urns
+
+
+ def register(self, sfa_record, hrn, pub_key):
+ """
+ Adding new user, slice, node or site should not be handled
+ by SFA.
+
+ ..warnings:: should not be used. Different components are in charge of
+ doing this task. Adding nodes = OAR
+ Adding users = LDAP Iotlab
+ Adding slice = Import from LDAP users
+ Adding site = OAR
+
+ :param sfa_record: record provided by the client of the
+ Register API call.
+ :type sfa_record: dict
+ :param pub_key: public key of the user
+ :type pub_key: string
+
+ .. note:: DOES NOTHING. Returns -1.
+
+ """
+ return -1
+
+
+ def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
+ """
+ No site or node record update allowed in Iotlab. The only modifications
+ authorized here are key deletion/addition on an existing user and
+ password change. On an existing user, CAN NOT BE MODIFIED: 'first_name',
+ 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio',
+ 'title', 'accepted_aup'. A slice is bound to its user, so modifying the
+ user's ssh key should nmodify the slice's GID after an import procedure.
+
+ :param old_sfa_record: what is in the db for this hrn
+ :param new_sfa_record: what was passed to the update call
+ :param new_key: the new user's public key
+ :param hrn: the user's sfa hrn
+ :type old_sfa_record: dict
+ :type new_sfa_record: dict
+ :type new_key: string
+ :type hrn: string
+
+ TODO: needs review
+ .. seealso:: update in driver.py.
+
+ """
+ pointer = old_sfa_record['pointer']
+ old_sfa_record_type = old_sfa_record['type']
+
+ # new_key implemented for users only
+ if new_key and old_sfa_record_type not in ['user']:
+ raise UnknownSfaType(old_sfa_record_type)
+
+ if old_sfa_record_type == "user":
+ update_fields = {}
+ all_fields = new_sfa_record
+ for key in all_fields.keys():
+ if key in ['key', 'password']:
+ update_fields[key] = all_fields[key]
+
+ if new_key:
+ # must check this key against the previous one if it exists
+ persons = self.iotlab_api.GetPersons([old_sfa_record])
+ person = persons[0]
+ keys = [person['pkey']]
+ #Get all the person's keys
+ keys_dict = self.iotlab_api.GetKeys(keys)
+
+ # Delete all stale keys, meaning the user has only one key
+ #at a time
+ #TODO: do we really want to delete all the other keys?
+ #Is this a problem with the GID generation to have multiple
+ #keys? SA 30/05/13
+ key_exists = False
+ if key in keys_dict:
+ key_exists = True
+ else:
+ #remove all the other keys
+ for key in keys_dict:
+ self.iotlab_api.DeleteKey(person, key)
+ self.iotlab_api.AddPersonKey(
+ person, {'sshPublicKey': person['pkey']},
+ {'sshPublicKey': new_key})
+ return True
+
+ def remove(self, sfa_record):
+ """
+
+ Removes users only. Mark the user as disabled in LDAP. The user and his
+ slice are then deleted from the db by running an import on the registry.
+
+ :param sfa_record: record is the existing sfa record in the db
+ :type sfa_record: dict
+
+ ..warning::As fas as the slice is concerned, here only the leases are
+ removed from the slice. The slice is record itself is not removed
+ from the db.
+
+ TODO: needs review
+
+ TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
+
+ TODO: return boolean for the slice part
+ """
+ sfa_record_type = sfa_record['type']
+ hrn = sfa_record['hrn']
+ if sfa_record_type == 'user':
+
+ #get user from iotlab ldap
+ person = self.iotlab_api.GetPersons(sfa_record)
+ #No registering at a given site in Iotlab.
+ #Once registered to the LDAP, all iotlab sites are
+ #accesible.
+ if person:
+ #Mark account as disabled in ldap
+ return self.iotlab_api.DeletePerson(sfa_record)
+
+ elif sfa_record_type == 'slice':
+ if self.iotlab_api.GetSlices(slice_filter=hrn,
+ slice_filter_type='slice_hrn'):
+ ret = self.iotlab_api.DeleteSlice(sfa_record)
+ return True
--- /dev/null
+"""
+File defining classes to handle the table in the iotlab dedicated database.
+"""
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+# from sfa.util.config import Config
+from sfa.util.sfalogging import logger
+
+from sqlalchemy import Column, Integer, String
+from sqlalchemy import Table, MetaData
+from sqlalchemy.ext.declarative import declarative_base
+
+# from sqlalchemy.dialects import postgresql
+
+from sqlalchemy.exc import NoSuchTableError
+
+
+#Dict holding the columns names of the table as keys
+#and their type, used for creation of the table
+slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \
+ CASCADE ON UPDATE CASCADE', 'oar_job_id': 'integer DEFAULT -1',
+ 'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'}
+
+#Dict with all the specific iotlab tables
+tablenames_dict = {'iotlab_xp': slice_table}
+
+
+IotlabBase = declarative_base()
+
+
+class IotlabXP (IotlabBase):
+ """ SQL alchemy class to manipulate the rows of the slice_iotlab table in
+ iotlab_sfa database. Handles the records representation and creates the
+ table if it does not exist yet.
+
+ """
+ __tablename__ = 'iotlab_xp'
+
+ slice_hrn = Column(String)
+ job_id = Column(Integer, primary_key=True)
+ end_time = Column(Integer, nullable=False)
+
+ def __init__(self, slice_hrn=None, job_id=None, end_time=None):
+ """
+ Defines a row of the slice_iotlab table
+ """
+ if slice_hrn:
+ self.slice_hrn = slice_hrn
+ if job_id:
+ self.job_id = job_id
+ if end_time:
+ self.end_time = end_time
+
+ def __repr__(self):
+ """Prints the SQLAlchemy record to the format defined
+ by the function.
+ """
+ result = "<iotlab_xp : slice_hrn = %s , job_id %s end_time = %s" \
+ % (self.slice_hrn, self.job_id, self.end_time)
+ result += ">"
+ return result
+
+
+class IotlabDB(object):
+ """ SQL Alchemy connection class.
+ From alchemy.py
+ """
+ # Stores the unique Singleton instance-
+ _connection_singleton = None
+ # defines the database name
+ dbname = "iotlab_sfa"
+
+ class Singleton:
+ """
+ Class used with this Python singleton design pattern to allow the
+ definition of one single instance of iotlab db session in the whole
+ code. Wherever a conenction to the database is needed, this class
+ returns the same instance every time. Removes the need for global
+ variable throughout the code.
+ """
+
+ def __init__(self, config, debug=False):
+ self.iotlab_engine = None
+ self.iotlab_session = None
+ self.url = None
+ self.create_iotlab_engine(config, debug)
+ self.session()
+
+ def create_iotlab_engine(self, config, debug=False):
+ """Creates the SQLAlchemy engine, which is the starting point for
+ any SQLAlchemy application.
+ :param config: configuration object created by SFA based on the
+ configuration file in /etc
+ :param debug: if set to true, echo and echo pool will be set to true
+ as well. If echo is True, all statements as well as a repr() of
+ their parameter lists to the engines logger, which defaults to
+ sys.stdout. If echo_pool is True, the connection pool will log all
+ checkouts/checkins to the logging stream. A python logger can be
+ used to configure this logging directly but so far it has not been
+ configured. Refer to sql alchemy engine documentation.
+
+ :type config: Config instance (sfa.util.config)
+ :type debug: bool
+
+ """
+
+ if debug is True:
+ l_echo_pool = True
+ l_echo = True
+ else:
+ l_echo_pool = False
+ l_echo = False
+ # the former PostgreSQL.py used the psycopg2 directly and was doing
+ #self.connection.set_client_encoding("UNICODE")
+ # it's unclear how to achieve this in sqlalchemy, nor if it's needed
+ # at all
+ # http://www.sqlalchemy.org/docs/dialects/postgresql.html#unicode
+ # we indeed have /var/lib/pgsql/data/postgresql.conf where
+ # this setting is unset, it might be an angle to tweak that if need
+ # be try a unix socket first
+ # - omitting the hostname does the trick
+ unix_url = "postgresql+psycopg2://%s:%s@:%s/%s" \
+ % (config.SFA_DB_USER, config.SFA_DB_PASSWORD,
+ config.SFA_DB_PORT, IotlabDB.dbname)
+
+ # the TCP fallback method
+ tcp_url = "postgresql+psycopg2://%s:%s@%s:%s/%s" \
+ % (config.SFA_DB_USER, config.SFA_DB_PASSWORD,
+ config.SFA_DB_HOST, config.SFA_DB_PORT, IotlabDB.dbname)
+
+ for url in [unix_url, tcp_url]:
+ try:
+ self.iotlab_engine = create_engine(
+ url, echo_pool=l_echo_pool, echo=l_echo)
+ self.check()
+ self.url = url
+ return
+ except:
+ pass
+ self.iotlab_engine = None
+
+ raise Exception("Could not connect to database")
+
+ def check(self):
+ """ Check if a table exists by trying a selection
+ on the table.
+
+ """
+ self.iotlab_engine.execute("select 1").scalar()
+
+
+ def session(self):
+ """
+ Creates a SQLalchemy session. Once the session object is created
+ it should be used throughout the code for all the operations on
+ tables for this given database.
+
+ """
+ if self.iotlab_session is None:
+ Session = sessionmaker()
+ self.iotlab_session = Session(bind=self.iotlab_engine)
+ return self.iotlab_session
+
+ def close_session(self):
+ """
+ Closes connection to database.
+
+ """
+ if self.iotlab_session is None:
+ return
+ self.iotlab_session.close()
+ self.iotlab_session = None
+
+
+ def update_jobs_in_iotlabdb(self, job_oar_list, jobs_psql):
+ """ Cleans the iotlab db by deleting expired and cancelled jobs.
+
+ Compares the list of job ids given by OAR with the job ids that
+ are already in the database, deletes the jobs that are no longer in
+ the OAR job id list.
+
+ :param job_oar_list: list of job ids coming from OAR
+ :type job_oar_list: list
+ :param job_psql: list of job ids from the database.
+ :type job_psql: list
+
+ :returns: None
+ """
+ #Turn the list into a set
+ set_jobs_psql = set(jobs_psql)
+
+ kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
+ logger.debug("\r\n \t update_jobs_in_iotlabdb jobs_psql %s \r\n \
+ job_oar_list %s kept_jobs %s "
+ % (set_jobs_psql, job_oar_list, kept_jobs))
+ deleted_jobs = set_jobs_psql.difference(kept_jobs)
+ deleted_jobs = list(deleted_jobs)
+ if len(deleted_jobs) > 0:
+ self.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
+ self.iotlab_session.commit()
+ return
+
+ def __init__(self, config, debug=False):
+ self.sl_base = IotlabBase
+
+ # Check whether we already have an instance
+ if IotlabDB._connection_singleton is None:
+ IotlabDB._connection_singleton = IotlabDB.Singleton(config, debug)
+
+ # Store instance reference as the only member in the handle
+ self._EventHandler_singleton = IotlabDB._connection_singleton
+
+ def __getattr__(self, aAttr):
+ """
+ Delegate access to implementation.
+
+ :param aAttr: Attribute wanted.
+ :returns: Attribute
+ """
+ return getattr(self._connection_singleton, aAttr)
+
+
+
+ # def __setattr__(self, aAttr, aValue):
+ # """Delegate access to implementation.
+
+ # :param attr: Attribute wanted.
+ # :param value: Vaule to be set.
+ # :return: Result of operation.
+ # """
+ # return setattr(self._connection_singleton, aAttr, aValue)
+
+ def exists(self, tablename):
+ """
+ Checks if the table specified as tablename exists.
+ :param tablename: name of the table in the db that has to be checked.
+ :type tablename: string
+ :returns: True if the table exists, False otherwise.
+ :rtype: bool
+
+ """
+ metadata = MetaData(bind=self.iotlab_engine)
+ try:
+ table = Table(tablename, metadata, autoload=True)
+ return True
+
+ except NoSuchTableError:
+ logger.log_exc("IOTLABPOSTGRES tablename %s does not exist"
+ % (tablename))
+ return False
+
+ def createtable(self):
+ """
+ Creates all the table sof the engine.
+ Uses the global dictionnary holding the tablenames and the table schema.
+
+ """
+
+ logger.debug("IOTLABPOSTGRES createtable \
+ IotlabBase.metadata.sorted_tables %s \r\n engine %s"
+ % (IotlabBase.metadata.sorted_tables, self.iotlab_engine))
+ IotlabBase.metadata.create_all(self.iotlab_engine)
+ return
--- /dev/null
+"""
+This file defines the IotlabSlices class by which all the slice checkings
+upon lease creation are done.
+"""
+from sfa.util.xrn import get_authority, urn_to_hrn
+from sfa.util.sfalogging import logger
+
+MAXINT = 2L**31-1
+
+
+class IotlabSlices:
+ """
+ This class is responsible for checking the slice when creating a
+ lease or a sliver. Those checks include verifying that the user is valid,
+ that the slice is known from the testbed or from our peers, that the list
+ of nodes involved has not changed (in this case the lease is modified
+ accordingly).
+ """
+ rspec_to_slice_tag = {'max_rate': 'net_max_rate'}
+
+ def __init__(self, driver):
+ """
+ Get the reference to the driver here.
+ """
+ self.driver = driver
+
+ def get_peer(self, xrn):
+ """
+ Finds the authority of a resource based on its xrn.
+ If the authority is Iotlab (local) return None,
+ Otherwise, look up in the DB if Iotlab is federated with this site
+ authority and returns its DB record if it is the case.
+
+ :param xrn: resource's xrn
+ :type xrn: string
+ :returns: peer record
+ :rtype: dict
+
+ """
+ hrn, hrn_type = urn_to_hrn(xrn)
+ #Does this slice belong to a local site or a peer iotlab site?
+ peer = None
+
+ # get this slice's authority (site)
+ slice_authority = get_authority(hrn)
+ #Iotlab stuff
+ #This slice belongs to the current site
+ if slice_authority == self.driver.iotlab_api.root_auth:
+ site_authority = slice_authority
+ return None
+
+ site_authority = get_authority(slice_authority).lower()
+ # get this site's authority (sfa root authority or sub authority)
+
+ logger.debug("IOTLABSLICES \t get_peer slice_authority %s \
+ site_authority %s hrn %s"
+ % (slice_authority, site_authority, hrn))
+
+ # check if we are already peered with this site_authority
+ #if so find the peer record
+ peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority)
+ for peer_record in peers:
+ if site_authority == peer_record.hrn:
+ peer = peer_record
+ logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer))
+ return peer
+
+ def get_sfa_peer(self, xrn):
+ """Returns the authority name for the xrn or None if the local site
+ is the authority.
+
+ :param xrn: the xrn of the resource we are looking the authority for.
+ :type xrn: string
+ :returns: the resources's authority name.
+ :rtype: string
+
+ """
+ hrn, hrn_type = urn_to_hrn(xrn)
+
+ # return the authority for this hrn or None if we are the authority
+ sfa_peer = None
+ slice_authority = get_authority(hrn)
+ site_authority = get_authority(slice_authority)
+
+ if site_authority != self.driver.hrn:
+ sfa_peer = site_authority
+
+ return sfa_peer
+
+ def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
+ """
+ Compare requested leases with the leases already scheduled/
+ running in OAR. If necessary, delete and recreate modified leases,
+ and delete no longer requested ones.
+
+ :param sfa_slice: sfa slice record
+ :param requested_jobs_dict: dictionary of requested leases
+ :param peer: sfa peer record
+
+ :type sfa_slice: dict
+ :type requested_jobs_dict: dict
+ :type peer: dict
+ :returns: leases list of dictionary
+ :rtype: list
+
+ """
+
+ logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s "
+ % (sfa_slice))
+ #First get the list of current leases from OAR
+ leases = self.driver.iotlab_api.GetLeases({'slice_hrn': sfa_slice['hrn']})
+ logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \
+ leases %s " % (requested_jobs_dict, leases))
+
+ current_nodes_reserved_by_start_time = {}
+ requested_nodes_by_start_time = {}
+ leases_by_start_time = {}
+ reschedule_jobs_dict = {}
+
+ #Create reduced dictionary with key start_time and value
+ # the list of nodes
+ #-for the leases already registered by OAR first
+ # then for the new leases requested by the user
+
+ #Leases already scheduled/running in OAR
+ for lease in leases:
+ current_nodes_reserved_by_start_time[lease['t_from']] = \
+ lease['reserved_nodes']
+ leases_by_start_time[lease['t_from']] = lease
+
+ #First remove job whose duration is too short
+ for job in requested_jobs_dict.values():
+ job['duration'] = \
+ str(int(job['duration']) \
+ * self.driver.iotlab_api.GetLeaseGranularity())
+ if job['duration'] < self.driver.iotlab_api.GetLeaseGranularity():
+ del requested_jobs_dict[job['start_time']]
+
+ #Requested jobs
+ for start_time in requested_jobs_dict:
+ requested_nodes_by_start_time[int(start_time)] = \
+ requested_jobs_dict[start_time]['hostname']
+ #Check if there is any difference between the leases already
+ #registered in OAR and the requested jobs.
+ #Difference could be:
+ #-Lease deleted in the requested jobs
+ #-Added/removed nodes
+ #-Newly added lease
+
+ logger.debug("IOTLABSLICES verify_slice_leases \
+ requested_nodes_by_start_time %s \
+ "% (requested_nodes_by_start_time))
+ #Find all deleted leases
+ start_time_list = \
+ list(set(leases_by_start_time.keys()).\
+ difference(requested_nodes_by_start_time.keys()))
+ deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
+ for start_time in start_time_list]
+
+
+ #Find added or removed nodes in exisiting leases
+ for start_time in requested_nodes_by_start_time:
+ logger.debug("IOTLABSLICES verify_slice_leases start_time %s \
+ "%( start_time))
+ if start_time in current_nodes_reserved_by_start_time:
+
+ if requested_nodes_by_start_time[start_time] == \
+ current_nodes_reserved_by_start_time[start_time]:
+ continue
+
+ else:
+ update_node_set = \
+ set(requested_nodes_by_start_time[start_time])
+ added_nodes = \
+ update_node_set.difference(\
+ current_nodes_reserved_by_start_time[start_time])
+ shared_nodes = \
+ update_node_set.intersection(\
+ current_nodes_reserved_by_start_time[start_time])
+ old_nodes_set = \
+ set(\
+ current_nodes_reserved_by_start_time[start_time])
+ removed_nodes = \
+ old_nodes_set.difference(\
+ requested_nodes_by_start_time[start_time])
+ logger.debug("IOTLABSLICES verify_slice_leases \
+ shared_nodes %s added_nodes %s removed_nodes %s"\
+ %(shared_nodes, added_nodes,removed_nodes ))
+ #If the lease is modified, delete it before
+ #creating it again.
+ #Add the deleted lease job id in the list
+ #WARNING :rescheduling does not work if there is already
+ # 2 running/scheduled jobs because deleting a job
+ #takes time SA 18/10/2012
+ if added_nodes or removed_nodes:
+ deleted_leases.append(\
+ leases_by_start_time[start_time]['lease_id'])
+ #Reschedule the job
+ if added_nodes or shared_nodes:
+ reschedule_jobs_dict[str(start_time)] = \
+ requested_jobs_dict[str(start_time)]
+
+ else:
+ #New lease
+
+ job = requested_jobs_dict[str(start_time)]
+ logger.debug("IOTLABSLICES \
+ NEWLEASE slice %s job %s"
+ % (sfa_slice, job))
+ self.driver.iotlab_api.AddLeases(
+ job['hostname'],
+ sfa_slice, int(job['start_time']),
+ int(job['duration']))
+
+ #Deleted leases are the ones with lease id not declared in the Rspec
+ if deleted_leases:
+ self.driver.iotlab_api.DeleteLeases(deleted_leases,
+ sfa_slice['user']['uid'])
+ logger.debug("IOTLABSLICES \
+ verify_slice_leases slice %s deleted_leases %s"
+ % (sfa_slice, deleted_leases))
+
+ if reschedule_jobs_dict:
+ for start_time in reschedule_jobs_dict:
+ job = reschedule_jobs_dict[start_time]
+ self.driver.iotlab_api.AddLeases(
+ job['hostname'],
+ sfa_slice, int(job['start_time']),
+ int(job['duration']))
+ return leases
+
+ def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
+ """Check for wanted and unwanted nodes in the slice.
+
+ Removes nodes and associated leases that the user does not want anymore
+ by deleteing the associated job in OAR (DeleteSliceFromNodes).
+ Returns the nodes' hostnames that are going to be in the slice.
+
+ :param sfa_slice: slice record. Must contain node_ids and list_node_ids.
+
+ :param requested_slivers: list of requested nodes' hostnames.
+ :param peer: unused so far.
+
+ :type sfa_slice: dict
+ :type requested_slivers: list
+ :type peer: string
+
+ :returns: list requested nodes hostnames
+ :rtype: list
+
+ .. warning:: UNUSED SQA 24/07/13
+ .. seealso:: DeleteSliceFromNodes
+ .. todo:: check what to do with the peer? Can not remove peer nodes from
+ slice here. Anyway, in this case, the peer should have gotten the
+ remove request too.
+
+ """
+ current_slivers = []
+ deleted_nodes = []
+
+ if 'node_ids' in sfa_slice:
+ nodes = self.driver.iotlab_api.GetNodes(
+ sfa_slice['list_node_ids'],
+ ['hostname'])
+ current_slivers = [node['hostname'] for node in nodes]
+
+ # remove nodes not in rspec
+ deleted_nodes = list(set(current_slivers).
+ difference(requested_slivers))
+
+ logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\
+ \r\n \r\n deleted_nodes %s"
+ % (sfa_slice, deleted_nodes))
+
+ if deleted_nodes:
+ #Delete the entire experience
+ self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice)
+ return nodes
+
+ def verify_slice(self, slice_hrn, slice_record, sfa_peer):
+ """Ensures slice record exists.
+
+ The slice record must exist either in Iotlab or in the other
+ federated testbed (sfa_peer). If the slice does not belong to Iotlab,
+ check if the user already exists in LDAP. In this case, adds the slice
+ to the sfa DB and associates its LDAP user.
+
+ :param slice_hrn: slice's name
+ :param slice_record: sfa record of the slice
+ :param sfa_peer: name of the peer authority if any.(not Iotlab).
+
+ :type slice_hrn: string
+ :type slice_record: dictionary
+ :type sfa_peer: string
+
+ .. seealso:: AddSlice
+
+
+ """
+
+ slicename = slice_hrn
+ # check if slice belongs to Iotlab
+ slices_list = self.driver.iotlab_api.GetSlices(
+ slice_filter=slicename, slice_filter_type='slice_hrn')
+
+ sfa_slice = None
+
+ if slices_list:
+ for sl in slices_list:
+
+ logger.debug("IOTLABSLICES \t verify_slice slicename %s \
+ slices_list %s sl %s \r slice_record %s"
+ % (slicename, slices_list, sl, slice_record))
+ sfa_slice = sl
+ sfa_slice.update(slice_record)
+
+ else:
+ #Search for user in ldap based on email SA 14/11/12
+ ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\
+ slice_record['user'])
+ logger.debug(" IOTLABSLICES \tverify_slice Oups \
+ slice_record %s sfa_peer %s ldap_user %s"
+ % (slice_record, sfa_peer, ldap_user))
+ #User already registered in ldap, meaning user should be in SFA db
+ #and hrn = sfa_auth+ uid
+ sfa_slice = {'hrn': slicename,
+ 'node_list': [],
+ 'authority': slice_record['authority'],
+ 'gid': slice_record['gid'],
+ 'slice_id': slice_record['record_id'],
+ 'reg-researchers': slice_record['reg-researchers'],
+ 'peer_authority': str(sfa_peer)
+ }
+
+ if ldap_user:
+ hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid']
+ user = self.driver.get_user_record(hrn)
+
+ logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s"
+ % (hrn, user))
+
+ # add the external slice to the local SFA iotlab DB
+ if sfa_slice:
+ self.driver.iotlab_api.AddSlice(sfa_slice, user)
+
+ logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK")
+ return sfa_slice
+
+
+ def verify_persons(self, slice_hrn, slice_record, users, options={}):
+ """Ensures the users in users list exist and are enabled in LDAP. Adds
+ person if needed.
+
+ Checking that a user exist is based on the user's email. If the user is
+ still not found in the LDAP, it means that the user comes from another
+ federated testbed. In this case an account has to be created in LDAP
+ so as to enable the user to use the testbed, since we trust the testbed
+ he comes from. This is done by calling AddPerson.
+
+ :param slice_hrn: slice name
+ :param slice_record: record of the slice_hrn
+ :param users: users is a record list. Records can either be
+ local records or users records from known and trusted federated
+ sites.If the user is from another site that iotlab doesn't trust yet,
+ then Resolve will raise an error before getting to create_sliver.
+
+ :type slice_hrn: string
+ :type slice_record: string
+ :type users: list
+
+ .. seealso:: AddPerson
+ .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13.
+
+
+ """
+ #TODO SA 21/08/12 verify_persons Needs review
+
+ logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \
+ \t slice_record %s\r\n users %s \t "
+ % (slice_hrn, slice_record, users))
+ users_by_id = {}
+
+ users_by_email = {}
+ #users_dict : dict whose keys can either be the user's hrn or its id.
+ #Values contains only id and hrn
+ users_dict = {}
+
+ #First create dicts by hrn and id for each user in the user record list:
+ for info in users:
+ if 'slice_record' in info:
+ slice_rec = info['slice_record']
+ if 'user' in slice_rec :
+ user = slice_rec['user']
+
+ if 'email' in user:
+ users_by_email[user['email']] = user
+ users_dict[user['email']] = user
+
+ logger.debug("IOTLABSLICES.PY \t verify_person \
+ users_dict %s \r\n user_by_email %s \r\n \
+ \tusers_by_id %s "
+ % (users_dict, users_by_email, users_by_id))
+
+ existing_user_ids = []
+ existing_user_emails = []
+ existing_users = []
+ # Check if user is in Iotlab LDAP using its hrn.
+ # Assuming Iotlab is centralised : one LDAP for all sites,
+ # user's record_id unknown from LDAP
+ # LDAP does not provide users id, therefore we rely on email to find the
+ # user in LDAP
+
+ if users_by_email:
+ #Construct the list of filters (list of dicts) for GetPersons
+ filter_user = [users_by_email[email] for email in users_by_email]
+ #Check user i in LDAP with GetPersons
+ #Needed because what if the user has been deleted in LDAP but
+ #is still in SFA?
+ existing_users = self.driver.iotlab_api.GetPersons(filter_user)
+ logger.debug(" \r\n IOTLABSLICES.PY \tverify_person filter_user \
+ %s existing_users %s "
+ % (filter_user, existing_users))
+ #User is in iotlab LDAP
+ if existing_users:
+ for user in existing_users:
+ users_dict[user['email']].update(user)
+ existing_user_emails.append(
+ users_dict[user['email']]['email'])
+
+
+ # User from another known trusted federated site. Check
+ # if a iotlab account matching the email has already been created.
+ else:
+ req = 'mail='
+ if isinstance(users, list):
+ req += users[0]['email']
+ else:
+ req += users['email']
+ ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req)
+
+ if ldap_reslt:
+ logger.debug(" IOTLABSLICES.PY \tverify_person users \
+ USER already in Iotlab \t ldap_reslt %s \
+ " % (ldap_reslt))
+ existing_users.append(ldap_reslt[1])
+
+ else:
+ #User not existing in LDAP
+ logger.debug("IOTLABSLICES.PY \tverify_person users \
+ not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
+ ldap_reslt %s " % (users, ldap_reslt))
+
+ requested_user_emails = users_by_email.keys()
+ requested_user_hrns = \
+ [users_by_email[user]['hrn'] for user in users_by_email]
+ logger.debug("IOTLABSLICES.PY \tverify_person \
+ users_by_email %s " % (users_by_email))
+
+ #Check that the user of the slice in the slice record
+ #matches one of the existing users
+ try:
+ if slice_record['PI'][0] in requested_user_hrns:
+ logger.debug(" IOTLABSLICES \tverify_person ['PI']\
+ slice_record %s" % (slice_record))
+
+ except KeyError:
+ pass
+
+ # users to be added, removed or updated
+ #One user in one iotlab slice : there should be no need
+ #to remove/ add any user from/to a slice.
+ #However a user from SFA which is not registered in Iotlab yet
+ #should be added to the LDAP.
+ added_user_emails = set(requested_user_emails).\
+ difference(set(existing_user_emails))
+
+
+ #self.verify_keys(existing_slice_users, updated_users_list, \
+ #peer, append)
+
+ added_persons = []
+ # add new users
+ #requested_user_email is in existing_user_emails
+ if len(added_user_emails) == 0:
+ slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
+ logger.debug(" IOTLABSLICES \tverify_person QUICK DIRTY %s"
+ % (slice_record))
+
+ for added_user_email in added_user_emails:
+ added_user = users_dict[added_user_email]
+ logger.debug(" IOTLABSLICES \r\n \r\n \t verify_person \
+ added_user %s" % (added_user))
+ person = {}
+ person['peer_person_id'] = None
+ k_list = ['first_name', 'last_name', 'person_id']
+ for k in k_list:
+ if k in added_user:
+ person[k] = added_user[k]
+
+ person['pkey'] = added_user['keys'][0]
+ person['mail'] = added_user['email']
+ person['email'] = added_user['email']
+ person['key_ids'] = added_user.get('key_ids', [])
+
+ ret = self.driver.iotlab_api.AddPerson(person)
+ if 'uid' in ret:
+ # meaning bool is True and the AddPerson was successful
+ person['uid'] = ret['uid']
+ slice_record['login'] = person['uid']
+ else:
+ # error message in ret
+ logger.debug(" IOTLABSLICES ret message %s" %(ret))
+
+ logger.debug(" IOTLABSLICES \r\n \r\n \t THE SECOND verify_person\
+ person %s" % (person))
+ #Update slice_Record with the id now known to LDAP
+
+
+ added_persons.append(person)
+ return added_persons
+
+
+ def verify_keys(self, persons, users, peer, options={}):
+ """
+ .. warning:: unused
+ """
+ # existing keys
+ key_ids = []
+ for person in persons:
+ key_ids.extend(person['key_ids'])
+ keylist = self.driver.iotlab_api.GetKeys(key_ids, ['key_id', 'key'])
+
+ keydict = {}
+ for key in keylist:
+ keydict[key['key']] = key['key_id']
+ existing_keys = keydict.keys()
+
+ persondict = {}
+ for person in persons:
+ persondict[person['email']] = person
+
+ # add new keys
+ requested_keys = []
+ updated_persons = []
+ users_by_key_string = {}
+ for user in users:
+ user_keys = user.get('keys', [])
+ updated_persons.append(user)
+ for key_string in user_keys:
+ users_by_key_string[key_string] = user
+ requested_keys.append(key_string)
+ if key_string not in existing_keys:
+ key = {'key': key_string, 'key_type': 'ssh'}
+ #try:
+ ##if peer:
+ #person = persondict[user['email']]
+ #self.driver.iotlab_api.UnBindObjectFromPeer(
+ # 'person',person['person_id'],
+ # peer['shortname'])
+ ret = self.driver.iotlab_api.AddPersonKey(
+ user['email'], key)
+ #if peer:
+ #key_index = user_keys.index(key['key'])
+ #remote_key_id = user['key_ids'][key_index]
+ #self.driver.iotlab_api.BindObjectToPeer('key', \
+ #key['key_id'], peer['shortname'], \
+ #remote_key_id)
+
+ #finally:
+ #if peer:
+ #self.driver.iotlab_api.BindObjectToPeer('person', \
+ #person['person_id'], peer['shortname'], \
+ #user['person_id'])
+
+ # remove old keys (only if we are not appending)
+ append = options.get('append', True)
+ if append is False:
+ removed_keys = set(existing_keys).difference(requested_keys)
+ for key in removed_keys:
+ #if peer:
+ #self.driver.iotlab_api.UnBindObjectFromPeer('key', \
+ #key, peer['shortname'])
+
+ user = users_by_key_string[key]
+ self.driver.iotlab_api.DeleteKey(user, key)
+
+ return
git://git.onelab.eu / sfa.git / commitdiff