From 8f8fa64f4144bfc02a925dd08e23467342528e58 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Tue, 15 Oct 2013 16:02:58 +0200 Subject: [PATCH] Create new cortexlab forlder, for the Cortex-lab testbed, based on the iotlab driver. So far, all the files in the cortexlab are copies of iotlab's, with a very few modifications. First draft. --- sfa/cortexlab/LDAPapi.py | 1024 +++++++++++++++++++ sfa/cortexlab/__init__.py | 0 sfa/cortexlab/cortexlabaggregate.py | 462 +++++++++ sfa/cortexlab/cortexlabapi.py | 1455 +++++++++++++++++++++++++++ sfa/cortexlab/cortexlabdriver.py | 798 +++++++++++++++ sfa/cortexlab/cortexlabpostgres.py | 264 +++++ sfa/cortexlab/cortexlabslices.py | 587 +++++++++++ 7 files changed, 4590 insertions(+) create mode 100644 sfa/cortexlab/LDAPapi.py create mode 100644 sfa/cortexlab/__init__.py create mode 100644 sfa/cortexlab/cortexlabaggregate.py create mode 100644 sfa/cortexlab/cortexlabapi.py create mode 100644 sfa/cortexlab/cortexlabdriver.py create mode 100644 sfa/cortexlab/cortexlabpostgres.py create mode 100644 sfa/cortexlab/cortexlabslices.py diff --git a/sfa/cortexlab/LDAPapi.py b/sfa/cortexlab/LDAPapi.py new file mode 100644 index 00000000..15067ac5 --- /dev/null +++ b/sfa/cortexlab/LDAPapi.py @@ -0,0 +1,1024 @@ +""" +This API is adapted for OpenLDAP. The file contains all LDAP classes and methods +needed to: +- Load the LDAP connection configuration file (login, address..) with LdapConfig +- Connect to LDAP with ldap_co +- Create a unique LDAP login and password for a user based on his email or last +name and first name with LoginPassword. +- Manage entries in LDAP using SFA records with LDAPapi (Search, Add, Delete, +Modify) + +""" +import random +from passlib.hash import ldap_salted_sha1 as lssha + +from sfa.util.xrn import get_authority +from sfa.util.sfalogging import logger +from sfa.util.config import Config + +import ldap +import ldap.modlist as modlist + +import os.path + + +class LdapConfig(): + """ + Ldap configuration class loads the configuration file and sets the + ldap IP address, password, people dn, web dn, group dn. All these settings + were defined in a separate file ldap_config.py to avoid sharing them in + the SFA git as it contains sensible information. + + """ + def __init__(self, config_file='/etc/sfa/ldap_config.py'): + """Loads configuration from file /etc/sfa/ldap_config.py and set the + parameters for connection to LDAP. + + """ + + try: + execfile(config_file, self.__dict__) + + self.config_file = config_file + # path to configuration data + self.config_path = os.path.dirname(config_file) + except IOError: + raise IOError, "Could not find or load the configuration file: %s" \ + % config_file + + +class ldap_co: + """ Set admin login and server configuration variables.""" + + def __init__(self): + """Fetch LdapConfig attributes (Ldap server connection parameters and + defines port , version and subtree scope. + + """ + #Iotlab PROD LDAP parameters + self.ldapserv = None + ldap_config = LdapConfig() + self.config = ldap_config + self.ldapHost = ldap_config.LDAP_IP_ADDRESS + self.ldapPeopleDN = ldap_config.LDAP_PEOPLE_DN + self.ldapGroupDN = ldap_config.LDAP_GROUP_DN + self.ldapAdminDN = ldap_config.LDAP_WEB_DN + self.ldapAdminPassword = ldap_config.LDAP_WEB_PASSWORD + self.ldapPort = ldap.PORT + self.ldapVersion = ldap.VERSION3 + self.ldapSearchScope = ldap.SCOPE_SUBTREE + + def connect(self, bind=True): + """Enables connection to the LDAP server. + + :param bind: Set the bind parameter to True if a bind is needed + (for add/modify/delete operations). Set to False otherwise. + :type bind: boolean + :returns: dictionary with status of the connection. True if Successful, + False if not and in this case the error + message( {'bool', 'message'} ). + :rtype: dict + + """ + try: + self.ldapserv = ldap.open(self.ldapHost) + except ldap.LDAPError, error: + return {'bool': False, 'message': error} + + # Bind with authentification + if(bind): + return self.bind() + + else: + return {'bool': True} + + def bind(self): + """ Binding method. + + :returns: dictionary with the bind status. True if Successful, + False if not and in this case the error message({'bool','message'}) + :rtype: dict + + """ + try: + # Opens a connection after a call to ldap.open in connect: + self.ldapserv = ldap.initialize("ldap://" + self.ldapHost) + + # Bind/authenticate with a user with apropriate + #rights to add objects + self.ldapserv.simple_bind_s(self.ldapAdminDN, + self.ldapAdminPassword) + + except ldap.LDAPError, error: + return {'bool': False, 'message': error} + + return {'bool': True} + + def close(self): + """Close the LDAP connection. + + Can throw an exception if the unbinding fails. + + :returns: dictionary with the bind status if the unbinding failed and + in this case the dict contains an error message. The dictionary keys + are : ({'bool','message'}) + :rtype: dict or None + + """ + try: + self.ldapserv.unbind_s() + except ldap.LDAPError, error: + return {'bool': False, 'message': error} + + +class LoginPassword(): + """ + + Class to handle login and password generation, using custom login generation + algorithm. + + """ + def __init__(self): + """ + + Sets password and login maximum length, and defines the characters that + can be found in a random generated password. + + """ + self.login_max_length = 8 + self.length_password = 8 + self.chars_password = ['!', '$', '(',')', '*', '+', ',', '-', '.', + '0', '1', '2', '3', '4', '5', '6', '7', '8', + '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', + 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', + 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '\''] + + @staticmethod + def clean_user_names(record): + """ + + Removes special characters such as '-', '_' , '[', ']' and ' ' from the + first name and last name. + + :param record: user's record + :type record: dict + :returns: lower_first_name and lower_last_name if they were found + in the user's record. Return None, none otherwise. + :rtype: string, string or None, None. + + """ + if 'first_name' in record and 'last_name' in record: + #Remove all special characters from first_name/last name + lower_first_name = record['first_name'].replace('-', '')\ + .replace('_', '').replace('[', '')\ + .replace(']', '').replace(' ', '')\ + .lower() + lower_last_name = record['last_name'].replace('-', '')\ + .replace('_', '').replace('[', '')\ + .replace(']', '').replace(' ', '')\ + .lower() + return lower_first_name, lower_last_name + else: + return None, None + + @staticmethod + def extract_name_from_email(record): + """ + + When there is no valid first name and last name in the record, + the email is used to generate the login. Here, we assume the email + is firstname.lastname@something.smthg. The first name and last names + are extracted from the email, special charcaters are removed and + they are changed into lower case. + + :param record: user's data + :type record: dict + :returns: the first name and last name taken from the user's email. + lower_first_name, lower_last_name. + :rtype: string, string + + """ + + email = record['email'] + email = email.split('@')[0].lower() + lower_first_name = None + lower_last_name = None + #Assume there is first name and last name in email + #if there is a separator + separator_list = ['.', '_', '-'] + for sep in separator_list: + if sep in email: + mail = email.split(sep) + lower_first_name = mail[0] + lower_last_name = mail[1] + break + + #Otherwise just take the part before the @ as the + #lower_first_name and lower_last_name + if lower_first_name is None: + lower_first_name = email + lower_last_name = email + + return lower_first_name, lower_last_name + + def get_user_firstname_lastname(self, record): + """ + + Get the user first name and last name from the information we have in + the record. + + :param record: user's information + :type record: dict + :returns: the user's first name and last name. + + .. seealso:: clean_user_names + .. seealso:: extract_name_from_email + + """ + lower_first_name, lower_last_name = self.clean_user_names(record) + + #No first name and last name check email + if lower_first_name is None and lower_last_name is None: + + lower_first_name, lower_last_name = \ + self.extract_name_from_email(record) + + return lower_first_name, lower_last_name + + def choose_sets_chars_for_login(self, lower_first_name, lower_last_name): + """ + + Algorithm to select sets of characters from the first name and last + name, depending on the lenght of the last name and the maximum login + length which in our case is set to 8 characters. + + :param lower_first_name: user's first name in lower case. + :param lower_last_name: usr's last name in lower case. + :returns: user's login + :rtype: string + + """ + length_last_name = len(lower_last_name) + self.login_max_length = 8 + + #Try generating a unique login based on first name and last name + + if length_last_name >= self.login_max_length: + login = lower_last_name[0:self.login_max_length] + index = 0 + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name >= 4: + login = lower_last_name + index = 0 + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name == 3: + login = lower_first_name[0:1] + lower_last_name + index = 1 + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name == 2: + if len(lower_first_name) >= 2: + login = lower_first_name[0:2] + lower_last_name + index = 2 + logger.debug("login : %s index : %s" % (login, index)) + else: + logger.error("LoginException : \ + Generation login error with \ + minimum four characters") + + else: + logger.error("LDAP LdapGenerateUniqueLogin failed : \ + impossible to generate unique login for %s %s" + % (lower_first_name, lower_last_name)) + return index, login + + def generate_password(self): + """ + + Generate a password upon adding a new user in LDAP Directory + (8 characters length). The generated password is composed of characters + from the chars_password list. + + :returns: the randomly generated password + :rtype: string + + """ + password = str() + + length = len(self.chars_password) + for index in range(self.length_password): + char_index = random.randint(0, length - 1) + password += self.chars_password[char_index] + + return password + + @staticmethod + def encrypt_password(password): + """ + + Use passlib library to make a RFC2307 LDAP encrypted password salt size + is 8, use sha-1 algorithm. + + :param password: password not encrypted. + :type password: string + :returns: Returns encrypted password. + :rtype: string + + """ + #Keep consistency with Java Iotlab's LDAP API + #RFC2307SSHAPasswordEncryptor so set the salt size to 8 bytes + return lssha.encrypt(password, salt_size=8) + + +class LDAPapi: + """Defines functions to insert and search entries in the LDAP. + + .. note:: class supposes the unix schema is used + + """ + def __init__(self): + logger.setLevelDebug() + + #SFA related config + + config = Config() + self.login_pwd = LoginPassword() + self.authname = config.SFA_REGISTRY_ROOT_AUTH + self.conn = ldap_co() + self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS + self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN + self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER + self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH + self.baseDN = self.conn.ldapPeopleDN + self.ldapShell = '/bin/bash' + + + def LdapGenerateUniqueLogin(self, record): + """ + + Generate login for adding a new user in LDAP Directory + (four characters minimum length). Get proper last name and + first name so that the user's login can be generated. + + :param record: Record must contain first_name and last_name. + :type record: dict + :returns: the generated login for the user described with record if the + login generation is successful, None if it fails. + :rtype: string or None + + """ + #For compatibility with other ldap func + if 'mail' in record and 'email' not in record: + record['email'] = record['mail'] + + lower_first_name, lower_last_name = \ + self.login_pwd.get_user_firstname_lastname(record) + + index, login = self.login_pwd.choose_sets_chars_for_login( + lower_first_name, lower_last_name) + + login_filter = '(uid=' + login + ')' + get_attrs = ['uid'] + try: + #Check if login already in use + + while (len(self.LdapSearch(login_filter, get_attrs)) is not 0): + + index += 1 + if index >= 9: + logger.error("LoginException : Generation login error \ + with minimum four characters") + else: + try: + login = \ + lower_first_name[0:index] + \ + lower_last_name[0: + self.login_pwd.login_max_length + - index] + login_filter = '(uid=' + login + ')' + except KeyError: + print "lower_first_name - lower_last_name too short" + + logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s" + % (login)) + return login + + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error)) + return None + + def find_max_uidNumber(self): + """Find the LDAP max uidNumber (POSIX uid attribute). + + Used when adding a new user in LDAP Directory + + :returns: max uidNumber + 1 + :rtype: string + + """ + #First, get all the users in the LDAP + get_attrs = "(uidNumber=*)" + login_filter = ['uidNumber'] + + result_data = self.LdapSearch(get_attrs, login_filter) + #It there is no user in LDAP yet, First LDAP user + if result_data == []: + max_uidnumber = self.ldapUserUidNumberMin + #Otherwise, get the highest uidNumber + else: + uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data] + logger.debug("LDAPapi.py \tfind_max_uidNumber \ + uidNumberList %s " % (uidNumberList)) + max_uidnumber = max(uidNumberList) + 1 + + return str(max_uidnumber) + + + def get_ssh_pkey(self, record): + """TODO ; Get ssh public key from sfa record + To be filled by N. Turro ? or using GID pl way? + + """ + return 'A REMPLIR ' + + @staticmethod + #TODO Handle OR filtering in the ldap query when + #dealing with a list of records instead of doing a for loop in GetPersons + def make_ldap_filters_from_record(record=None): + """Helper function to make LDAP filter requests out of SFA records. + + :param record: user's sfa record. Should contain first_name,last_name, + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. + :type record: dict + :returns: LDAP request + :rtype: string + + """ + req_ldap = '' + req_ldapdict = {} + if record : + if 'first_name' in record and 'last_name' in record: + if record['first_name'] != record['last_name']: + req_ldapdict['cn'] = str(record['first_name'])+" "\ + + str(record['last_name']) + if 'email' in record: + req_ldapdict['mail'] = record['email'] + if 'mail' in record: + req_ldapdict['mail'] = record['mail'] + if 'enabled' in record: + if record['enabled'] is True: + req_ldapdict['shadowExpire'] = '-1' + else: + req_ldapdict['shadowExpire'] = '0' + + #Hrn should not be part of the filter because the hrn + #presented by a certificate of a SFA user not imported in + #Iotlab does not include the iotlab login in it + #Plus, the SFA user may already have an account with iotlab + #using another login. + + logger.debug("\r\n \t LDAP.PY make_ldap_filters_from_record \ + record %s req_ldapdict %s" + % (record, req_ldapdict)) + + for k in req_ldapdict: + req_ldap += '(' + str(k) + '=' + str(req_ldapdict[k]) + ')' + if len(req_ldapdict.keys()) >1 : + req_ldap = req_ldap[:0]+"(&"+req_ldap[0:] + size = len(req_ldap) + req_ldap = req_ldap[:(size-1)] + ')' + req_ldap[(size-1):] + else: + req_ldap = "(cn=*)" + + return req_ldap + + def make_ldap_attributes_from_record(self, record): + """ + + When adding a new user to Iotlab's LDAP, creates an attributes + dictionnary from the SFA record understandable by LDAP. Generates the + user's LDAP login.User is automatically validated (account enabled) + and described as a SFA USER FROM OUTSIDE IOTLAB. + + :param record: must contain the following keys and values: + first_name, last_name, mail, pkey (ssh key). + :type record: dict + :returns: dictionary of attributes using LDAP data structure model. + :rtype: dict + + """ + + attrs = {} + attrs['objectClass'] = ["top", "person", "inetOrgPerson", + "organizationalPerson", "posixAccount", + "shadowAccount", "systemQuotas", + "ldapPublicKey"] + + attrs['uid'] = self.LdapGenerateUniqueLogin(record) + try: + attrs['givenName'] = str(record['first_name']).lower().capitalize() + attrs['sn'] = str(record['last_name']).lower().capitalize() + attrs['cn'] = attrs['givenName'] + ' ' + attrs['sn'] + attrs['gecos'] = attrs['givenName'] + ' ' + attrs['sn'] + + except KeyError: + attrs['givenName'] = attrs['uid'] + attrs['sn'] = attrs['uid'] + attrs['cn'] = attrs['uid'] + attrs['gecos'] = attrs['uid'] + + attrs['quota'] = self.ldapUserQuotaNFS + attrs['homeDirectory'] = self.ldapUserHomePath + attrs['uid'] + attrs['loginShell'] = self.ldapShell + attrs['gidNumber'] = self.ldapUserGidNumber + attrs['uidNumber'] = self.find_max_uidNumber() + attrs['mail'] = record['mail'].lower() + try: + attrs['sshPublicKey'] = record['pkey'] + except KeyError: + attrs['sshPublicKey'] = self.get_ssh_pkey(record) + + + #Password is automatically generated because SFA user don't go + #through the Iotlab website used to register new users, + #There is no place in SFA where users can enter such information + #yet. + #If the user wants to set his own password , he must go to the Iotlab + #website. + password = self.login_pwd.generate_password() + attrs['userPassword'] = self.login_pwd.encrypt_password(password) + + #Account automatically validated (no mail request to admins) + #Set to 0 to disable the account, -1 to enable it, + attrs['shadowExpire'] = '-1' + + #Motivation field in Iotlab + attrs['description'] = 'SFA USER FROM OUTSIDE SENSLAB' + + attrs['ou'] = 'SFA' #Optional: organizational unit + #No info about those here: + attrs['l'] = 'To be defined'#Optional: Locality. + attrs['st'] = 'To be defined' #Optional: state or province (country). + + return attrs + + + + def LdapAddUser(self, record) : + """Add SFA user to LDAP if it is not in LDAP yet. + + :param record: dictionnary with the user's data. + :returns: a dictionary with the status (Fail= False, Success= True) + and the uid of the newly added user if successful, or the error + message it is not. Dict has keys bool and message in case of + failure, and bool uid in case of success. + :rtype: dict + + .. seealso:: make_ldap_filters_from_record + + """ + logger.debug(" \r\n \t LDAP LdapAddUser \r\n\r\n ================\r\n ") + user_ldap_attrs = self.make_ldap_attributes_from_record(record) + + #Check if user already in LDAP wih email, first name and last name + filter_by = self.make_ldap_filters_from_record(user_ldap_attrs) + user_exist = self.LdapSearch(filter_by) + if user_exist: + logger.warning(" \r\n \t LDAP LdapAddUser user %s %s \ + already exists" % (user_ldap_attrs['sn'], + user_ldap_attrs['mail'])) + return {'bool': False} + + #Bind to the server + result = self.conn.connect() + + if(result['bool']): + + # A dict to help build the "body" of the object + logger.debug(" \r\n \t LDAP LdapAddUser attrs %s " + % user_ldap_attrs) + + # The dn of our new entry/object + dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN + + try: + ldif = modlist.addModlist(user_ldap_attrs) + logger.debug("LDAPapi.py add attrs %s \r\n ldif %s" + % (user_ldap_attrs, ldif)) + self.conn.ldapserv.add_s(dn, ldif) + + logger.info("Adding user %s login %s in LDAP" + % (user_ldap_attrs['cn'], user_ldap_attrs['uid'])) + except ldap.LDAPError, error: + logger.log_exc("LDAP Add Error %s" % error) + return {'bool': False, 'message': error} + + self.conn.close() + return {'bool': True, 'uid': user_ldap_attrs['uid']} + else: + return result + + def LdapDelete(self, person_dn): + """Deletes a person in LDAP. Uses the dn of the user. + + :param person_dn: user's ldap dn. + :type person_dn: string + :returns: dictionary with bool True if successful, bool False + and the error if not. + :rtype: dict + + """ + #Connect and bind + result = self.conn.connect() + if(result['bool']): + try: + self.conn.ldapserv.delete_s(person_dn) + self.conn.close() + return {'bool': True} + + except ldap.LDAPError, error: + logger.log_exc("LDAP Delete Error %s" % error) + return {'bool': False, 'message': error} + + def LdapDeleteUser(self, record_filter): + """Deletes a SFA person in LDAP, based on the user's hrn. + + :param record_filter: Filter to find the user to be deleted. Must + contain at least the user's email. + :type record_filter: dict + :returns: dict with bool True if successful, bool False and error + message otherwise. + :rtype: dict + + .. seealso:: LdapFindUser docstring for more info on record filter. + .. seealso:: LdapDelete for user deletion + + """ + #Find uid of the person + person = self.LdapFindUser(record_filter, []) + logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s" + % (record_filter, person)) + + if person: + dn = 'uid=' + person['uid'] + "," + self.baseDN + else: + return {'bool': False} + + result = self.LdapDelete(dn) + return result + + def LdapModify(self, dn, old_attributes_dict, new_attributes_dict): + """ Modifies a LDAP entry, replaces user's old attributes with + the new ones given. + + :param dn: user's absolute name in the LDAP hierarchy. + :param old_attributes_dict: old user's attributes. Keys must match + the ones used in the LDAP model. + :param new_attributes_dict: new user's attributes. Keys must match + the ones used in the LDAP model. + :type dn: string + :type old_attributes_dict: dict + :type new_attributes_dict: dict + :returns: dict bool True if Successful, bool False if not. + :rtype: dict + + """ + + ldif = modlist.modifyModlist(old_attributes_dict, new_attributes_dict) + # Connect and bind/authenticate + result = self.conn.connect() + if (result['bool']): + try: + self.conn.ldapserv.modify_s(dn, ldif) + self.conn.close() + return {'bool': True} + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapModify Error %s" % error) + return {'bool': False} + + + def LdapModifyUser(self, user_record, new_attributes_dict): + """ + + Gets the record from one user based on the user sfa recordand changes + the attributes according to the specified new_attributes. Do not use + this if we need to modify the uid. Use a ModRDN operation instead + ( modify relative DN ). + + :param user_record: sfa user record. + :param new_attributes_dict: new user attributes, keys must be the + same as the LDAP model. + :type user_record: dict + :type new_attributes_dict: dict + :returns: bool True if successful, bool False if not. + :rtype: dict + + .. seealso:: make_ldap_filters_from_record for info on what is mandatory + in the user_record. + .. seealso:: make_ldap_attributes_from_record for the LDAP objectclass. + + """ + if user_record is None: + logger.error("LDAP \t LdapModifyUser Need user record ") + return {'bool': False} + + #Get all the attributes of the user_uid_login + #person = self.LdapFindUser(record_filter,[]) + req_ldap = self.make_ldap_filters_from_record(user_record) + person_list = self.LdapSearch(req_ldap, []) + logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s" + % (person_list)) + + if person_list and len(person_list) > 1: + logger.error("LDAP \t LdapModifyUser Too many users returned") + return {'bool': False} + if person_list is None: + logger.error("LDAP \t LdapModifyUser User %s doesn't exist " + % (user_record)) + return {'bool': False} + + # The dn of our existing entry/object + #One result only from ldapSearch + person = person_list[0][1] + dn = 'uid=' + person['uid'][0] + "," + self.baseDN + + if new_attributes_dict: + old = {} + for k in new_attributes_dict: + if k not in person: + old[k] = '' + else: + old[k] = person[k] + logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s" + % (new_attributes_dict)) + result = self.LdapModify(dn, old, new_attributes_dict) + return result + else: + logger.error("LDAP \t LdapModifyUser No new attributes given. ") + return {'bool': False} + + + def LdapMarkUserAsDeleted(self, record): + """ + + Sets shadowExpire to 0, disabling the user in LDAP. Calls LdapModifyUser + to change the shadowExpire of the user. + + :param record: the record of the user who has to be disabled. + Should contain first_name,last_name, email or mail, and if the + record is enabled or not. If the dict record does not have all of + these, must at least contain the user's email. + :type record: dict + :returns: {bool: True} if successful or {bool: False} if not + :rtype: dict + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record + """ + + new_attrs = {} + #Disable account + new_attrs['shadowExpire'] = '0' + logger.debug(" LDAPapi.py \t LdapMarkUserAsDeleted ") + ret = self.LdapModifyUser(record, new_attrs) + return ret + + def LdapResetPassword(self, record): + """Resets password for the user whose record is the parameter and + changes the corresponding entry in the LDAP. + + :param record: user's sfa record whose Ldap password must be reset. + Should contain first_name,last_name, + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. + :type record: dict + :returns: return value of LdapModifyUser. True if successful, False + otherwise. + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record + + """ + password = self.login_pwd.generate_password() + attrs = {} + attrs['userPassword'] = self.login_pwd.encrypt_password(password) + logger.debug("LDAP LdapResetPassword encrypt_password %s" + % (attrs['userPassword'])) + result = self.LdapModifyUser(record, attrs) + return result + + + def LdapSearch(self, req_ldap=None, expected_fields=None): + """ + Used to search directly in LDAP, by using ldap filters and return + fields. When req_ldap is None, returns all the entries in the LDAP. + + :param req_ldap: ldap style request, with appropriate filters, + example: (cn=*). + :param expected_fields: Fields in the user ldap entry that has to be + returned. If None is provided, will return 'mail', 'givenName', + 'sn', 'uid', 'sshPublicKey', 'shadowExpire'. + :type req_ldap: string + :type expected_fields: list + + .. seealso:: make_ldap_filters_from_record for req_ldap format. + + """ + result = self.conn.connect(bind=False) + if (result['bool']): + + return_fields_list = [] + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey', 'shadowExpire'] + else: + return_fields_list = expected_fields + #No specifc request specified, get the whole LDAP + if req_ldap is None: + req_ldap = '(cn=*)' + + logger.debug("LDAP.PY \t LdapSearch req_ldap %s \ + return_fields_list %s" \ + %(req_ldap, return_fields_list)) + + try: + msg_id = self.conn.ldapserv.search( + self.baseDN, ldap.SCOPE_SUBTREE, + req_ldap, return_fields_list) + #Get all the results matching the search from ldap in one + #shot (1 value) + result_type, result_data = \ + self.conn.ldapserv.result(msg_id, 1) + + self.conn.close() + + logger.debug("LDAP.PY \t LdapSearch result_data %s" + % (result_data)) + + return result_data + + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapSearch Error %s" % error) + return [] + + else: + logger.error("LDAP.PY \t Connection Failed") + return + + def _process_ldap_info_for_all_users(self, result_data): + """Process the data of all enabled users in LDAP. + + :param result_data: Contains information of all enabled users in LDAP + and is coming from LdapSearch. + :param result_data: list + + .. seealso:: LdapSearch + + """ + results = [] + logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s " + % (result_data)) + for ldapentry in result_data: + logger.debug(" LDAP.py _process_ldap_info_for_all_users \ + ldapentry name : %s " % (ldapentry[1]['uid'][0])) + tmpname = ldapentry[1]['uid'][0] + hrn = self.authname + "." + tmpname + + tmpemail = ldapentry[1]['mail'][0] + if ldapentry[1]['mail'][0] == "unknown": + tmpemail = None + + try: + results.append({ + 'type': 'user', + 'pkey': ldapentry[1]['sshPublicKey'][0], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname , + 'email':tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry[1]['givenName'][0], + 'last_name': ldapentry[1]['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': self.authname, + 'peer_authority': '', + 'pointer': -1, + 'hrn': hrn, + }) + except KeyError, error: + logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s" + % (error)) + return + + return results + + def _process_ldap_info_for_one_user(self, record, result_data): + """ + + Put the user's ldap data into shape. Only deals with one user + record and one user data from ldap. + + :param record: user record + :param result_data: Raw ldap data coming from LdapSearch + :returns: user's data dict with 'type','pkey','uid', 'email', + 'first_name' 'last_name''serial''authority''peer_authority' + 'pointer''hrn' + :type record: dict + :type result_data: list + :rtype :dict + + """ + #One entry only in the ldap data because we used a filter + #to find one user only + ldapentry = result_data[0][1] + logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" % (ldapentry)) + tmpname = ldapentry['uid'][0] + + tmpemail = ldapentry['mail'][0] + if ldapentry['mail'][0] == "unknown": + tmpemail = None + + parent_hrn = None + peer_authority = None + if 'hrn' in record: + hrn = record['hrn'] + parent_hrn = get_authority(hrn) + if parent_hrn != self.authname: + peer_authority = parent_hrn + #In case the user was not imported from Iotlab LDAP + #but from another federated site, has an account in + #iotlab but currently using his hrn from federated site + #then the login is different from the one found in its hrn + if tmpname != hrn.split('.')[1]: + hrn = None + else: + hrn = None + + results = { + 'type': 'user', + 'pkey': ldapentry['sshPublicKey'], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname, + 'email': tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry['givenName'][0], + 'last_name': ldapentry['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': parent_hrn, + 'peer_authority': peer_authority, + 'pointer': -1, + 'hrn': hrn, + } + return results + + def LdapFindUser(self, record=None, is_user_enabled=None, + expected_fields=None): + """ + + Search a SFA user with a hrn. User should be already registered + in Iotlab LDAP. + + :param record: sfa user's record. Should contain first_name,last_name, + email or mail. If no record is provided, returns all the users found + in LDAP. + :type record: dict + :param is_user_enabled: is the user's iotlab account already valid. + :type is_user_enabled: Boolean. + :returns: LDAP entries from ldap matching the filter provided. Returns + a single entry if one filter has been given and a list of + entries otherwise. + :rtype: dict or list + + """ + custom_record = {} + if is_user_enabled: + custom_record['enabled'] = is_user_enabled + if record: + custom_record.update(record) + + req_ldap = self.make_ldap_filters_from_record(custom_record) + return_fields_list = [] + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey'] + else: + return_fields_list = expected_fields + + result_data = self.LdapSearch(req_ldap, return_fields_list) + logger.debug("LDAP.PY \t LdapFindUser result_data %s" % (result_data)) + + if len(result_data) == 0: + return None + #Asked for a specific user + if record is not None: + results = self._process_ldap_info_for_one_user(record, result_data) + + else: + #Asked for all users in ldap + results = self._process_ldap_info_for_all_users(result_data) + return results \ No newline at end of file diff --git a/sfa/cortexlab/__init__.py b/sfa/cortexlab/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sfa/cortexlab/cortexlabaggregate.py b/sfa/cortexlab/cortexlabaggregate.py new file mode 100644 index 00000000..68cb1ec0 --- /dev/null +++ b/sfa/cortexlab/cortexlabaggregate.py @@ -0,0 +1,462 @@ +""" +File providing methods to generate valid RSpecs for the Iotlab testbed. +Contains methods to get information on slice, slivers, nodes and leases, +formatting them and turn it into a RSpec. +""" +from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority + +from sfa.rspecs.rspec import RSpec +#from sfa.rspecs.elements.location import Location +from sfa.rspecs.elements.hardware_type import HardwareType +from sfa.rspecs.elements.login import Login +from sfa.rspecs.elements.services import Services +from sfa.rspecs.elements.sliver import Sliver +from sfa.rspecs.elements.lease import Lease +from sfa.rspecs.elements.granularity import Granularity +from sfa.rspecs.version_manager import VersionManager + +from sfa.rspecs.elements.versions.cortexlabv1Node import IotlabPosition, \ + IotlabNode, IotlabLocation, IotlabMobility + +from sfa.util.sfalogging import logger +from sfa.util.xrn import Xrn + + +def cortexlab_xrn_to_hostname(xrn): + """Returns a node's hostname from its xrn. + :param xrn: The nodes xrn identifier. + :type xrn: Xrn (from sfa.util.xrn) + + :returns: node's hostname. + :rtype: string + + """ + return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf()) + + +def cortexlab_xrn_object(root_auth, hostname): + """Creates a valid xrn object from the node's hostname and the authority + of the SFA server. + + :param hostname: the node's hostname. + :param root_auth: the SFA root authority. + :type hostname: string + :type root_auth: string + + :returns: the cortexlab node's xrn + :rtype: Xrn + + """ + return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node') + + +class CortexlabAggregate: + """Aggregate manager class for Iotlab. """ + + sites = {} + nodes = {} + api = None + interfaces = {} + links = {} + node_tags = {} + + prepared = False + + user_options = {} + + def __init__(self, driver): + self.driver = driver + + def get_slice_and_slivers(self, slice_xrn, login=None): + """ + Get the slices and the associated leases if any, from the cortexlab + testbed. One slice can have mutliple leases. + For each slice, get the nodes in the associated lease + and create a sliver with the necessary info and insert it into the + sliver dictionary, keyed on the node hostnames. + Returns a dict of slivers based on the sliver's node_id. + Called by get_rspec. + + + :param slice_xrn: xrn of the slice + :param login: user's login on cortexlab ldap + + :type slice_xrn: string + :type login: string + :returns: a list of slices dict and a list of Sliver object + :rtype: (list, list) + + .. note: There is no real slivers in cortexlab, only leases. The goal + is to be consistent with the SFA standard. + + """ + slivers = {} + sfa_slice = None + if slice_xrn is None: + return (sfa_slice, slivers) + slice_urn = hrn_to_urn(slice_xrn, 'slice') + slice_hrn, _ = urn_to_hrn(slice_xrn) + slice_name = slice_hrn + + slices = self.driver.cortexlab_api.GetSlices(slice_filter=str(slice_name), + slice_filter_type='slice_hrn', + login=login) + + logger.debug("CortexlabAggregate api \tget_slice_and_slivers \ + slice_hrn %s \r\n slices %s self.driver.hrn %s" + % (slice_hrn, slices, self.driver.hrn)) + + if slices == []: + return (sfa_slice, slivers) + + # sort slivers by node id , if there is a job + #and therefore, node allocated to this slice + for sfa_slice in slices: + try: + node_ids_list = sfa_slice['node_ids'] + except KeyError: + logger.log_exc("CortexlabAggregate \t \ + get_slice_and_slivers No nodes in the slice \ + - KeyError ") + node_ids_list = [] + continue + + for node in node_ids_list: + sliver_xrn = Xrn(slice_urn, type='sliver', id=node) + sliver_xrn.set_authority(self.driver.hrn) + sliver = Sliver({'sliver_id': sliver_xrn.urn, + 'name': sfa_slice['hrn'], + 'type': 'cortexlab-node', + 'tags': []}) + + slivers[node] = sliver + + #Add default sliver attribute : + #connection information for cortexlab + if get_authority(sfa_slice['hrn']) == self.driver.cortexlab_api.root_auth: + tmp = sfa_slice['hrn'].split('.') + ldap_username = tmp[1].split('_')[0] + ssh_access = None + slivers['default_sliver'] = {'ssh': ssh_access, + 'login': ldap_username} + + #TODO get_slice_and_slivers Find the login of the external user + + logger.debug("CortexlabAggregate api get_slice_and_slivers slivers %s " + % (slivers)) + return (slices, slivers) + + + def get_nodes(self, slices=None, slivers=[], options=None): + """Returns the nodes in the slice using the rspec format, with all the + nodes' properties. + + Fetch the nodes ids in the slices dictionary and get all the nodes + properties from OAR. Makes a rspec dicitonary out of this and returns + it. If the slice does not have any job running or scheduled, that is + it has no reserved nodes, then returns an empty list. + + :param slices: list of slices (record dictionaries) + :param slivers: the list of slivers in all the slices + :type slices: list of dicts + :type slivers: list of Sliver object (dictionaries) + :returns: An empty list if the slice has no reserved nodes, a rspec + list with all the nodes and their properties (a dict per node) + otherwise. + :rtype: list + + .. seealso:: get_slice_and_slivers + + """ + # NT: the semantic of this function is not clear to me : + # if slice is not defined, then all the nodes should be returned + # if slice is defined, we should return only the nodes that + # are part of this slice + # but what is the role of the slivers parameter ? + # So i assume that slice['node_ids'] will be the same as slivers for us + slice_nodes_list = [] + if slices is not None: + for one_slice in slices: + try: + slice_nodes_list = one_slice['node_ids'] + # if we are dealing with a slice that has no node just + # return an empty list. In cortexlab a slice can have multiple + # jobs scheduled, so it either has at least one lease or + # not at all. + except KeyError: + return [] + + # get the granularity in second for the reservation system + grain = self.driver.cortexlab_api.GetLeaseGranularity() + + nodes = self.driver.cortexlab_api.GetNodes() + + nodes_dict = {} + + #if slices, this means we got to list all the nodes given to this slice + # Make a list of all the nodes in the slice before getting their + #attributes + rspec_nodes = [] + + logger.debug("CortexlabAggregate api get_nodes slices %s " + % (slices)) + + reserved_nodes = self.driver.cortexlab_api.GetNodesCurrentlyInUse() + logger.debug("CortexlabAggregate api get_nodes slice_nodes_list %s " + % (slice_nodes_list)) + for node in nodes: + nodes_dict[node['node_id']] = node + if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: + + rspec_node = IotlabNode() + # xxx how to retrieve site['login_base'] + #site_id=node['site_id'] + #site=sites_dict[site_id] + + # rspec_node['mobile'] = node['mobile'] + rspec_node['archi'] = node['archi'] + rspec_node['radio'] = node['radio'] + + cortexlab_xrn = cortexlab_xrn_object(self.driver.cortexlab_api.root_auth, + node['hostname']) + rspec_node['component_id'] = cortexlab_xrn.urn + rspec_node['component_name'] = node['hostname'] + rspec_node['component_manager_id'] = \ + hrn_to_urn(self.driver.cortexlab_api.root_auth, + 'authority+sa') + + # Iotlab's nodes are federated : there is only one authority + # for all Iotlab sites, registered in SFA. + # Removing the part including the site + # in authority_id SA 27/07/12 + rspec_node['authority_id'] = rspec_node['component_manager_id'] + + # do not include boot state ( element) + #in the manifest rspec + + rspec_node['boot_state'] = node['boot_state'] + if node['hostname'] in reserved_nodes: + rspec_node['boot_state'] = "Reserved" + rspec_node['exclusive'] = 'true' + rspec_node['hardware_types'] = [HardwareType({'name': + 'cortexlab-node'})] + + + location = IotlabLocation({'country':'France', 'site': + node['site']}) + rspec_node['location'] = location + + # Adding mobility of the node in the rspec + mobility = IotlabMobility() + for field in mobility: + try: + mobility[field] = node[field] + except KeyError, error: + logger.log_exc("CortexlabAggregate\t get_nodes \ + mobility %s " % (error)) + rspec_node['mobility'] = mobility + + position = IotlabPosition() + for field in position: + try: + position[field] = node[field] + except KeyError, error: + logger.log_exc("CortexlabAggregate\t get_nodes \ + position %s " % (error)) + + rspec_node['position'] = position + #rspec_node['interfaces'] = [] + + # Granularity + granularity = Granularity({'grain': grain}) + rspec_node['granularity'] = granularity + rspec_node['tags'] = [] + if node['hostname'] in slivers: + # add sliver info + sliver = slivers[node['hostname']] + rspec_node['sliver_id'] = sliver['sliver_id'] + rspec_node['client_id'] = node['hostname'] + rspec_node['slivers'] = [sliver] + + # slivers always provide the ssh service + login = Login({'authentication': 'ssh-keys', + 'hostname': node['hostname'], 'port': '22', + 'username': sliver['name']}) + service = Services({'login': login}) + rspec_node['services'] = [service] + rspec_nodes.append(rspec_node) + + return (rspec_nodes) + + def get_all_leases(self, ldap_username): + """ + + Get list of lease dictionaries which all have the mandatory keys + ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration'). + All the leases running or scheduled are returned. + + :param ldap_username: if ldap uid is not None, looks for the leases + belonging to this user. + :type ldap_username: string + :returns: rspec lease dictionary with keys lease_id, component_id, + slice_id, start_time, duration. + :rtype: dict + + .. note::There is no filtering of leases within a given time frame. + All the running or scheduled leases are returned. options + removed SA 15/05/2013 + + + """ + + #now = int(time.time()) + #lease_filter = {'clip': now } + + #if slice_record: + #lease_filter.update({'name': slice_record['name']}) + + #leases = self.driver.cortexlab_api.GetLeases(lease_filter) + + logger.debug("CortexlabAggregate get_all_leases ldap_username %s " + % (ldap_username)) + leases = self.driver.cortexlab_api.GetLeases(login=ldap_username) + grain = self.driver.cortexlab_api.GetLeaseGranularity() + # site_ids = [] + rspec_leases = [] + for lease in leases: + #as many leases as there are nodes in the job + for node in lease['reserved_nodes']: + rspec_lease = Lease() + rspec_lease['lease_id'] = lease['lease_id'] + #site = node['site_id'] + cortexlab_xrn = cortexlab_xrn_object(self.driver.cortexlab_api.root_auth, + node) + rspec_lease['component_id'] = cortexlab_xrn.urn + #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ + #site, node['hostname']) + try: + rspec_lease['slice_id'] = lease['slice_id'] + except KeyError: + #No info on the slice used in cortexlab_xp table + pass + rspec_lease['start_time'] = lease['t_from'] + rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ + / grain + rspec_leases.append(rspec_lease) + return rspec_leases + + def get_rspec(self, slice_xrn=None, login=None, version=None, + options=None): + """ + Returns xml rspec: + - a full advertisement rspec with the testbed resources if slice_xrn is + not specified.If a lease option is given, also returns the leases + scheduled on the testbed. + - a manifest Rspec with the leases and nodes in slice's leases if + slice_xrn is not None. + + :param slice_xrn: srn of the slice + :type slice_xrn: string + :param login: user'uid (ldap login) on cortexlab + :type login: string + :param version: can be set to sfa or cortexlab + :type version: RSpecVersion + :param options: used to specify if the leases should also be included in + the returned rspec. + :type options: dict + + :returns: Xml Rspec. + :rtype: XML + + + """ + + ldap_username = None + rspec = None + version_manager = VersionManager() + version = version_manager.get_version(version) + logger.debug("CortexlabAggregate \t get_rspec ***version %s \ + version.type %s version.version %s options %s \r\n" + % (version, version.type, version.version, options)) + + if slice_xrn is None: + rspec_version = version_manager._get_version(version.type, + version.version, 'ad') + + else: + rspec_version = version_manager._get_version( + version.type, version.version, 'manifest') + + slices, slivers = self.get_slice_and_slivers(slice_xrn, login) + if slice_xrn and slices is not None: + #Get user associated with this slice + #for one_slice in slices : + ldap_username = slices[0]['reg_researchers'][0].__dict__['hrn'] + # ldap_username = slices[0]['user'] + tmp = ldap_username.split('.') + ldap_username = tmp[1] + logger.debug("CortexlabAggregate \tget_rspec **** \ + LDAP USERNAME %s \r\n" \ + % (ldap_username)) + #at this point sliver may be empty if no cortexlab job + #is running for this user/slice. + rspec = RSpec(version=rspec_version, user_options=options) + + logger.debug("\r\n \r\n CortexlabAggregate \tget_rspec *** \ + slice_xrn %s slices %s\r\n \r\n" + % (slice_xrn, slices)) + + if options is not None and 'list_leases' in options: + lease_option = options['list_leases'] + else: + #If no options are specified, at least print the resources + lease_option = 'all' + #if slice_xrn : + #lease_option = 'all' + + if lease_option in ['all', 'resources']: + #if not options.get('list_leases') or options.get('list_leases') + #and options['list_leases'] != 'leases': + nodes = self.get_nodes(slices, slivers) + if slice_xrn and slices is None: + nodes = [] + logger.debug("\r\n") + logger.debug("CortexlabAggregate \t lease_option %s \ + get rspec ******* nodes %s" + % (lease_option, nodes)) + + sites_set = set([node['location']['site'] for node in nodes]) + + #In case creating a job, slice_xrn is not set to None + rspec.version.add_nodes(nodes) + if slice_xrn and slices is not None: + # #Get user associated with this slice + # #for one_slice in slices : + # ldap_username = slices[0]['reg_researchers'] + # # ldap_username = slices[0]['user'] + # tmp = ldap_username.split('.') + # ldap_username = tmp[1] + # # ldap_username = tmp[1].split('_')[0] + + logger.debug("CortexlabAggregate \tget_rspec **** \ + version type %s ldap_ user %s \r\n" \ + % (version.type, ldap_username)) + if version.type == "Iotlab": + rspec.version.add_connection_information( + ldap_username, sites_set) + + default_sliver = slivers.get('default_sliver', []) + if default_sliver and len(nodes) is not 0: + #default_sliver_attribs = default_sliver.get('tags', []) + logger.debug("CortexlabAggregate \tget_rspec **** \ + default_sliver%s \r\n" % (default_sliver)) + for attrib in default_sliver: + rspec.version.add_default_sliver_attribute( + attrib, default_sliver[attrib]) + + if lease_option in ['all','leases']: + leases = self.get_all_leases(ldap_username) + rspec.version.add_leases(leases) + logger.debug("CortexlabAggregate \tget_rspec **** \ + FINAL RSPEC %s \r\n" % (rspec.toxml())) + return rspec.toxml() diff --git a/sfa/cortexlab/cortexlabapi.py b/sfa/cortexlab/cortexlabapi.py new file mode 100644 index 00000000..2ffad40d --- /dev/null +++ b/sfa/cortexlab/cortexlabapi.py @@ -0,0 +1,1455 @@ +""" +File containing the IotlabTestbedAPI, used to interact with nodes, users, +slices, leases and keys, as well as the dedicated iotlab database and table, +holding information about which slice is running which job. + +""" +from datetime import datetime + +from sfa.util.sfalogging import logger + +from sfa.storage.alchemy import dbsession +from sqlalchemy.orm import joinedload +from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey +from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP +from sfa.iotlab.OARrestapi import OARrestapi +from sfa.iotlab.LDAPapi import LDAPapi + +from sfa.util.xrn import Xrn, hrn_to_urn, get_authority + +from sfa.trust.certificate import Keypair, convert_public_key +from sfa.trust.gid import create_uuid +from sfa.trust.hierarchy import Hierarchy + +from sfa.iotlab.iotlabaggregate import iotlab_xrn_object + +class CortexlabTestbedAPI(): + """ Class enabled to use LDAP and OAR api calls. """ + + _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins + + def __init__(self, config): + """Creates an instance of OARrestapi and LDAPapi which will be used to + issue calls to OAR or LDAP methods. + Set the time format and the testbed granularity used for OAR + reservation and leases. + + :param config: configuration object from sfa.util.config + :type config: Config object + """ + self.iotlab_db = IotlabDB(config) + self.oar = OARrestapi() + self.ldap = LDAPapi() + self.time_format = "%Y-%m-%d %H:%M:%S" + self.root_auth = config.SFA_REGISTRY_ROOT_AUTH + self.grain = 60 # 10 mins lease minimum, 60 sec granularity + #import logging, logging.handlers + #from sfa.util.sfalogging import _SfaLogger + #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \ + #level=logging.DEBUG) + return + + @staticmethod + def GetMinExperimentDurationInGranularity(): + """ Returns the minimum allowed duration for an experiment on the + testbed. In seconds. + + """ + return IotlabTestbedAPI._MINIMUM_DURATION + + @staticmethod + def GetPeers(peer_filter=None ): + """ Gathers registered authorities in SFA DB and looks for specific peer + if peer_filter is specified. + :param peer_filter: name of the site authority looked for. + :type peer_filter: string + :returns: list of records. + + """ + + existing_records = {} + existing_hrns_by_types = {} + logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter)) + all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all() + + for record in all_records: + existing_records[(record.hrn, record.type)] = record + if record.type not in existing_hrns_by_types: + existing_hrns_by_types[record.type] = [record.hrn] + else: + existing_hrns_by_types[record.type].append(record.hrn) + + logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s " + % (existing_hrns_by_types)) + records_list = [] + + try: + if peer_filter: + records_list.append(existing_records[(peer_filter, + 'authority')]) + else: + for hrn in existing_hrns_by_types['authority']: + records_list.append(existing_records[(hrn, 'authority')]) + + logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s " + % (records_list)) + + except KeyError: + pass + + return_records = records_list + logger.debug("CORTEXLAB_API \tGetPeer return_records %s " + % (return_records)) + return return_records + + #TODO : Handling OR request in make_ldap_filters_from_records + #instead of the for loop + #over the records' list + def GetPersons(self, person_filter=None): + """ + Get the enabled users and their properties from Iotlab LDAP. + If a filter is specified, looks for the user whose properties match + the filter, otherwise returns the whole enabled users'list. + + :param person_filter: Must be a list of dictionnaries with users + properties when not set to None. + :type person_filter: list of dict + + :returns: Returns a list of users whose accounts are enabled + found in ldap. + :rtype: list of dicts + + """ + logger.debug("CORTEXLAB_API \tGetPersons person_filter %s" + % (person_filter)) + person_list = [] + if person_filter and isinstance(person_filter, list): + #If we are looking for a list of users (list of dict records) + #Usually the list contains only one user record + for searched_attributes in person_filter: + + #Get only enabled user accounts in iotlab LDAP : + #add a filter for make_ldap_filters_from_record + person = self.ldap.LdapFindUser(searched_attributes, + is_user_enabled=True) + #If a person was found, append it to the list + if person: + person_list.append(person) + + #If the list is empty, return None + if len(person_list) is 0: + person_list = None + + else: + #Get only enabled user accounts in iotlab LDAP : + #add a filter for make_ldap_filters_from_record + person_list = self.ldap.LdapFindUser(is_user_enabled=True) + + return person_list + + + #def GetTimezone(self): + #""" Returns the OAR server time and timezone. + #Unused SA 30/05/13""" + #server_timestamp, server_tz = self.oar.parser.\ + #SendRequest("GET_timezone") + #return server_timestamp, server_tz + + def DeleteLeases(self, lease_id, username): + """ + + Deletes the lease with the specified lease_id and username on OAR by + posting a delete request to OAR. + + :param lease_id: Reservation identifier. + :param username: user's iotlab login in LDAP. + :type lease_id: Depends on what tou are using, could be integer or + string + :type username: string + + :returns: dictionary with the lease id and if delete has been successful + (True) or no (False) + :rtype: dict + + """ + + # Here delete the lease specified + + # If the username is not necessary to delete the lease, then you can + # remove it from the parameters, given that you propagate the changes + + + # Return delete status so that you know if the delete has been + # successuf or not + if answer['status'] == 'Delete request registered': + ret = {lease_id: True} + else: + ret = {lease_id: False} + logger.debug("CORTEXLAB_API \DeleteLeases lease_id %s \r\n answer %s \ + username %s" % (lease_id, answer, username)) + return ret + + + + + + + def GetJobsResources(self, job_id, username = None): + """ Gets the list of nodes associated with the job_id and username + if provided. + Transforms the iotlab hostnames to the corresponding + SFA nodes hrns. + Rertuns dict key :'node_ids' , value : hostnames list + :param username: user's LDAP login + :paran job_id: job's OAR identifier. + :type username: string + :type job_id: integer + + :returns: dicionary with nodes' hostnames belonging to the job. + :rtype: dict + """ + + req = "GET_jobs_id_resources" + + + #Get job resources list from OAR + node_id_list = self.oar.parser.SendRequest(req, job_id, username) + logger.debug("CORTEXLAB_API \t GetJobsResources %s " %(node_id_list)) + + hostname_list = \ + self.__get_hostnames_from_oar_node_ids(node_id_list) + + + #Replaces the previous entry "assigned_network_address" / + #"reserved_resources" with "node_ids" + job_info = {'node_ids': hostname_list} + + return job_info + + + + def GetNodesCurrentlyInUse(self): + """Returns a list of all the nodes already involved in an oar running + job. + :rtype: list of nodes hostnames. + """ + return self.oar.parser.SendRequest("GET_running_jobs") + + def __get_hostnames_from_oar_node_ids(self, resource_id_list ): + """Get the hostnames of the nodes from their OAR identifiers. + Get the list of nodes dict using GetNodes and find the hostname + associated with the identifier. + :param resource_id_list: list of nodes identifiers + :returns: list of node hostnames. + """ + full_nodes_dict_list = self.GetNodes() + #Put the full node list into a dictionary keyed by oar node id + oar_id_node_dict = {} + for node in full_nodes_dict_list: + oar_id_node_dict[node['oar_id']] = node + + hostname_list = [] + for resource_id in resource_id_list: + #Because jobs requested "asap" do not have defined resources + if resource_id is not "Undefined": + hostname_list.append(\ + oar_id_node_dict[resource_id]['hostname']) + + #hostname_list.append(oar_id_node_dict[resource_id]['hostname']) + return hostname_list + + def GetReservedNodes(self, username=None): + """ Get list of leases. Get the leases for the username if specified, + otherwise get all the leases. Finds the nodes hostnames for each + OAR node identifier. + :param username: user's LDAP login + :type username: string + :returns: list of reservations dict + :rtype: dict list + """ + + #Get the nodes in use and the reserved nodes + reservation_dict_list = \ + self.oar.parser.SendRequest("GET_reserved_nodes", \ + username = username) + + + for resa in reservation_dict_list: + logger.debug ("GetReservedNodes resa %s"%(resa)) + #dict list of hostnames and their site + resa['reserved_nodes'] = \ + self.__get_hostnames_from_oar_node_ids(resa['resource_ids']) + + #del resa['resource_ids'] + return reservation_dict_list + + def GetNodes(self, node_filter_dict=None, return_fields_list=None): + """ + + Make a list of iotlab nodes and their properties from information + given by OAR. Search for specific nodes if some filters are + specified. Nodes properties returned if no return_fields_list given: + 'hrn','archi','mobile','hostname','site','boot_state','node_id', + 'radio','posx','posy','oar_id','posz'. + + :param node_filter_dict: dictionnary of lists with node properties. For + instance, if you want to look for a specific node with its hrn, + the node_filter_dict should be {'hrn': [hrn_of_the_node]} + :type node_filter_dict: dict + :param return_fields_list: list of specific fields the user wants to be + returned. + :type return_fields_list: list + :returns: list of dictionaries with node properties + :rtype: list + + """ + node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full") + node_dict_list = node_dict_by_id.values() + logger.debug (" CORTEXLAB_API GetNodes node_filter_dict %s \ + return_fields_list %s " % (node_filter_dict, return_fields_list)) + #No filtering needed return the list directly + if not (node_filter_dict or return_fields_list): + return node_dict_list + + return_node_list = [] + if node_filter_dict: + for filter_key in node_filter_dict: + try: + #Filter the node_dict_list by each value contained in the + #list node_filter_dict[filter_key] + for value in node_filter_dict[filter_key]: + for node in node_dict_list: + if node[filter_key] == value: + if return_fields_list: + tmp = {} + for k in return_fields_list: + tmp[k] = node[k] + return_node_list.append(tmp) + else: + return_node_list.append(node) + except KeyError: + logger.log_exc("GetNodes KeyError") + return + + + return return_node_list + + + + @staticmethod + def AddSlice(slice_record, user_record): + """ + + Add slice to the local iotlab sfa tables if the slice comes + from a federated site and is not yet in the iotlab sfa DB, + although the user has already a LDAP login. + Called by verify_slice during lease/sliver creation. + + :param slice_record: record of slice, must contain hrn, gid, slice_id + and authority of the slice. + :type slice_record: dictionary + :param user_record: record of the user + :type user_record: RegUser + + """ + + sfa_record = RegSlice(hrn=slice_record['hrn'], + gid=slice_record['gid'], + pointer=slice_record['slice_id'], + authority=slice_record['authority']) + logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s" + % (sfa_record, user_record)) + sfa_record.just_created() + dbsession.add(sfa_record) + dbsession.commit() + #Update the reg-researcher dependance table + sfa_record.reg_researchers = [user_record] + dbsession.commit() + + return + + + def GetSites(self, site_filter_name_list=None, return_fields_list=None): + """Returns the list of Iotlab's sites with the associated nodes and + their properties as dictionaries. + + Uses the OAR request GET_sites to find the Iotlab's sites. + + :param site_filter_name_list: used to specify specific sites + :param return_fields_list: field that has to be returned + :type site_filter_name_list: list + :type return_fields_list: list + + .. warning:: unused + """ + site_dict = self.oar.parser.SendRequest("GET_sites") + #site_dict : dict where the key is the sit ename + return_site_list = [] + if not (site_filter_name_list or return_fields_list): + return_site_list = site_dict.values() + return return_site_list + + for site_filter_name in site_filter_name_list: + if site_filter_name in site_dict: + if return_fields_list: + for field in return_fields_list: + tmp = {} + try: + tmp[field] = site_dict[site_filter_name][field] + except KeyError: + logger.error("GetSites KeyError %s " % (field)) + return None + return_site_list.append(tmp) + else: + return_site_list.append(site_dict[site_filter_name]) + + return return_site_list + + + #TODO : Check rights to delete person + def DeletePerson(self, person_record): + """Disable an existing account in iotlab LDAP. + + Users and techs can only delete themselves. PIs can only + delete themselves and other non-PIs at their sites. + ins can delete anyone. + + :param person_record: user's record + :type person_record: dict + :returns: True if successful, False otherwise. + :rtype: boolean + + .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF. + """ + #Disable user account in iotlab LDAP + ret = self.ldap.LdapMarkUserAsDeleted(person_record) + logger.warning("CORTEXLAB_API DeletePerson %s " % (person_record)) + return ret['bool'] + + def DeleteSlice(self, slice_record): + """Deletes the specified slice and kills the jobs associated with + the slice if any, using DeleteSliceFromNodes. + + :param slice_record: record of the slice, must contain oar_job_id, user + :type slice_record: dict + :returns: True if all the jobs in the slice have been deleted, + or the list of jobs that could not be deleted otherwise. + :rtype: list or boolean + + .. seealso:: DeleteSliceFromNodes + + """ + ret = self.DeleteSliceFromNodes(slice_record) + delete_failed = None + for job_id in ret: + if False in ret[job_id]: + if delete_failed is None: + delete_failed = [] + delete_failed.append(job_id) + + logger.info("CORTEXLAB_API DeleteSlice %s answer %s"%(slice_record, \ + delete_failed)) + return delete_failed or True + + @staticmethod + def __add_person_to_db(user_dict): + """ + Add a federated user straight to db when the user issues a lease + request with iotlab nodes and that he has not registered with iotlab + yet (that is he does not have a LDAP entry yet). + Uses parts of the routines in SlabImport when importing user from LDAP. + Called by AddPerson, right after LdapAddUser. + :param user_dict: Must contain email, hrn and pkey to get a GID + and be added to the SFA db. + :type user_dict: dict + + """ + check_if_exists = \ + dbsession.query(RegUser).filter_by(email = user_dict['email']).first() + #user doesn't exists + if not check_if_exists: + logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \ + " %(user_dict)) + hrn = user_dict['hrn'] + person_urn = hrn_to_urn(hrn, 'user') + pubkey = user_dict['pkey'] + try: + pkey = convert_public_key(pubkey) + except TypeError: + #key not good. create another pkey + logger.warn('__add_person_to_db: unable to convert public \ + key for %s' %(hrn )) + pkey = Keypair(create=True) + + + if pubkey is not None and pkey is not None : + hierarchy = Hierarchy() + person_gid = hierarchy.create_gid(person_urn, create_uuid(), \ + pkey) + if user_dict['email']: + logger.debug("__add_person_to_db \r\n \r\n \ + IOTLAB IMPORTER PERSON EMAIL OK email %s "\ + %(user_dict['email'])) + person_gid.set_email(user_dict['email']) + + user_record = RegUser(hrn=hrn , pointer= '-1', \ + authority=get_authority(hrn), \ + email=user_dict['email'], gid = person_gid) + user_record.reg_keys = [RegKey(user_dict['pkey'])] + user_record.just_created() + dbsession.add (user_record) + dbsession.commit() + return + + + def AddPerson(self, record): + """ + + Adds a new account. Any fields specified in records are used, + otherwise defaults are used. Creates an appropriate login by calling + LdapAddUser. + + :param record: dictionary with the sfa user's properties. + :returns: a dicitonary with the status. If successful, the dictionary + boolean is set to True and there is a 'uid' key with the new login + added to LDAP, otherwise the bool is set to False and a key + 'message' is in the dictionary, with the error message. + :rtype: dict + + """ + ret = self.ldap.LdapAddUser(record) + + if ret['bool'] is True: + record['hrn'] = self.root_auth + '.' + ret['uid'] + logger.debug("CORTEXLAB_API AddPerson return code %s record %s " + % (ret, record)) + self.__add_person_to_db(record) + return ret + + + + + + #TODO AddPersonKey 04/07/2012 SA + def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict): + """Adds a new key to the specified account. Adds the key to the + iotlab ldap, provided that the person_uid is valid. + + Non-admins can only modify their own keys. + + :param person_uid: user's iotlab login in LDAP + :param old_attributes_dict: dict with the user's old sshPublicKey + :param new_key_dict: dict with the user's new sshPublicKey + :type person_uid: string + + + :rtype: Boolean + :returns: True if the key has been modified, False otherwise. + + """ + ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \ + new_key_dict) + logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ") + return ret['bool'] + + def DeleteLeases(self, leases_id_list, slice_hrn): + """ + + Deletes several leases, based on their job ids and the slice + they are associated with. Uses DeleteJobs to delete the jobs + on OAR. Note that one slice can contain multiple jobs, and in this + case all the jobs in the leases_id_list MUST belong to ONE slice, + since there is only one slice hrn provided here. + + :param leases_id_list: list of job ids that belong to the slice whose + slice hrn is provided. + :param slice_hrn: the slice hrn. + :type slice_hrn: string + + .. warning:: Does not have a return value since there was no easy + way to handle failure when dealing with multiple job delete. Plus, + there was no easy way to report it to the user. + + """ + logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \ + \r\n " %(leases_id_list, slice_hrn)) + for job_id in leases_id_list: + self.DeleteJobs(job_id, slice_hrn) + + return + + @staticmethod + def _process_walltime(duration): + """ Calculates the walltime in seconds from the duration in H:M:S + specified in the RSpec. + + """ + if duration: + # Fixing the walltime by adding a few delays. + # First put the walltime in seconds oarAdditionalDelay = 20; + # additional delay for /bin/sleep command to + # take in account prologue and epilogue scripts execution + # int walltimeAdditionalDelay = 240; additional delay + #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT + #in oar.conf + # Put the duration in seconds first + #desired_walltime = duration * 60 + desired_walltime = duration + total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12 + sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12 + walltime = [] + #Put the walltime back in str form + #First get the hours + walltime.append(str(total_walltime / 3600)) + total_walltime = total_walltime - 3600 * int(walltime[0]) + #Get the remaining minutes + walltime.append(str(total_walltime / 60)) + total_walltime = total_walltime - 60 * int(walltime[1]) + #Get the seconds + walltime.append(str(total_walltime)) + + else: + logger.log_exc(" __process_walltime duration null") + + return walltime, sleep_walltime + + @staticmethod + def _create_job_structure_request_for_OAR(lease_dict): + """ Creates the structure needed for a correct POST on OAR. + Makes the timestamp transformation into the appropriate format. + Sends the POST request to create the job with the resources in + added_nodes. + + """ + + nodeid_list = [] + reqdict = {} + + + reqdict['workdir'] = '/tmp' + reqdict['resource'] = "{network_address in (" + + for node in lease_dict['added_nodes']: + logger.debug("\r\n \r\n OARrestapi \t \ + __create_job_structure_request_for_OAR node %s" %(node)) + + # Get the ID of the node + nodeid = node + reqdict['resource'] += "'" + nodeid + "', " + nodeid_list.append(nodeid) + + custom_length = len(reqdict['resource'])- 2 + reqdict['resource'] = reqdict['resource'][0:custom_length] + \ + ")}/nodes=" + str(len(nodeid_list)) + + + walltime, sleep_walltime = \ + IotlabTestbedAPI._process_walltime(\ + int(lease_dict['lease_duration'])) + + + reqdict['resource'] += ",walltime=" + str(walltime[0]) + \ + ":" + str(walltime[1]) + ":" + str(walltime[2]) + reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime) + + #In case of a scheduled experiment (not immediate) + #To run an XP immediately, don't specify date and time in RSpec + #They will be set to None. + if lease_dict['lease_start_time'] is not '0': + #Readable time accepted by OAR + start_time = datetime.fromtimestamp( \ + int(lease_dict['lease_start_time'])).\ + strftime(lease_dict['time_format']) + reqdict['reservation'] = start_time + #If there is not start time, Immediate XP. No need to add special + # OAR parameters + + + reqdict['type'] = "deploy" + reqdict['directory'] = "" + reqdict['name'] = "SFA_" + lease_dict['slice_user'] + + return reqdict + + + def LaunchExperimentOnOAR(self, added_nodes, slice_name, \ + lease_start_time, lease_duration, slice_user=None): + + """ + Create a job request structure based on the information provided + and post the job on OAR. + :param added_nodes: list of nodes that belong to the described lease. + :param slice_name: the slice hrn associated to the lease. + :param lease_start_time: timestamp of the lease startting time. + :param lease_duration: lease durationin minutes + + """ + lease_dict = {} + lease_dict['lease_start_time'] = lease_start_time + lease_dict['lease_duration'] = lease_duration + lease_dict['added_nodes'] = added_nodes + lease_dict['slice_name'] = slice_name + lease_dict['slice_user'] = slice_user + lease_dict['grain'] = self.GetLeaseGranularity() + lease_dict['time_format'] = self.time_format + + + logger.debug("CORTEXLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\ + \r\n " %(slice_user)) + #Create the request for OAR + reqdict = self._create_job_structure_request_for_OAR(lease_dict) + # first step : start the OAR job and update the job + logger.debug("CORTEXLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\ + \r\n " %(reqdict)) + + answer = self.oar.POSTRequestToOARRestAPI('POST_job', \ + reqdict, slice_user) + logger.debug("CORTEXLAB_API \tLaunchExperimentOnOAR jobid %s " %(answer)) + try: + jobid = answer['id'] + except KeyError: + logger.log_exc("CORTEXLAB_API \tLaunchExperimentOnOAR \ + Impossible to create job %s " %(answer)) + return None + + + + + if jobid : + logger.debug("CORTEXLAB_API \tLaunchExperimentOnOAR jobid %s \ + added_nodes %s slice_user %s" %(jobid, added_nodes, \ + slice_user)) + + + return jobid + + + def AddLeases(self, hostname_list, slice_record, + lease_start_time, lease_duration): + + """Creates a job in OAR corresponding to the information provided + as parameters. Adds the job id and the slice hrn in the iotlab + database so that we are able to know which slice has which nodes. + + :param hostname_list: list of nodes' OAR hostnames. + :param slice_record: sfa slice record, must contain login and hrn. + :param lease_start_time: starting time , unix timestamp format + :param lease_duration: duration in minutes + + :type hostname_list: list + :type slice_record: dict + :type lease_start_time: integer + :type lease_duration: integer + + """ + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \ + slice_record %s lease_start_time %s lease_duration %s "\ + %( hostname_list, slice_record , lease_start_time, \ + lease_duration)) + + #tmp = slice_record['reg-researchers'][0].split(".") + username = slice_record['login'] + #username = tmp[(len(tmp)-1)] + job_id = self.LaunchExperimentOnOAR(hostname_list, \ + slice_record['hrn'], \ + lease_start_time, lease_duration, \ + username) + start_time = \ + datetime.fromtimestamp(int(lease_start_time)).\ + strftime(self.time_format) + end_time = lease_start_time + lease_duration + + + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ + %s %s %s "%(slice_record['hrn'], job_id, end_time)) + + + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \ + %(type(slice_record['hrn']), type(job_id), type(end_time))) + + iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id, + end_time= end_time) + + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \ + %(iotlab_ex_row)) + self.iotlab_db.iotlab_session.add(iotlab_ex_row) + self.iotlab_db.iotlab_session.commit() + + logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \ + %(start_time)) + + return + + + #Delete the jobs from job_iotlab table + def DeleteSliceFromNodes(self, slice_record): + """ + + Deletes all the running or scheduled jobs of a given slice + given its record. + + :param slice_record: record of the slice, must contain oar_job_id, user + :type slice_record: dict + + :returns: dict of the jobs'deletion status. Success= True, Failure= + False, for each job id. + :rtype: dict + + """ + logger.debug("CORTEXLAB_API \t DeleteSliceFromNodes %s " + % (slice_record)) + + if isinstance(slice_record['oar_job_id'], list): + oar_bool_answer = {} + for job_id in slice_record['oar_job_id']: + ret = self.DeleteJobs(job_id, slice_record['user']) + + oar_bool_answer.update(ret) + + else: + oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], + slice_record['user'])] + + return oar_bool_answer + + + + def GetLeaseGranularity(self): + """ Returns the granularity of an experiment in the Iotlab testbed. + OAR uses seconds for experiments duration , the granulaity is also + defined in seconds. + Experiments which last less than 10 min (600 sec) are invalid""" + return self.grain + + + # @staticmethod + # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql): + # """ Cleans the iotlab db by deleting expired and cancelled jobs. + # Compares the list of job ids given by OAR with the job ids that + # are already in the database, deletes the jobs that are no longer in + # the OAR job id list. + # :param job_oar_list: list of job ids coming from OAR + # :type job_oar_list: list + # :param job_psql: list of job ids cfrom the database. + # type job_psql: list + # """ + # #Turn the list into a set + # set_jobs_psql = set(jobs_psql) + + # kept_jobs = set(job_oar_list).intersection(set_jobs_psql) + # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \ + # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) + # deleted_jobs = set_jobs_psql.difference(kept_jobs) + # deleted_jobs = list(deleted_jobs) + # if len(deleted_jobs) > 0: + # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + # self.iotlab_db.iotlab_session.commit() + + # return + + @staticmethod + def filter_lease_name(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \ + % (reservation_list)) + for reservation in reservation_list: + if 'slice_hrn' in reservation and \ + reservation['slice_hrn'] != filter_value: + filtered_reservation_list.remove(reservation) + + logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \ + % (filtered_reservation_list)) + return filtered_reservation_list + + @staticmethod + def filter_lease_start_time(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + + for reservation in reservation_list: + if 't_from' in reservation and \ + reservation['t_from'] > filter_value: + filtered_reservation_list.remove(reservation) + + return filtered_reservation_list + + def GetLeases(self, lease_filter_dict=None, login=None): + """ + + Get the list of leases from OAR with complete information + about which slice owns which jobs and nodes. + Two purposes: + -Fetch all the jobs from OAR (running, waiting..) + complete the reservation information with slice hrn + found in iotlab_xp table. If not available in the table, + assume it is a iotlab slice. + -Updates the iotlab table, deleting jobs when necessary. + + :returns: reservation_list, list of dictionaries with 'lease_id', + 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', + 'slice_hrn', 'resource_ids', 't_from', 't_until' + :rtype: list + + """ + + unfiltered_reservation_list = self.GetReservedNodes(login) + + reservation_list = [] + #Find the slice associated with this user iotlab ldap uid + logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\ + unfiltered_reservation_list %s " + % (login, unfiltered_reservation_list)) + #Create user dict first to avoid looking several times for + #the same user in LDAP SA 27/07/12 + job_oar_list = [] + + jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all() + jobs_psql_dict = dict([(row.job_id, row.__dict__) + for row in jobs_psql_query]) + #jobs_psql_dict = jobs_psql_dict) + logger.debug("CORTEXLAB_API \tGetLeases jobs_psql_dict %s" + % (jobs_psql_dict)) + jobs_psql_id_list = [row.job_id for row in jobs_psql_query] + + for resa in unfiltered_reservation_list: + logger.debug("CORTEXLAB_API \tGetLeases USER %s" + % (resa['user'])) + #Construct list of jobs (runing, waiting..) in oar + job_oar_list.append(resa['lease_id']) + #If there is information on the job in IOTLAB DB ] + #(slice used and job id) + if resa['lease_id'] in jobs_psql_dict: + job_info = jobs_psql_dict[resa['lease_id']] + logger.debug("CORTEXLAB_API \tGetLeases job_info %s" + % (job_info)) + resa['slice_hrn'] = job_info['slice_hrn'] + resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') + + #otherwise, assume it is a iotlab slice: + else: + resa['slice_id'] = hrn_to_urn(self.root_auth + '.' + + resa['user'] + "_slice", 'slice') + resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn() + + resa['component_id_list'] = [] + #Transform the hostnames into urns (component ids) + for node in resa['reserved_nodes']: + + iotlab_xrn = iotlab_xrn_object(self.root_auth, node) + resa['component_id_list'].append(iotlab_xrn.urn) + + if lease_filter_dict: + logger.debug("CORTEXLAB_API \tGetLeases \ + \r\n leasefilter %s" % ( lease_filter_dict)) + + filter_dict_functions = { + 'slice_hrn' : IotlabTestbedAPI.filter_lease_name, + 't_from' : IotlabTestbedAPI.filter_lease_start_time + } + reservation_list = list(unfiltered_reservation_list) + for filter_type in lease_filter_dict: + logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \ + % (reservation_list)) + reservation_list = filter_dict_functions[filter_type](\ + reservation_list,lease_filter_dict[filter_type] ) + + # Filter the reservation list with a maximum timespan so that the + # leases and jobs running after this timestamp do not appear + # in the result leases. + # if 'start_time' in : + # if resa['start_time'] < lease_filter_dict['start_time']: + # reservation_list.append(resa) + + + # if 'name' in lease_filter_dict and \ + # lease_filter_dict['name'] == resa['slice_hrn']: + # reservation_list.append(resa) + + + if lease_filter_dict is None: + reservation_list = unfiltered_reservation_list + + self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list) + + logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s" + % (reservation_list)) + return reservation_list + + + + +#TODO FUNCTIONS SECTION 04/07/2012 SA + + ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing + ##04/07/2012 SA + #@staticmethod + #def UnBindObjectFromPeer( auth, object_type, object_id, shortname): + #""" This method is a hopefully temporary hack to let the sfa correctly + #detach the objects it creates from a remote peer object. This is + #needed so that the sfa federation link can work in parallel with + #RefreshPeer, as RefreshPeer depends on remote objects being correctly + #marked. + #Parameters: + #auth : struct, API authentication structure + #AuthMethod : string, Authentication method to use + #object_type : string, Object type, among 'site','person','slice', + #'node','key' + #object_id : int, object_id + #shortname : string, peer shortname + #FROM PLC DOC + + #""" + #logger.warning("CORTEXLAB_API \tUnBindObjectFromPeer EMPTY-\ + #DO NOTHING \r\n ") + #return + + ##TODO Is BindObjectToPeer still necessary ? Currently does nothing + ##04/07/2012 SA + #|| Commented out 28/05/13 SA + #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \ + #remote_object_id=None): + #"""This method is a hopefully temporary hack to let the sfa correctly + #attach the objects it creates to a remote peer object. This is needed + #so that the sfa federation link can work in parallel with RefreshPeer, + #as RefreshPeer depends on remote objects being correctly marked. + #Parameters: + #shortname : string, peer shortname + #remote_object_id : int, remote object_id, set to 0 if unknown + #FROM PLC API DOC + + #""" + #logger.warning("CORTEXLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ") + #return + + ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA + ##Funciton should delete and create another job since oin iotlab slice=job + #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None): + #"""Updates the parameters of an existing slice with the values in + #slice_fields. + #Users may only update slices of which they are members. + #PIs may update any of the slices at their sites, or any slices of + #which they are members. Admins may update any slice. + #Only PIs and admins may update max_nodes. Slices cannot be renewed + #(by updating the expires parameter) more than 8 weeks into the future. + #Returns 1 if successful, faults otherwise. + #FROM PLC API DOC + + #""" + #logger.warning("CORTEXLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ") + #return + + #Unused SA 30/05/13, we only update the user's key or we delete it. + ##TODO UpdatePerson 04/07/2012 SA + #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None): + #"""Updates a person. Only the fields specified in person_fields + #are updated, all other fields are left untouched. + #Users and techs can only update themselves. PIs can only update + #themselves and other non-PIs at their sites. + #Returns 1 if successful, faults otherwise. + #FROM PLC API DOC + + #""" + ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn) + ##self.iotlab_db.iotlab_session.add(new_row) + ##self.iotlab_db.iotlab_session.commit() + + #logger.debug("CORTEXLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ") + #return + + @staticmethod + def GetKeys(key_filter=None): + """Returns a dict of dict based on the key string. Each dict entry + contains the key id, the ssh key, the user's email and the + user's hrn. + If key_filter is specified and is an array of key identifiers, + only keys matching the filter will be returned. + + Admin may query all keys. Non-admins may only query their own keys. + FROM PLC API DOC + + :returns: dict with ssh key as key and dicts as value. + :rtype: dict + """ + if key_filter is None: + keys = dbsession.query(RegKey).options(joinedload('reg_user')).all() + else: + keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all() + + key_dict = {} + for key in keys: + key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, + 'email': key.reg_user.email, + 'hrn': key.reg_user.hrn} + + #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) + #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ + #for user in ldap_rslt) + + logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict)) + return key_dict + + #TODO : test + def DeleteKey(self, user_record, key_string): + """Deletes a key in the LDAP entry of the specified user. + + Removes the key_string from the user's key list and updates the LDAP + user's entry with the new key attributes. + + :param key_string: The ssh key to remove + :param user_record: User's record + :type key_string: string + :type user_record: dict + :returns: True if sucessful, False if not. + :rtype: Boolean + + """ + all_user_keys = user_record['keys'] + all_user_keys.remove(key_string) + new_attributes = {'sshPublicKey':all_user_keys} + ret = self.ldap.LdapModifyUser(user_record, new_attributes) + logger.debug("CORTEXLAB_API DeleteKey %s- " % (ret)) + return ret['bool'] + + + + + @staticmethod + def _sql_get_slice_info(slice_filter): + """ + Get the slice record based on the slice hrn. Fetch the record of the + user associated with the slice by using joinedload based on the + reg_researcher relationship. + + :param slice_filter: the slice hrn we are looking for + :type slice_filter: string + :returns: the slice record enhanced with the user's information if the + slice was found, None it wasn't. + + :rtype: dict or None. + """ + #DO NOT USE RegSlice - reg_researchers to get the hrn + #of the user otherwise will mess up the RegRecord in + #Resolve, don't know why - SA 08/08/2012 + + #Only one entry for one user = one slice in iotlab_xp table + #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() + raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() + #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() + if raw_slicerec: + #load_reg_researcher + #raw_slicerec.reg_researchers + raw_slicerec = raw_slicerec.__dict__ + logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \ + raw_slicerec %s" % (slice_filter, raw_slicerec)) + slicerec = raw_slicerec + #only one researcher per slice so take the first one + #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] + #del slicerec['reg_researchers']['_sa_instance_state'] + return slicerec + + else: + return None + + @staticmethod + def _sql_get_slice_info_from_user(slice_filter): + """ + Get the slice record based on the user recordid by using a joinedload + on the relationship reg_slices_as_researcher. Format the sql record + into a dict with the mandatory fields for user and slice. + :returns: dict with slice record and user record if the record was found + based on the user's id, None if not.. + :rtype:dict or None.. + """ + #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() + raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first() + #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() + #Put it in correct order + user_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'email', 'pointer'] + slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'pointer'] + if raw_slicerec: + #raw_slicerec.reg_slices_as_researcher + raw_slicerec = raw_slicerec.__dict__ + slicerec = {} + slicerec = \ + dict([(k, raw_slicerec[ + 'reg_slices_as_researcher'][0].__dict__[k]) + for k in slice_needed_fields]) + slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) + for k in user_needed_fields]) + #TODO Handle multiple slices for one user SA 10/12/12 + #for now only take the first slice record associated to the rec user + ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__ + #del raw_slicerec['reg_slices_as_researcher'] + #slicerec['reg_researchers'] = raw_slicerec + ##del slicerec['_sa_instance_state'] + + return slicerec + + else: + return None + + def _get_slice_records(self, slice_filter=None, + slice_filter_type=None): + """ + Get the slice record depending on the slice filter and its type. + :param slice_filter: Can be either the slice hrn or the user's record + id. + :type slice_filter: string + :param slice_filter_type: describes the slice filter type used, can be + slice_hrn or record_id_user + :type: string + :returns: the slice record + :rtype:dict + .. seealso::_sql_get_slice_info_from_user + .. seealso:: _sql_get_slice_info + """ + + #Get list of slices based on the slice hrn + if slice_filter_type == 'slice_hrn': + + #if get_authority(slice_filter) == self.root_auth: + #login = slice_filter.split(".")[1].split("_")[0] + + slicerec = self._sql_get_slice_info(slice_filter) + + if slicerec is None: + return None + #return login, None + + #Get slice based on user id + if slice_filter_type == 'record_id_user': + + slicerec = self._sql_get_slice_info_from_user(slice_filter) + + if slicerec: + fixed_slicerec_dict = slicerec + #At this point if there is no login it means + #record_id_user filter has been used for filtering + #if login is None : + ##If theslice record is from iotlab + #if fixed_slicerec_dict['peer_authority'] is None: + #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + #return login, fixed_slicerec_dict + return fixed_slicerec_dict + else: + return None + + + def GetSlices(self, slice_filter=None, slice_filter_type=None, + login=None): + """Get the slice records from the iotlab db and add lease information + if any. + + :param slice_filter: can be the slice hrn or slice record id in the db + depending on the slice_filter_type. + :param slice_filter_type: defines the type of the filtering used, Can be + either 'slice_hrn' or "record_id'. + :type slice_filter: string + :type slice_filter_type: string + :returns: a slice dict if slice_filter and slice_filter_type + are specified and a matching entry is found in the db. The result + is put into a list.Or a list of slice dictionnaries if no filters + arespecified. + + :rtype: list + + """ + #login = None + authorized_filter_types_list = ['slice_hrn', 'record_id_user'] + return_slicerec_dictlist = [] + + #First try to get information on the slice based on the filter provided + if slice_filter_type in authorized_filter_types_list: + fixed_slicerec_dict = self._get_slice_records(slice_filter, + slice_filter_type) + # if the slice was not found in the sfa db + if fixed_slicerec_dict is None: + return return_slicerec_dictlist + + slice_hrn = fixed_slicerec_dict['hrn'] + + logger.debug(" CORTEXLAB_API \tGetSlices login %s \ + slice record %s slice_filter %s \ + slice_filter_type %s " % (login, + fixed_slicerec_dict, slice_filter, + slice_filter_type)) + + + #Now we have the slice record fixed_slicerec_dict, get the + #jobs associated to this slice + leases_list = [] + + leases_list = self.GetLeases(login=login) + #If no job is running or no job scheduled + #return only the slice record + if leases_list == [] and fixed_slicerec_dict: + return_slicerec_dictlist.append(fixed_slicerec_dict) + + # if the jobs running don't belong to the user/slice we are looking + # for + leases_hrn = [lease['slice_hrn'] for lease in leases_list] + if slice_hrn not in leases_hrn: + return_slicerec_dictlist.append(fixed_slicerec_dict) + #If several jobs for one slice , put the slice record into + # each lease information dict + for lease in leases_list: + slicerec_dict = {} + logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \ + \t lease['slice_hrn'] %s" + % (slice_filter, lease['slice_hrn'])) + if lease['slice_hrn'] == slice_hrn: + slicerec_dict['oar_job_id'] = lease['lease_id'] + #Update lease dict with the slice record + if fixed_slicerec_dict: + fixed_slicerec_dict['oar_job_id'] = [] + fixed_slicerec_dict['oar_job_id'].append( + slicerec_dict['oar_job_id']) + slicerec_dict.update(fixed_slicerec_dict) + #slicerec_dict.update({'hrn':\ + #str(fixed_slicerec_dict['slice_hrn'])}) + slicerec_dict['slice_hrn'] = lease['slice_hrn'] + slicerec_dict['hrn'] = lease['slice_hrn'] + slicerec_dict['user'] = lease['user'] + slicerec_dict.update( + {'list_node_ids': + {'hostname': lease['reserved_nodes']}}) + slicerec_dict.update({'node_ids': lease['reserved_nodes']}) + + + + return_slicerec_dictlist.append(slicerec_dict) + logger.debug("CORTEXLAB_API.PY \tGetSlices \ + OHOHOHOH %s" %(return_slicerec_dictlist)) + + logger.debug("CORTEXLAB_API.PY \tGetSlices \ + slicerec_dict %s return_slicerec_dictlist %s \ + lease['reserved_nodes'] \ + %s" % (slicerec_dict, return_slicerec_dictlist, + lease['reserved_nodes'])) + + logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ + return_slicerec_dictlist %s" + % (return_slicerec_dictlist)) + + return return_slicerec_dictlist + + + else: + #Get all slices from the iotlab sfa database , + #put them in dict format + #query_slice_list = dbsession.query(RegRecord).all() + query_slice_list = \ + dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() + + for record in query_slice_list: + tmp = record.__dict__ + tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__ + #del tmp['reg_researchers']['_sa_instance_state'] + return_slicerec_dictlist.append(tmp) + #return_slicerec_dictlist.append(record.__dict__) + + #Get all the jobs reserved nodes + leases_list = self.GetReservedNodes() + + for fixed_slicerec_dict in return_slicerec_dictlist: + slicerec_dict = {} + #Check if the slice belongs to a iotlab user + if fixed_slicerec_dict['peer_authority'] is None: + owner = fixed_slicerec_dict['hrn'].split( + ".")[1].split("_")[0] + else: + owner = None + for lease in leases_list: + if owner == lease['user']: + slicerec_dict['oar_job_id'] = lease['lease_id'] + + #for reserved_node in lease['reserved_nodes']: + logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s " + % (lease)) + slicerec_dict.update(fixed_slicerec_dict) + slicerec_dict.update({'node_ids': + lease['reserved_nodes']}) + slicerec_dict.update({'list_node_ids': + {'hostname': + lease['reserved_nodes']}}) + + #slicerec_dict.update({'hrn':\ + #str(fixed_slicerec_dict['slice_hrn'])}) + #return_slicerec_dictlist.append(slicerec_dict) + fixed_slicerec_dict.update(slicerec_dict) + + logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ + return_slicerec_dictlist %s \slice_filter %s " \ + %(return_slicerec_dictlist, slice_filter)) + + return return_slicerec_dictlist + + + + #Update slice unused, therefore sfa_fields_to_iotlab_fields unused + #SA 30/05/13 + #@staticmethod + #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record): + #""" + #""" + + #iotlab_record = {} + ##for field in record: + ## iotlab_record[field] = record[field] + + #if sfa_type == "slice": + ##instantion used in get_slivers ? + #if not "instantiation" in iotlab_record: + #iotlab_record["instantiation"] = "iotlab-instantiated" + ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn) + ##Unused hrn_to_pl_slicename because Iotlab's hrn already + ##in the appropriate form SA 23/07/12 + #iotlab_record["hrn"] = hrn + #logger.debug("CORTEXLAB_API.PY sfa_fields_to_iotlab_fields \ + #iotlab_record %s " %(iotlab_record['hrn'])) + #if "url" in record: + #iotlab_record["url"] = record["url"] + #if "description" in record: + #iotlab_record["description"] = record["description"] + #if "expires" in record: + #iotlab_record["expires"] = int(record["expires"]) + + ##nodes added by OAR only and then imported to SFA + ##elif type == "node": + ##if not "hostname" in iotlab_record: + ##if not "hostname" in record: + ##raise MissingSfaInfo("hostname") + ##iotlab_record["hostname"] = record["hostname"] + ##if not "model" in iotlab_record: + ##iotlab_record["model"] = "geni" + + ##One authority only + ##elif type == "authority": + ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn) + + ##if not "name" in iotlab_record: + ##iotlab_record["name"] = hrn + + ##if not "abbreviated_name" in iotlab_record: + ##iotlab_record["abbreviated_name"] = hrn + + ##if not "enabled" in iotlab_record: + ##iotlab_record["enabled"] = True + + ##if not "is_public" in iotlab_record: + ##iotlab_record["is_public"] = True + + #return iotlab_record + + + + + + + + + + diff --git a/sfa/cortexlab/cortexlabdriver.py b/sfa/cortexlab/cortexlabdriver.py new file mode 100644 index 00000000..e99c725f --- /dev/null +++ b/sfa/cortexlab/cortexlabdriver.py @@ -0,0 +1,798 @@ +""" +Implements what a driver should provide for SFA to work. +""" +from sfa.util.faults import SliverDoesNotExist, UnknownSfaType +from sfa.util.sfalogging import logger +from sfa.storage.alchemy import dbsession +from sfa.storage.model import RegRecord + +from sfa.managers.driver import Driver +from sfa.rspecs.version_manager import VersionManager +from sfa.rspecs.rspec import RSpec + +from sfa.util.xrn import Xrn, hrn_to_urn, get_authority + +from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname +from sfa.iotlab.iotlabslices import IotlabSlices + + +from sfa.iotlab.iotlabapi import IotlabTestbedAPI + + +class CortexlabDriver(Driver): + """ Cortexlab Driver class inherited from Driver generic class. + + Contains methods compliant with the SFA standard and the testbed + infrastructure (calls to LDAP and scheduler to book the nodes). + + .. seealso::: Driver class + + """ + def __init__(self, config): + """ + + Sets the iotlab SFA config parameters, + instanciates the testbed api and the iotlab database. + + :param config: iotlab SFA configuration object + :type config: Config object + + """ + Driver.__init__(self, config) + self.config = config + self.iotlab_api = IotlabTestbedAPI(config) + self.cache = None + + def augment_records_with_testbed_info(self, record_list): + """ + + Adds specific testbed info to the records. + + :param record_list: list of sfa dictionaries records + :type record_list: list + :returns: list of records with extended information in each record + :rtype: list + + """ + return self.fill_record_info(record_list) + + def fill_record_info(self, record_list): + """ + + For each SFA record, fill in the iotlab specific and SFA specific + fields in the record. + + :param record_list: list of sfa dictionaries records + :type record_list: list + :returns: list of records with extended information in each record + :rtype: list + + .. warning:: Should not be modifying record_list directly because modi + fication are kept outside the method's scope. Howerver, there is no + other way to do it given the way it's called in registry manager. + + """ + + logger.debug("IOTLABDRIVER \tfill_record_info records %s " + % (record_list)) + if not isinstance(record_list, list): + record_list = [record_list] + + try: + for record in record_list: + + if str(record['type']) == 'node': + # look for node info using GetNodes + # the record is about one node only + filter_dict = {'hrn': [record['hrn']]} + node_info = self.iotlab_api.GetNodes(filter_dict) + # the node_info is about one node only, but it is formatted + # as a list + record.update(node_info[0]) + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info NODE" % (record)) + + #If the record is a SFA slice record, then add information + #about the user of this slice. This kind of + #information is in the Iotlab's DB. + if str(record['type']) == 'slice': + if 'reg_researchers' in record and isinstance(record + ['reg_researchers'], + list): + record['reg_researchers'] = \ + record['reg_researchers'][0].__dict__ + record.update( + {'PI': [record['reg_researchers']['hrn']], + 'researcher': [record['reg_researchers']['hrn']], + 'name': record['hrn'], + 'oar_job_id': [], + 'node_ids': [], + 'person_ids': [record['reg_researchers'] + ['record_id']], + # For client_helper.py compatibility + 'geni_urn': '', + # For client_helper.py compatibility + 'keys': '', + # For client_helper.py compatibility + 'key_ids': ''}) + + #Get iotlab slice record and oar job id if any. + recslice_list = self.iotlab_api.GetSlices( + slice_filter=str(record['hrn']), + slice_filter_type='slice_hrn') + + logger.debug("IOTLABDRIVER \tfill_record_info \ + TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\ + %s " % (record['hrn'], record['oar_job_id'])) + del record['reg_researchers'] + try: + for rec in recslice_list: + logger.debug("IOTLABDRIVER\r\n \t \ + fill_record_info oar_job_id %s " + % (rec['oar_job_id'])) + + record['node_ids'] = [self.iotlab_api.root_auth + + '.' + hostname for hostname + in rec['node_ids']] + except KeyError: + pass + + logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \ + recslice_list %s \r\n \t RECORD %s \r\n \ + \r\n" % (recslice_list, record)) + + if str(record['type']) == 'user': + #The record is a SFA user record. + #Get the information about his slice from Iotlab's DB + #and add it to the user record. + recslice_list = self.iotlab_api.GetSlices( + slice_filter=record['record_id'], + slice_filter_type='record_id_user') + + logger.debug("IOTLABDRIVER.PY \t fill_record_info \ + TYPE USER recslice_list %s \r\n \t RECORD %s \r\n" + % (recslice_list, record)) + #Append slice record in records list, + #therefore fetches user and slice info again(one more loop) + #Will update PIs and researcher for the slice + + recuser = recslice_list[0]['reg_researchers'] + logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \ + recuser %s \r\n \r\n" % (recuser)) + recslice = {} + recslice = recslice_list[0] + recslice.update( + {'PI': [recuser['hrn']], + 'researcher': [recuser['hrn']], + 'name': record['hrn'], + 'node_ids': [], + 'oar_job_id': [], + 'person_ids': [recuser['record_id']]}) + try: + for rec in recslice_list: + recslice['oar_job_id'].append(rec['oar_job_id']) + except KeyError: + pass + + recslice.update({'type': 'slice', + 'hrn': recslice_list[0]['hrn']}) + + #GetPersons takes [] as filters + user_iotlab = self.iotlab_api.GetPersons([record]) + + record.update(user_iotlab[0]) + #For client_helper.py compatibility + record.update( + {'geni_urn': '', + 'keys': '', + 'key_ids': ''}) + record_list.append(recslice) + + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info ADDING SLICE\ + INFO TO USER records %s" % (record_list)) + + except TypeError, error: + logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s" + % (error)) + + return record_list + + def sliver_status(self, slice_urn, slice_hrn): + """ + Receive a status request for slice named urn/hrn + urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice + shall return a structure as described in + http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus + NT : not sure if we should implement this or not, but used by sface. + + :param slice_urn: slice urn + :type slice_urn: string + :param slice_hrn: slice hrn + :type slice_hrn: string + + """ + + #First get the slice with the slice hrn + slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn, + slice_filter_type='slice_hrn') + + if len(slice_list) == 0: + raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn)) + + #Used for fetching the user info witch comes along the slice info + one_slice = slice_list[0] + + #Make a list of all the nodes hostnames in use for this slice + slice_nodes_list = [] + slice_nodes_list = one_slice['node_ids'] + #Get all the corresponding nodes details + nodes_all = self.iotlab_api.GetNodes( + {'hostname': slice_nodes_list}, + ['node_id', 'hostname', 'site', 'boot_state']) + nodeall_byhostname = dict([(one_node['hostname'], one_node) + for one_node in nodes_all]) + + for single_slice in slice_list: + #For compatibility + top_level_status = 'empty' + result = {} + result.fromkeys( + ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status', + 'geni_resources'], None) + # result.fromkeys(\ + # ['geni_urn','geni_error', 'pl_login','geni_status', + # 'geni_resources'], None) + # result['pl_login'] = one_slice['reg_researchers'][0].hrn + result['iotlab_login'] = one_slice['user'] + logger.debug("Slabdriver - sliver_status Sliver status \ + urn %s hrn %s single_slice %s \r\n " + % (slice_urn, slice_hrn, single_slice)) + + if 'node_ids' not in single_slice: + #No job in the slice + result['geni_status'] = top_level_status + result['geni_resources'] = [] + return result + + top_level_status = 'ready' + + #A job is running on Iotlab for this slice + # report about the local nodes that are in the slice only + + result['geni_urn'] = slice_urn + + resources = [] + for node_hostname in single_slice['node_ids']: + res = {} + res['iotlab_hostname'] = node_hostname + res['iotlab_boot_state'] = \ + nodeall_byhostname[node_hostname]['boot_state'] + + #res['pl_hostname'] = node['hostname'] + #res['pl_boot_state'] = \ + #nodeall_byhostname[node['hostname']]['boot_state'] + #res['pl_last_contact'] = strftime(self.time_format, \ + #gmtime(float(timestamp))) + sliver_id = Xrn( + slice_urn, type='slice', + id=nodeall_byhostname[node_hostname]['node_id']).urn + + res['geni_urn'] = sliver_id + #node_name = node['hostname'] + if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive': + + res['geni_status'] = 'ready' + else: + res['geni_status'] = 'failed' + top_level_status = 'failed' + + res['geni_error'] = '' + + resources.append(res) + + result['geni_status'] = top_level_status + result['geni_resources'] = resources + logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s " + % (resources, res)) + return result + + @staticmethod + def get_user_record(hrn): + """ + + Returns the user record based on the hrn from the SFA DB . + + :param hrn: user's hrn + :type hrn: string + :returns: user record from SFA database + :rtype: RegUser + + """ + return dbsession.query(RegRecord).filter_by(hrn=hrn).first() + + def testbed_name(self): + """ + + Returns testbed's name. + :returns: testbed authority name. + :rtype: string + + """ + return self.hrn + + # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory + def aggregate_version(self): + """ + + Returns the testbed's supported rspec advertisement and request + versions. + :returns: rspec versions supported ad a dictionary. + :rtype: dict + + """ + version_manager = VersionManager() + ad_rspec_versions = [] + request_rspec_versions = [] + for rspec_version in version_manager.versions: + if rspec_version.content_type in ['*', 'ad']: + ad_rspec_versions.append(rspec_version.to_dict()) + if rspec_version.content_type in ['*', 'request']: + request_rspec_versions.append(rspec_version.to_dict()) + return { + 'testbed': self.testbed_name(), + 'geni_request_rspec_versions': request_rspec_versions, + 'geni_ad_rspec_versions': ad_rspec_versions} + + def _get_requested_leases_list(self, rspec): + """ + Process leases in rspec depending on the rspec version (format) + type. Find the lease requests in the rspec and creates + a lease request list with the mandatory information ( nodes, + start time and duration) of the valid leases (duration above or + equal to the iotlab experiment minimum duration). + + :param rspec: rspec request received. + :type rspec: RSpec + :returns: list of lease requests found in the rspec + :rtype: list + """ + requested_lease_list = [] + for lease in rspec.version.get_leases(): + single_requested_lease = {} + logger.debug("IOTLABDRIVER.PY \t \ + _get_requested_leases_list lease %s " % (lease)) + + if not lease.get('lease_id'): + if get_authority(lease['component_id']) == \ + self.iotlab_api.root_auth: + single_requested_lease['hostname'] = \ + iotlab_xrn_to_hostname(\ + lease.get('component_id').strip()) + single_requested_lease['start_time'] = \ + lease.get('start_time') + single_requested_lease['duration'] = lease.get('duration') + #Check the experiment's duration is valid before adding + #the lease to the requested leases list + duration_in_seconds = \ + int(single_requested_lease['duration']) + if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInGranularity(): + requested_lease_list.append(single_requested_lease) + + return requested_lease_list + + @staticmethod + def _group_leases_by_start_time(requested_lease_list): + """ + Create dict of leases by start_time, regrouping nodes reserved + at the same time, for the same amount of time so as to + define one job on OAR. + + :param requested_lease_list: list of leases + :type requested_lease_list: list + :returns: Dictionary with key = start time, value = list of leases + with the same start time. + :rtype: dictionary + + """ + + requested_job_dict = {} + for lease in requested_lease_list: + + #In case it is an asap experiment start_time is empty + if lease['start_time'] == '': + lease['start_time'] = '0' + + if lease['start_time'] not in requested_job_dict: + if isinstance(lease['hostname'], str): + lease['hostname'] = [lease['hostname']] + + requested_job_dict[lease['start_time']] = lease + + else: + job_lease = requested_job_dict[lease['start_time']] + if lease['duration'] == job_lease['duration']: + job_lease['hostname'].append(lease['hostname']) + + return requested_job_dict + + def _process_requested_jobs(self, rspec): + """ + Turns the requested leases and information into a dictionary + of requested jobs, grouped by starting time. + + :param rspec: RSpec received + :type rspec : RSpec + :rtype: dictionary + + """ + requested_lease_list = self._get_requested_leases_list(rspec) + logger.debug("IOTLABDRIVER _process_requested_jobs \ + requested_lease_list %s" % (requested_lease_list)) + job_dict = self._group_leases_by_start_time(requested_lease_list) + logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\ + %s" % (job_dict)) + + return job_dict + + def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string, + users, options): + """Answer to CreateSliver. + + Creates the leases and slivers for the users from the information + found in the rspec string. + Launch experiment on OAR if the requested leases is valid. Delete + no longer requested leases. + + + :param creds: user's credentials + :type creds: string + :param users: user record list + :type users: list + :param options: + :type options: + + :returns: a valid Rspec for the slice which has just been + modified. + :rtype: RSpec + + + """ + aggregate = IotlabAggregate(self) + + slices = IotlabSlices(self) + peer = slices.get_peer(slice_hrn) + sfa_peer = slices.get_sfa_peer(slice_hrn) + slice_record = None + + if not isinstance(creds, list): + creds = [creds] + + if users: + slice_record = users[0].get('slice_record', {}) + logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\ + creds %s \r\n \r\n users %s" + % (creds, users)) + slice_record['user'] = {'keys': users[0]['keys'], + 'email': users[0]['email'], + 'hrn': slice_record['reg-researchers'][0]} + # parse rspec + rspec = RSpec(rspec_string) + logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \ + %s slice_record %s users %s" + % (rspec.version, slice_record, users)) + + # ensure site record exists? + # ensure slice record exists + #Removed options in verify_slice SA 14/08/12 + #Removed peer record in verify_slice SA 18/07/13 + sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer) + + # ensure person records exists + #verify_persons returns added persons but the return value + #is not used + #Removed peer record and sfa_peer in verify_persons SA 18/07/13 + slices.verify_persons(slice_hrn, sfa_slice, users, options=options) + #requested_attributes returned by rspec.version.get_slice_attributes() + #unused, removed SA 13/08/12 + #rspec.version.get_slice_attributes() + + logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice)) + + # add/remove slice from nodes + + #requested_slivers = [node.get('component_id') \ + #for node in rspec.version.get_nodes_with_slivers()\ + #if node.get('authority_id') is self.iotlab_api.root_auth] + #l = [ node for node in rspec.version.get_nodes_with_slivers() ] + #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \ + #requested_slivers %s listnodes %s" \ + #%(requested_slivers,l)) + #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. + #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) + + requested_job_dict = self._process_requested_jobs(rspec) + + logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s " + % (requested_job_dict)) + #verify_slice_leases returns the leases , but the return value is unused + #here. Removed SA 13/08/12 + slices.verify_slice_leases(sfa_slice, + requested_job_dict, peer) + + return aggregate.get_rspec(slice_xrn=slice_urn, + login=sfa_slice['login'], + version=rspec.version) + + def delete_sliver(self, slice_urn, slice_hrn, creds, options): + """ + Deletes the lease associated with the slice hrn and the credentials + if the slice belongs to iotlab. Answer to DeleteSliver. + + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused + + :returns: 1 if the slice to delete was not found on iotlab, + True if the deletion was successful, False otherwise otherwise. + + .. note:: Should really be named delete_leases because iotlab does + not have any slivers, but only deals with leases. However, + SFA api only have delete_sliver define so far. SA 13/05/2013 + .. note:: creds are unused, and are not used either in the dummy driver + delete_sliver . + """ + + sfa_slice_list = self.iotlab_api.GetSlices( + slice_filter=slice_hrn, + slice_filter_type='slice_hrn') + + if not sfa_slice_list: + return 1 + + #Delete all leases in the slice + for sfa_slice in sfa_slice_list: + logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice)) + slices = IotlabSlices(self) + # determine if this is a peer slice + + peer = slices.get_peer(slice_hrn) + + logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \ + \r\n \t sfa_slice %s " % (peer, sfa_slice)) + try: + self.iotlab_api.DeleteSliceFromNodes(sfa_slice) + return True + except: + return False + + def list_resources (self, slice_urn, slice_hrn, creds, options): + """ + + List resources from the iotlab aggregate and returns a Rspec + advertisement with resources found when slice_urn and slice_hrn are + None (in case of resource discovery). + If a slice hrn and urn are provided, list experiment's slice + nodes in a rspec format. Answer to ListResources. + Caching unused. + + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused + :param options: options used when listing resources (list_leases, info, + geni_available) + :returns: rspec string in xml + :rtype: string + + .. note:: creds are unused + """ + + #cached_requested = options.get('cached', True) + + version_manager = VersionManager() + # get the rspec's return format from options + rspec_version = \ + version_manager.get_version(options.get('geni_rspec_version')) + version_string = "rspec_%s" % (rspec_version) + + #panos adding the info option to the caching key (can be improved) + if options.get('info'): + version_string = version_string + "_" + \ + options.get('info', 'default') + + # Adding the list_leases option to the caching key + if options.get('list_leases'): + version_string = version_string + "_" + \ + options.get('list_leases', 'default') + + # Adding geni_available to caching key + if options.get('geni_available'): + version_string = version_string + "_" + \ + str(options.get('geni_available')) + + # look in cache first + #if cached_requested and self.cache and not slice_hrn: + #rspec = self.cache.get(version_string) + #if rspec: + #logger.debug("IotlabDriver.ListResources: \ + #returning cached advertisement") + #return rspec + + #panos: passing user-defined options + aggregate = IotlabAggregate(self) + + rspec = aggregate.get_rspec(slice_xrn=slice_urn, + version=rspec_version, options=options) + + # cache the result + #if self.cache and not slice_hrn: + #logger.debug("Iotlab.ListResources: stores advertisement in cache") + #self.cache.add(version_string, rspec) + + return rspec + + + def list_slices(self, creds, options): + """Answer to ListSlices. + + List slices belonging to iotlab, returns slice urns list. + No caching used. Options unused but are defined in the SFA method + api prototype. + + :returns: slice urns list + :rtype: list + + .. note:: creds are unused + """ + # look in cache first + #if self.cache: + #slices = self.cache.get('slices') + #if slices: + #logger.debug("PlDriver.list_slices returns from cache") + #return slices + + # get data from db + + slices = self.iotlab_api.GetSlices() + logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" + % (slices)) + slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices] + + slice_urns = [hrn_to_urn(slice_hrn, 'slice') + for slice_hrn in slice_hrns] + + # cache the result + #if self.cache: + #logger.debug ("IotlabDriver.list_slices stores value in cache") + #self.cache.add('slices', slice_urns) + + return slice_urns + + + def register(self, sfa_record, hrn, pub_key): + """ + Adding new user, slice, node or site should not be handled + by SFA. + + ..warnings:: should not be used. Different components are in charge of + doing this task. Adding nodes = OAR + Adding users = LDAP Iotlab + Adding slice = Import from LDAP users + Adding site = OAR + + :param sfa_record: record provided by the client of the + Register API call. + :type sfa_record: dict + :param pub_key: public key of the user + :type pub_key: string + + .. note:: DOES NOTHING. Returns -1. + + """ + return -1 + + + def update(self, old_sfa_record, new_sfa_record, hrn, new_key): + """ + No site or node record update allowed in Iotlab. The only modifications + authorized here are key deletion/addition on an existing user and + password change. On an existing user, CAN NOT BE MODIFIED: 'first_name', + 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio', + 'title', 'accepted_aup'. A slice is bound to its user, so modifying the + user's ssh key should nmodify the slice's GID after an import procedure. + + :param old_sfa_record: what is in the db for this hrn + :param new_sfa_record: what was passed to the update call + :param new_key: the new user's public key + :param hrn: the user's sfa hrn + :type old_sfa_record: dict + :type new_sfa_record: dict + :type new_key: string + :type hrn: string + + TODO: needs review + .. seealso:: update in driver.py. + + """ + pointer = old_sfa_record['pointer'] + old_sfa_record_type = old_sfa_record['type'] + + # new_key implemented for users only + if new_key and old_sfa_record_type not in ['user']: + raise UnknownSfaType(old_sfa_record_type) + + if old_sfa_record_type == "user": + update_fields = {} + all_fields = new_sfa_record + for key in all_fields.keys(): + if key in ['key', 'password']: + update_fields[key] = all_fields[key] + + if new_key: + # must check this key against the previous one if it exists + persons = self.iotlab_api.GetPersons([old_sfa_record]) + person = persons[0] + keys = [person['pkey']] + #Get all the person's keys + keys_dict = self.iotlab_api.GetKeys(keys) + + # Delete all stale keys, meaning the user has only one key + #at a time + #TODO: do we really want to delete all the other keys? + #Is this a problem with the GID generation to have multiple + #keys? SA 30/05/13 + key_exists = False + if key in keys_dict: + key_exists = True + else: + #remove all the other keys + for key in keys_dict: + self.iotlab_api.DeleteKey(person, key) + self.iotlab_api.AddPersonKey( + person, {'sshPublicKey': person['pkey']}, + {'sshPublicKey': new_key}) + return True + + def remove(self, sfa_record): + """ + + Removes users only. Mark the user as disabled in LDAP. The user and his + slice are then deleted from the db by running an import on the registry. + + :param sfa_record: record is the existing sfa record in the db + :type sfa_record: dict + + ..warning::As fas as the slice is concerned, here only the leases are + removed from the slice. The slice is record itself is not removed + from the db. + + TODO: needs review + + TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, + + TODO: return boolean for the slice part + """ + sfa_record_type = sfa_record['type'] + hrn = sfa_record['hrn'] + if sfa_record_type == 'user': + + #get user from iotlab ldap + person = self.iotlab_api.GetPersons(sfa_record) + #No registering at a given site in Iotlab. + #Once registered to the LDAP, all iotlab sites are + #accesible. + if person: + #Mark account as disabled in ldap + return self.iotlab_api.DeletePerson(sfa_record) + + elif sfa_record_type == 'slice': + if self.iotlab_api.GetSlices(slice_filter=hrn, + slice_filter_type='slice_hrn'): + ret = self.iotlab_api.DeleteSlice(sfa_record) + return True diff --git a/sfa/cortexlab/cortexlabpostgres.py b/sfa/cortexlab/cortexlabpostgres.py new file mode 100644 index 00000000..cd4fc585 --- /dev/null +++ b/sfa/cortexlab/cortexlabpostgres.py @@ -0,0 +1,264 @@ +""" +File defining classes to handle the table in the iotlab dedicated database. +""" + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +# from sfa.util.config import Config +from sfa.util.sfalogging import logger + +from sqlalchemy import Column, Integer, String +from sqlalchemy import Table, MetaData +from sqlalchemy.ext.declarative import declarative_base + +# from sqlalchemy.dialects import postgresql + +from sqlalchemy.exc import NoSuchTableError + + +#Dict holding the columns names of the table as keys +#and their type, used for creation of the table +slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \ + CASCADE ON UPDATE CASCADE', 'oar_job_id': 'integer DEFAULT -1', + 'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'} + +#Dict with all the specific iotlab tables +tablenames_dict = {'iotlab_xp': slice_table} + + +IotlabBase = declarative_base() + + +class IotlabXP (IotlabBase): + """ SQL alchemy class to manipulate the rows of the slice_iotlab table in + iotlab_sfa database. Handles the records representation and creates the + table if it does not exist yet. + + """ + __tablename__ = 'iotlab_xp' + + slice_hrn = Column(String) + job_id = Column(Integer, primary_key=True) + end_time = Column(Integer, nullable=False) + + def __init__(self, slice_hrn=None, job_id=None, end_time=None): + """ + Defines a row of the slice_iotlab table + """ + if slice_hrn: + self.slice_hrn = slice_hrn + if job_id: + self.job_id = job_id + if end_time: + self.end_time = end_time + + def __repr__(self): + """Prints the SQLAlchemy record to the format defined + by the function. + """ + result = " 0: + self.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + self.iotlab_session.commit() + return + + def __init__(self, config, debug=False): + self.sl_base = IotlabBase + + # Check whether we already have an instance + if IotlabDB._connection_singleton is None: + IotlabDB._connection_singleton = IotlabDB.Singleton(config, debug) + + # Store instance reference as the only member in the handle + self._EventHandler_singleton = IotlabDB._connection_singleton + + def __getattr__(self, aAttr): + """ + Delegate access to implementation. + + :param aAttr: Attribute wanted. + :returns: Attribute + """ + return getattr(self._connection_singleton, aAttr) + + + + # def __setattr__(self, aAttr, aValue): + # """Delegate access to implementation. + + # :param attr: Attribute wanted. + # :param value: Vaule to be set. + # :return: Result of operation. + # """ + # return setattr(self._connection_singleton, aAttr, aValue) + + def exists(self, tablename): + """ + Checks if the table specified as tablename exists. + :param tablename: name of the table in the db that has to be checked. + :type tablename: string + :returns: True if the table exists, False otherwise. + :rtype: bool + + """ + metadata = MetaData(bind=self.iotlab_engine) + try: + table = Table(tablename, metadata, autoload=True) + return True + + except NoSuchTableError: + logger.log_exc("IOTLABPOSTGRES tablename %s does not exist" + % (tablename)) + return False + + def createtable(self): + """ + Creates all the table sof the engine. + Uses the global dictionnary holding the tablenames and the table schema. + + """ + + logger.debug("IOTLABPOSTGRES createtable \ + IotlabBase.metadata.sorted_tables %s \r\n engine %s" + % (IotlabBase.metadata.sorted_tables, self.iotlab_engine)) + IotlabBase.metadata.create_all(self.iotlab_engine) + return diff --git a/sfa/cortexlab/cortexlabslices.py b/sfa/cortexlab/cortexlabslices.py new file mode 100644 index 00000000..2ef77154 --- /dev/null +++ b/sfa/cortexlab/cortexlabslices.py @@ -0,0 +1,587 @@ +""" +This file defines the IotlabSlices class by which all the slice checkings +upon lease creation are done. +""" +from sfa.util.xrn import get_authority, urn_to_hrn +from sfa.util.sfalogging import logger + +MAXINT = 2L**31-1 + + +class IotlabSlices: + """ + This class is responsible for checking the slice when creating a + lease or a sliver. Those checks include verifying that the user is valid, + that the slice is known from the testbed or from our peers, that the list + of nodes involved has not changed (in this case the lease is modified + accordingly). + """ + rspec_to_slice_tag = {'max_rate': 'net_max_rate'} + + def __init__(self, driver): + """ + Get the reference to the driver here. + """ + self.driver = driver + + def get_peer(self, xrn): + """ + Finds the authority of a resource based on its xrn. + If the authority is Iotlab (local) return None, + Otherwise, look up in the DB if Iotlab is federated with this site + authority and returns its DB record if it is the case. + + :param xrn: resource's xrn + :type xrn: string + :returns: peer record + :rtype: dict + + """ + hrn, hrn_type = urn_to_hrn(xrn) + #Does this slice belong to a local site or a peer iotlab site? + peer = None + + # get this slice's authority (site) + slice_authority = get_authority(hrn) + #Iotlab stuff + #This slice belongs to the current site + if slice_authority == self.driver.iotlab_api.root_auth: + site_authority = slice_authority + return None + + site_authority = get_authority(slice_authority).lower() + # get this site's authority (sfa root authority or sub authority) + + logger.debug("IOTLABSLICES \t get_peer slice_authority %s \ + site_authority %s hrn %s" + % (slice_authority, site_authority, hrn)) + + # check if we are already peered with this site_authority + #if so find the peer record + peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority) + for peer_record in peers: + if site_authority == peer_record.hrn: + peer = peer_record + logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer)) + return peer + + def get_sfa_peer(self, xrn): + """Returns the authority name for the xrn or None if the local site + is the authority. + + :param xrn: the xrn of the resource we are looking the authority for. + :type xrn: string + :returns: the resources's authority name. + :rtype: string + + """ + hrn, hrn_type = urn_to_hrn(xrn) + + # return the authority for this hrn or None if we are the authority + sfa_peer = None + slice_authority = get_authority(hrn) + site_authority = get_authority(slice_authority) + + if site_authority != self.driver.hrn: + sfa_peer = site_authority + + return sfa_peer + + def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer): + """ + Compare requested leases with the leases already scheduled/ + running in OAR. If necessary, delete and recreate modified leases, + and delete no longer requested ones. + + :param sfa_slice: sfa slice record + :param requested_jobs_dict: dictionary of requested leases + :param peer: sfa peer record + + :type sfa_slice: dict + :type requested_jobs_dict: dict + :type peer: dict + :returns: leases list of dictionary + :rtype: list + + """ + + logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s " + % (sfa_slice)) + #First get the list of current leases from OAR + leases = self.driver.iotlab_api.GetLeases({'slice_hrn': sfa_slice['hrn']}) + logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \ + leases %s " % (requested_jobs_dict, leases)) + + current_nodes_reserved_by_start_time = {} + requested_nodes_by_start_time = {} + leases_by_start_time = {} + reschedule_jobs_dict = {} + + #Create reduced dictionary with key start_time and value + # the list of nodes + #-for the leases already registered by OAR first + # then for the new leases requested by the user + + #Leases already scheduled/running in OAR + for lease in leases: + current_nodes_reserved_by_start_time[lease['t_from']] = \ + lease['reserved_nodes'] + leases_by_start_time[lease['t_from']] = lease + + #First remove job whose duration is too short + for job in requested_jobs_dict.values(): + job['duration'] = \ + str(int(job['duration']) \ + * self.driver.iotlab_api.GetLeaseGranularity()) + if job['duration'] < self.driver.iotlab_api.GetLeaseGranularity(): + del requested_jobs_dict[job['start_time']] + + #Requested jobs + for start_time in requested_jobs_dict: + requested_nodes_by_start_time[int(start_time)] = \ + requested_jobs_dict[start_time]['hostname'] + #Check if there is any difference between the leases already + #registered in OAR and the requested jobs. + #Difference could be: + #-Lease deleted in the requested jobs + #-Added/removed nodes + #-Newly added lease + + logger.debug("IOTLABSLICES verify_slice_leases \ + requested_nodes_by_start_time %s \ + "% (requested_nodes_by_start_time)) + #Find all deleted leases + start_time_list = \ + list(set(leases_by_start_time.keys()).\ + difference(requested_nodes_by_start_time.keys())) + deleted_leases = [leases_by_start_time[start_time]['lease_id'] \ + for start_time in start_time_list] + + + #Find added or removed nodes in exisiting leases + for start_time in requested_nodes_by_start_time: + logger.debug("IOTLABSLICES verify_slice_leases start_time %s \ + "%( start_time)) + if start_time in current_nodes_reserved_by_start_time: + + if requested_nodes_by_start_time[start_time] == \ + current_nodes_reserved_by_start_time[start_time]: + continue + + else: + update_node_set = \ + set(requested_nodes_by_start_time[start_time]) + added_nodes = \ + update_node_set.difference(\ + current_nodes_reserved_by_start_time[start_time]) + shared_nodes = \ + update_node_set.intersection(\ + current_nodes_reserved_by_start_time[start_time]) + old_nodes_set = \ + set(\ + current_nodes_reserved_by_start_time[start_time]) + removed_nodes = \ + old_nodes_set.difference(\ + requested_nodes_by_start_time[start_time]) + logger.debug("IOTLABSLICES verify_slice_leases \ + shared_nodes %s added_nodes %s removed_nodes %s"\ + %(shared_nodes, added_nodes,removed_nodes )) + #If the lease is modified, delete it before + #creating it again. + #Add the deleted lease job id in the list + #WARNING :rescheduling does not work if there is already + # 2 running/scheduled jobs because deleting a job + #takes time SA 18/10/2012 + if added_nodes or removed_nodes: + deleted_leases.append(\ + leases_by_start_time[start_time]['lease_id']) + #Reschedule the job + if added_nodes or shared_nodes: + reschedule_jobs_dict[str(start_time)] = \ + requested_jobs_dict[str(start_time)] + + else: + #New lease + + job = requested_jobs_dict[str(start_time)] + logger.debug("IOTLABSLICES \ + NEWLEASE slice %s job %s" + % (sfa_slice, job)) + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) + + #Deleted leases are the ones with lease id not declared in the Rspec + if deleted_leases: + self.driver.iotlab_api.DeleteLeases(deleted_leases, + sfa_slice['user']['uid']) + logger.debug("IOTLABSLICES \ + verify_slice_leases slice %s deleted_leases %s" + % (sfa_slice, deleted_leases)) + + if reschedule_jobs_dict: + for start_time in reschedule_jobs_dict: + job = reschedule_jobs_dict[start_time] + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) + return leases + + def verify_slice_nodes(self, sfa_slice, requested_slivers, peer): + """Check for wanted and unwanted nodes in the slice. + + Removes nodes and associated leases that the user does not want anymore + by deleteing the associated job in OAR (DeleteSliceFromNodes). + Returns the nodes' hostnames that are going to be in the slice. + + :param sfa_slice: slice record. Must contain node_ids and list_node_ids. + + :param requested_slivers: list of requested nodes' hostnames. + :param peer: unused so far. + + :type sfa_slice: dict + :type requested_slivers: list + :type peer: string + + :returns: list requested nodes hostnames + :rtype: list + + .. warning:: UNUSED SQA 24/07/13 + .. seealso:: DeleteSliceFromNodes + .. todo:: check what to do with the peer? Can not remove peer nodes from + slice here. Anyway, in this case, the peer should have gotten the + remove request too. + + """ + current_slivers = [] + deleted_nodes = [] + + if 'node_ids' in sfa_slice: + nodes = self.driver.iotlab_api.GetNodes( + sfa_slice['list_node_ids'], + ['hostname']) + current_slivers = [node['hostname'] for node in nodes] + + # remove nodes not in rspec + deleted_nodes = list(set(current_slivers). + difference(requested_slivers)) + + logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\ + \r\n \r\n deleted_nodes %s" + % (sfa_slice, deleted_nodes)) + + if deleted_nodes: + #Delete the entire experience + self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice) + return nodes + + def verify_slice(self, slice_hrn, slice_record, sfa_peer): + """Ensures slice record exists. + + The slice record must exist either in Iotlab or in the other + federated testbed (sfa_peer). If the slice does not belong to Iotlab, + check if the user already exists in LDAP. In this case, adds the slice + to the sfa DB and associates its LDAP user. + + :param slice_hrn: slice's name + :param slice_record: sfa record of the slice + :param sfa_peer: name of the peer authority if any.(not Iotlab). + + :type slice_hrn: string + :type slice_record: dictionary + :type sfa_peer: string + + .. seealso:: AddSlice + + + """ + + slicename = slice_hrn + # check if slice belongs to Iotlab + slices_list = self.driver.iotlab_api.GetSlices( + slice_filter=slicename, slice_filter_type='slice_hrn') + + sfa_slice = None + + if slices_list: + for sl in slices_list: + + logger.debug("IOTLABSLICES \t verify_slice slicename %s \ + slices_list %s sl %s \r slice_record %s" + % (slicename, slices_list, sl, slice_record)) + sfa_slice = sl + sfa_slice.update(slice_record) + + else: + #Search for user in ldap based on email SA 14/11/12 + ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\ + slice_record['user']) + logger.debug(" IOTLABSLICES \tverify_slice Oups \ + slice_record %s sfa_peer %s ldap_user %s" + % (slice_record, sfa_peer, ldap_user)) + #User already registered in ldap, meaning user should be in SFA db + #and hrn = sfa_auth+ uid + sfa_slice = {'hrn': slicename, + 'node_list': [], + 'authority': slice_record['authority'], + 'gid': slice_record['gid'], + 'slice_id': slice_record['record_id'], + 'reg-researchers': slice_record['reg-researchers'], + 'peer_authority': str(sfa_peer) + } + + if ldap_user: + hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid'] + user = self.driver.get_user_record(hrn) + + logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" + % (hrn, user)) + + # add the external slice to the local SFA iotlab DB + if sfa_slice: + self.driver.iotlab_api.AddSlice(sfa_slice, user) + + logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK") + return sfa_slice + + + def verify_persons(self, slice_hrn, slice_record, users, options={}): + """Ensures the users in users list exist and are enabled in LDAP. Adds + person if needed. + + Checking that a user exist is based on the user's email. If the user is + still not found in the LDAP, it means that the user comes from another + federated testbed. In this case an account has to be created in LDAP + so as to enable the user to use the testbed, since we trust the testbed + he comes from. This is done by calling AddPerson. + + :param slice_hrn: slice name + :param slice_record: record of the slice_hrn + :param users: users is a record list. Records can either be + local records or users records from known and trusted federated + sites.If the user is from another site that iotlab doesn't trust yet, + then Resolve will raise an error before getting to create_sliver. + + :type slice_hrn: string + :type slice_record: string + :type users: list + + .. seealso:: AddPerson + .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13. + + + """ + #TODO SA 21/08/12 verify_persons Needs review + + logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \ + \t slice_record %s\r\n users %s \t " + % (slice_hrn, slice_record, users)) + users_by_id = {} + + users_by_email = {} + #users_dict : dict whose keys can either be the user's hrn or its id. + #Values contains only id and hrn + users_dict = {} + + #First create dicts by hrn and id for each user in the user record list: + for info in users: + if 'slice_record' in info: + slice_rec = info['slice_record'] + if 'user' in slice_rec : + user = slice_rec['user'] + + if 'email' in user: + users_by_email[user['email']] = user + users_dict[user['email']] = user + + logger.debug("IOTLABSLICES.PY \t verify_person \ + users_dict %s \r\n user_by_email %s \r\n \ + \tusers_by_id %s " + % (users_dict, users_by_email, users_by_id)) + + existing_user_ids = [] + existing_user_emails = [] + existing_users = [] + # Check if user is in Iotlab LDAP using its hrn. + # Assuming Iotlab is centralised : one LDAP for all sites, + # user's record_id unknown from LDAP + # LDAP does not provide users id, therefore we rely on email to find the + # user in LDAP + + if users_by_email: + #Construct the list of filters (list of dicts) for GetPersons + filter_user = [users_by_email[email] for email in users_by_email] + #Check user i in LDAP with GetPersons + #Needed because what if the user has been deleted in LDAP but + #is still in SFA? + existing_users = self.driver.iotlab_api.GetPersons(filter_user) + logger.debug(" \r\n IOTLABSLICES.PY \tverify_person filter_user \ + %s existing_users %s " + % (filter_user, existing_users)) + #User is in iotlab LDAP + if existing_users: + for user in existing_users: + users_dict[user['email']].update(user) + existing_user_emails.append( + users_dict[user['email']]['email']) + + + # User from another known trusted federated site. Check + # if a iotlab account matching the email has already been created. + else: + req = 'mail=' + if isinstance(users, list): + req += users[0]['email'] + else: + req += users['email'] + ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req) + + if ldap_reslt: + logger.debug(" IOTLABSLICES.PY \tverify_person users \ + USER already in Iotlab \t ldap_reslt %s \ + " % (ldap_reslt)) + existing_users.append(ldap_reslt[1]) + + else: + #User not existing in LDAP + logger.debug("IOTLABSLICES.PY \tverify_person users \ + not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \ + ldap_reslt %s " % (users, ldap_reslt)) + + requested_user_emails = users_by_email.keys() + requested_user_hrns = \ + [users_by_email[user]['hrn'] for user in users_by_email] + logger.debug("IOTLABSLICES.PY \tverify_person \ + users_by_email %s " % (users_by_email)) + + #Check that the user of the slice in the slice record + #matches one of the existing users + try: + if slice_record['PI'][0] in requested_user_hrns: + logger.debug(" IOTLABSLICES \tverify_person ['PI']\ + slice_record %s" % (slice_record)) + + except KeyError: + pass + + # users to be added, removed or updated + #One user in one iotlab slice : there should be no need + #to remove/ add any user from/to a slice. + #However a user from SFA which is not registered in Iotlab yet + #should be added to the LDAP. + added_user_emails = set(requested_user_emails).\ + difference(set(existing_user_emails)) + + + #self.verify_keys(existing_slice_users, updated_users_list, \ + #peer, append) + + added_persons = [] + # add new users + #requested_user_email is in existing_user_emails + if len(added_user_emails) == 0: + slice_record['login'] = users_dict[requested_user_emails[0]]['uid'] + logger.debug(" IOTLABSLICES \tverify_person QUICK DIRTY %s" + % (slice_record)) + + for added_user_email in added_user_emails: + added_user = users_dict[added_user_email] + logger.debug(" IOTLABSLICES \r\n \r\n \t verify_person \ + added_user %s" % (added_user)) + person = {} + person['peer_person_id'] = None + k_list = ['first_name', 'last_name', 'person_id'] + for k in k_list: + if k in added_user: + person[k] = added_user[k] + + person['pkey'] = added_user['keys'][0] + person['mail'] = added_user['email'] + person['email'] = added_user['email'] + person['key_ids'] = added_user.get('key_ids', []) + + ret = self.driver.iotlab_api.AddPerson(person) + if 'uid' in ret: + # meaning bool is True and the AddPerson was successful + person['uid'] = ret['uid'] + slice_record['login'] = person['uid'] + else: + # error message in ret + logger.debug(" IOTLABSLICES ret message %s" %(ret)) + + logger.debug(" IOTLABSLICES \r\n \r\n \t THE SECOND verify_person\ + person %s" % (person)) + #Update slice_Record with the id now known to LDAP + + + added_persons.append(person) + return added_persons + + + def verify_keys(self, persons, users, peer, options={}): + """ + .. warning:: unused + """ + # existing keys + key_ids = [] + for person in persons: + key_ids.extend(person['key_ids']) + keylist = self.driver.iotlab_api.GetKeys(key_ids, ['key_id', 'key']) + + keydict = {} + for key in keylist: + keydict[key['key']] = key['key_id'] + existing_keys = keydict.keys() + + persondict = {} + for person in persons: + persondict[person['email']] = person + + # add new keys + requested_keys = [] + updated_persons = [] + users_by_key_string = {} + for user in users: + user_keys = user.get('keys', []) + updated_persons.append(user) + for key_string in user_keys: + users_by_key_string[key_string] = user + requested_keys.append(key_string) + if key_string not in existing_keys: + key = {'key': key_string, 'key_type': 'ssh'} + #try: + ##if peer: + #person = persondict[user['email']] + #self.driver.iotlab_api.UnBindObjectFromPeer( + # 'person',person['person_id'], + # peer['shortname']) + ret = self.driver.iotlab_api.AddPersonKey( + user['email'], key) + #if peer: + #key_index = user_keys.index(key['key']) + #remote_key_id = user['key_ids'][key_index] + #self.driver.iotlab_api.BindObjectToPeer('key', \ + #key['key_id'], peer['shortname'], \ + #remote_key_id) + + #finally: + #if peer: + #self.driver.iotlab_api.BindObjectToPeer('person', \ + #person['person_id'], peer['shortname'], \ + #user['person_id']) + + # remove old keys (only if we are not appending) + append = options.get('append', True) + if append is False: + removed_keys = set(existing_keys).difference(requested_keys) + for key in removed_keys: + #if peer: + #self.driver.iotlab_api.UnBindObjectFromPeer('key', \ + #key, peer['shortname']) + + user = users_by_key_string[key] + self.driver.iotlab_api.DeleteKey(user, key) + + return -- 2.43.0