From: Sandrine Avakian Date: Mon, 24 Jun 2013 12:58:34 +0000 (+0200) Subject: SA X-Git-Tag: sfa-2.1-27~64 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=3fe7429faa466a0a229c462eb54dbf70bffc942b;p=sfa.git SA bug hunt --- diff --git a/sfa/importer/slabimporter.py b/sfa/importer/slabimporter.py index 60243a29..fdf1ba1f 100644 --- a/sfa/importer/slabimporter.py +++ b/sfa/importer/slabimporter.py @@ -16,37 +16,37 @@ from sqlalchemy.exc import SQLAlchemyError class SlabImporter: - """ + """ SlabImporter class, generic importer_class. Used to populate the SFA DB - with senslab resources' records. + with senslab resources' records. Used to update records when new resources, users or nodes, are added or deleted. """ - + def __init__ (self, auth_hierarchy, loc_logger): - """ - Sets and defines import logger and the authority name. Gathers all the + """ + Sets and defines import logger and the authority name. Gathers all the records already registerd in the SFA DB, broke them into 3 dicts, by type and hrn, by email and by type and pointer. - + :param auth_hierarchy: authority name :type auth_hierarchy: string :param loc_logger: local logger :type loc_logger: _SfaLogger - + """ self.auth_hierarchy = auth_hierarchy self.logger = loc_logger self.logger.setLevelDebug() #retrieve all existing SFA objects self.all_records = dbsession.query(RegRecord).all() - - # initialize record.stale to True by default, + + # initialize record.stale to True by default, # then mark stale=False on the ones that are in use - for record in self.all_records: + for record in self.all_records: record.stale = True - #create hash by (type,hrn) - #used to know if a given record is already known to SFA + #create hash by (type,hrn) + #used to know if a given record is already known to SFA self.records_by_type_hrn = \ dict([( (record.type,record.hrn), record) \ for record in self.all_records]) @@ -54,21 +54,21 @@ class SlabImporter: self.users_rec_by_email = \ dict([ (record.email, record) \ for record in self.all_records if record.type == 'user']) - - # create hash by (type,pointer) + + # create hash by (type,pointer) self.records_by_type_pointer = \ dict([ ( (str(record.type), record.pointer) , record) \ for record in self.all_records if record.pointer != -1]) - - - + + + @staticmethod def hostname_to_hrn_escaped(root_auth, hostname): - """ - - Returns a node's hrn based on its hostname and the root + """ + + Returns a node's hrn based on its hostname and the root authority and by removing special caracters from the hostname. - + :param root_auth: root authority name :param hostname: nodes's hostname :type root_auth: string @@ -81,59 +81,59 @@ class SlabImporter: @staticmethod def slicename_to_hrn(person_hrn): """ - + Returns the slicename associated to a given person's hrn. - + :param person_hrn: user's hrn :type person_hrn: string :rtype: string """ return (person_hrn +'_slice') - + def add_options (self, parser): # we don't have any options for now pass - + def find_record_by_type_hrn(self, record_type, hrn): """ - + Returns the record associated with a given hrn and hrn type. - Returns None if the key tuple is not in dictionary. - + Returns None if the key tuple is not in dictionary. + :param record_type: the record's type (slice, node, authority...) :type record_type: string :param hrn: Human readable name of the object's record :type hrn: string :rtype: RegUser if user, RegSlice if slice, RegNode if node... or None if record does not exist. - + """ return self.records_by_type_hrn.get ( (record_type, hrn), None) - + def locate_by_type_pointer (self, record_type, pointer): """ - + Returns the record corresponding to the key pointer and record type. Returns None if the record does not exist and is not in the records_by_type_pointer dictionnary. - + :param record_type: the record's type (slice, node, authority...) :type record_type: string - :param pointer:Pointer to where the record is in the origin db, + :param pointer:Pointer to where the record is in the origin db, used in case the record comes from a trusted authority. :type pointer: integer :rtype: RegUser if user, RegSlice if slice, RegNode if node... or None if record does not exist. """ return self.records_by_type_pointer.get ( (record_type, pointer), None) - - + + def update_just_added_records_dict (self, record): """ - - Updates the records_by_type_hrn dictionnary if the record has + + Updates the records_by_type_hrn dictionnary if the record has just been created. - + :param record: Record to add in the records_by_type_hrn dict. :type record: dictionary """ @@ -143,21 +143,21 @@ class SlabImporter: duplicate (%s,%s)"%rec_tuple) return self.records_by_type_hrn [ rec_tuple ] = record - + def import_sites_and_nodes(self, slabdriver): - """ - + """ + Gets all the sites and nodes from OAR, process the information, creates hrns and RegAuthority for sites, and feed them to the database. - For each site, import the site's nodes to the DB by calling + For each site, import the site's nodes to the DB by calling import_nodes. - + :param slabdriver: SlabDriver object, used to have access to slabdriver methods and fetching info on sites and nodes. - :type slabdriver: SlabDriver + :type slabdriver: SlabDriver """ - - sites_listdict = slabdriver.slab_api.GetSites() + + sites_listdict = slabdriver.slab_api.GetSites() nodes_listdict = slabdriver.slab_api.GetNodes() nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) for site in sites_listdict: @@ -165,46 +165,46 @@ class SlabImporter: site_record = self.find_record_by_type_hrn ('authority', site_hrn) if not site_record: try: - urn = hrn_to_urn(site_hrn, 'authority') + urn = hrn_to_urn(site_hrn, 'authority') if not self.auth_hierarchy.auth_exists(urn): self.auth_hierarchy.create_auth(urn) - + auth_info = self.auth_hierarchy.get_auth_info(urn) site_record = RegAuthority(hrn=site_hrn, \ - gid=auth_info.get_gid_object(), - pointer='-1', - authority=get_authority(site_hrn)) + gid=auth_info.get_gid_object(), + pointer='-1', + authority=get_authority(site_hrn)) site_record.just_created() dbsession.add(site_record) dbsession.commit() self.logger.info("SlabImporter: imported authority (site) \ - %s" % site_record) + %s" % site_record) self.update_just_added_records_dict(site_record) except SQLAlchemyError: - # if the site import fails then there is no point in + # if the site import fails then there is no point in # trying to import the # site's child records(node, slices, persons), so skip them. self.logger.log_exc("SlabImporter: failed to import site. \ - Skipping child records") + Skipping child records") continue else: # xxx update the record ... pass - - + + site_record.stale = False self.import_nodes(site['node_ids'], nodes_by_id, slabdriver) - - return - - def import_nodes(self, site_node_ids, nodes_by_id, slabdriver) : - """ - + + return + + def import_nodes(self, site_node_ids, nodes_by_id, slabdriver): + """ + Creates appropriate hostnames and RegNode records for each node in site_node_ids, based on the information given by the - dict nodes_by_id that was made from data from OAR. + dict nodes_by_id that was made from data from OAR. Saves the records to the DB. - + :param site_node_ids: site's node ids :type site_node_ids: list of integers :param nodes_by_id: dictionary , key is the node id, value is the a dict @@ -213,65 +213,65 @@ class SlabImporter: :param slabdriver:SlabDriver object, used to have access to slabdriver attributes. :type slabdriver:SlabDriver - + """ - + for node_id in site_node_ids: try: node = nodes_by_id[node_id] except KeyError: self.logger.warning ("SlabImporter: cannot find node_id %s \ - ignored" %(node_id)) - continue + continue escaped_hrn = \ self.hostname_to_hrn_escaped(slabdriver.slab_api.root_auth, \ node['hostname']) - self.logger.info("SLABIMPORTER node %s " %(node)) + self.logger.info("SLABIMPORTER node %s " %(node)) hrn = node['hrn'] # xxx this sounds suspicious - if len(hrn) > 64: + if len(hrn) > 64: hrn = hrn[:64] node_record = self.find_record_by_type_hrn( 'node', hrn ) if not node_record: pkey = Keypair(create=True) - urn = hrn_to_urn(escaped_hrn, 'node') + urn = hrn_to_urn(escaped_hrn, 'node') node_gid = \ self.auth_hierarchy.create_gid(urn, \ create_uuid(), pkey) - + def slab_get_authority(hrn): return hrn.split(".")[0] - - node_record = RegNode(hrn=hrn, gid=node_gid, + + node_record = RegNode(hrn=hrn, gid=node_gid, pointer = '-1', authority=slab_get_authority(hrn)) try: - + node_record.just_created() dbsession.add(node_record) dbsession.commit() self.logger.info("SlabImporter: imported node: %s" \ - % node_record) + % node_record) self.update_just_added_records_dict(node_record) except SQLAlchemyError: self.logger.log_exc("SlabImporter: \ - failed to import node") + failed to import node") else: #TODO: xxx update the record ... pass node_record.stale = False - + def init_person_key (self, person, slab_key): - """ - + """ + Returns a tuple pubkey and pkey. - + :param person Person's data. :type person: dict - :param slab_key: SSH public key, from LDAP user's data. + :param slab_key: SSH public key, from LDAP user's data. RSA type supported. :type slab_key: string :rtype (string, Keypair) @@ -280,7 +280,7 @@ class SlabImporter: if person['pkey']: # randomly pick first key in set pubkey = slab_key - + try: pkey = convert_public_key(pubkey) except TypeError: @@ -289,19 +289,19 @@ class SlabImporter: unable to convert public \ key for %s' %person['hrn']) pkey = Keypair(create=True) - + else: - # the user has no keys. + # the user has no keys. #Creating a random keypair for the user's gid self.logger.warn("SlabImporter: person %s does not have a \ public key" %(person['hrn'])) - pkey = Keypair(create=True) + pkey = Keypair(create=True) return (pubkey, pkey) - - + + def import_persons_and_slices(self, slabdriver): """ - + Gets user data from LDAP, process the information. Creates hrn for the user's slice, the user's gid, creates the RegUser record associated with user. Creates the RegKey record @@ -309,18 +309,18 @@ class SlabImporter: Saves those records into the SFA DB. import the user's slice onto the database as well by calling import_slice. - + :param slabdriver:SlabDriver object, used to have access to slabdriver attributes. :type slabdriver:SlabDriver """ ldap_person_listdict = slabdriver.slab_api.GetPersons() self.logger.info("SLABIMPORT \t ldap_person_listdict %s \r\n" \ - %(ldap_person_listdict)) - + %(ldap_person_listdict)) + # import persons - for person in ldap_person_listdict : - + for person in ldap_person_listdict : + self.logger.info("SlabImporter: person :" %(person)) if 'ssh-rsa' not in person['pkey']: #people with invalid ssh key (ssh-dss, empty, bullshit keys...) @@ -328,31 +328,31 @@ class SlabImporter: continue person_hrn = person['hrn'] slice_hrn = self.slicename_to_hrn(person['hrn']) - + # xxx suspicious again - if len(person_hrn) > 64: + if len(person_hrn) > 64: person_hrn = person_hrn[:64] person_urn = hrn_to_urn(person_hrn, 'user') - - + + self.logger.info("SlabImporter: users_rec_by_email %s " \ %(self.users_rec_by_email)) - + #Check if user using person['email'] from LDAP is already registered #in SFA. One email = one person. In this case, do not create another #record for this person - #person_hrn returned by GetPerson based on senslab root auth + + #person_hrn returned by GetPerson based on senslab root auth + #uid ldap user_record = self.find_record_by_type_hrn('user', person_hrn) - + if not user_record and person['email'] in self.users_rec_by_email: user_record = self.users_rec_by_email[person['email']] person_hrn = user_record.hrn person_urn = hrn_to_urn(person_hrn, 'user') - - + + slice_record = self.find_record_by_type_hrn ('slice', slice_hrn) - + slab_key = person['pkey'] # new person if not user_record: @@ -366,41 +366,41 @@ class SlabImporter: PERSON EMAIL OK email %s " %(person['email'])) person_gid.set_email(person['email']) user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', + gid=person_gid, + pointer='-1', authority=get_authority(person_hrn), email=person['email']) else: user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', + gid=person_gid, + pointer='-1', authority=get_authority(person_hrn)) - - if pubkey: + + if pubkey: user_record.reg_keys = [RegKey(pubkey)] else: self.logger.warning("No key found for user %s" \ %(user_record)) - - try: + + try: user_record.just_created() dbsession.add (user_record) dbsession.commit() self.logger.info("SlabImporter: imported person %s"\ %(user_record)) self.update_just_added_records_dict( user_record ) - + except SQLAlchemyError: self.logger.log_exc("SlabImporter: \ - failed to import person %s"%(person)) + failed to import person %s"%(person)) else: # update the record ? - # if user's primary key has changed then we need to update + # if user's primary key has changed then we need to update # the users gid by forcing an update here sfa_keys = user_record.reg_keys - + new_key = False - if slab_key is not sfa_keys : + if slab_key is not sfa_keys : new_key = True if new_key: self.logger.info("SlabImporter: \t \t USER UPDATE \ @@ -415,59 +415,59 @@ class SlabImporter: user_record.reg_keys = [RegKey(pubkey)] self.logger.info("SlabImporter: updated person: %s" \ % (user_record)) - + if person['email']: user_record.email = person['email'] - - try: + + try: dbsession.commit() user_record.stale = False except SQLAlchemyError: self.logger.log_exc("SlabImporter: \ - failed to update person %s"%(person)) - + failed to update person %s"%(person)) + self.import_slice(slice_hrn, slice_record, user_record) - - + + def import_slice(self, slice_hrn, slice_record, user_record): """ - + Create RegSlice record according to the slice hrn if the slice - does not exist yet.Creates a relationship with the user record + does not exist yet.Creates a relationship with the user record associated with the slice. Commit the record to the database. - - + + :param slice_hrn: Human readable name of the slice. :type slice_hrn: string :param slice_record: record of the slice found in the DB, if any. :type slice_record: RegSlice or None :param user_record: user record found in the DB if any. :type user_record: RegUser - + .. todo::Update the record if a slice record already exists. """ - if not slice_record : + if not slice_record : pkey = Keypair(create=True) urn = hrn_to_urn(slice_hrn, 'slice') slice_gid = \ self.auth_hierarchy.create_gid(urn, \ create_uuid(), pkey) - slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, + slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, pointer='-1', authority=get_authority(slice_hrn)) try: slice_record.just_created() dbsession.add(slice_record) dbsession.commit() - - + + self.update_just_added_records_dict ( slice_record ) except SQLAlchemyError: self.logger.log_exc("SlabImporter: failed to import slice") - - #No slice update upon import in senslab + + #No slice update upon import in senslab else: # xxx update the record ... self.logger.warning ("Slice update not yet implemented") @@ -478,11 +478,11 @@ class SlabImporter: slice_record.reg_researchers = [user_record] try: dbsession.commit() - slice_record.stale = False + slice_record.stale = False except SQLAlchemyError: self.logger.log_exc("SlabImporter: failed to update slice") - - + + def run (self, options): """ Create the special senslab table, slab_xp, in the senslab database. @@ -490,53 +490,53 @@ class SlabImporter: and LDAP) into the SFA database. Delete stale records that are no longer in OAR or LDAP. :param options: - :type options: + :type options: """ config = Config() - + slabdriver = SlabDriver(config) - - #Create special slice table for senslab - + + #Create special slice table for senslab + if not slabdriver.db.exists('slab_xp'): slabdriver.db.createtable() self.logger.info ("SlabImporter.run: slab_xp table created ") - + # import site and node records in site into the SFA db. self.import_sites_and_nodes(slabdriver) - #import users and slice into the SFA DB. + #import users and slice into the SFA DB. self.import_persons_and_slices(slabdriver) - + ### remove stale records # special records must be preserved system_hrns = [slabdriver.hrn, slabdriver.slab_api.root_auth, \ slabdriver.hrn+ '.slicemanager'] - for record in self.all_records: - if record.hrn in system_hrns: + for record in self.all_records: + if record.hrn in system_hrns: record.stale = False if record.peer_authority: record.stale = False - - for record in self.all_records: + + for record in self.all_records: if record.type == 'user': self.logger.info("SlabImporter: stale records: hrn %s %s" \ %(record.hrn,record.stale) ) - try: + try: stale = record.stale - except : + except : stale = True self.logger.warning("stale not found with %s"%record) if stale: self.logger.info("SlabImporter: deleting stale record: %s" \ %(record)) - + try: dbsession.delete(record) - dbsession.commit() + dbsession.commit() except SQLAlchemyError: self.logger.log_exc("SlabImporter: failed to delete stale \ record %s" %(record) ) - + diff --git a/sfa/senslab/LDAPapi.py b/sfa/senslab/LDAPapi.py index 6d34298c..2123e07d 100644 --- a/sfa/senslab/LDAPapi.py +++ b/sfa/senslab/LDAPapi.py @@ -1,6 +1,6 @@ import random from passlib.hash import ldap_salted_sha1 as lssha -from sfa.util.xrn import get_authority +from sfa.util.xrn import get_authority import ldap from sfa.util.config import Config @@ -14,27 +14,27 @@ import os.path class LdapConfig(): def __init__(self, config_file = '/etc/sfa/ldap_config.py'): - + try: execfile(config_file, self.__dict__) - + self.config_file = config_file # path to configuration data self.config_path = os.path.dirname(config_file) except IOError: raise IOError, "Could not find or load the configuration file: %s" \ % config_file - - + + class ldap_co: """ Set admin login and server configuration variables.""" - + def __init__(self): #Senslab PROD LDAP parameters self.ldapserv = None ldap_config = LdapConfig() self.config = ldap_config - self.ldapHost = ldap_config.LDAP_IP_ADDRESS + self.ldapHost = ldap_config.LDAP_IP_ADDRESS self.ldapPeopleDN = ldap_config.LDAP_PEOPLE_DN self.ldapGroupDN = ldap_config.LDAP_GROUP_DN self.ldapAdminDN = ldap_config.LDAP_WEB_DN @@ -53,7 +53,7 @@ class ldap_co: (for add/modify/delete operations). Set to False otherwise. :type bind : boolean - :return: dictionary with status of the connection. True if Successful, + :return: dictionary with status of the connection. True if Successful, False if not and in this case the error message( {'bool', 'message'} ) :rtype:dict """ @@ -61,26 +61,26 @@ class ldap_co: self.ldapserv = ldap.open(self.ldapHost) except ldap.LDAPError, error: return {'bool' : False, 'message' : error } - + # Bind with authentification - if(bind): + if(bind): return self.bind() - - else: - return {'bool': True} - + + else: + return {'bool': True} + def bind(self): - """ Binding method. - :return: dictionary with the bind status. True if Successful, + """ Binding method. + :return: dictionary with the bind status. True if Successful, False if not and in this case the error message( {'bool', 'message'} ) :rtype: dict - + """ try: # Opens a connection after a call to ldap.open in connect: self.ldapserv = ldap.initialize("ldap://" + self.ldapHost) - - # Bind/authenticate with a user with apropriate + + # Bind/authenticate with a user with apropriate #rights to add objects self.ldapserv.simple_bind_s(self.ldapAdminDN, \ self.ldapAdminPassword) @@ -89,7 +89,7 @@ class ldap_co: return {'bool' : False, 'message' : error } return {'bool': True} - + def close(self): """ Close the LDAP connection. Can throw an exception if the unbinding fails. @@ -98,7 +98,7 @@ class ldap_co: self.ldapserv.unbind_s() except ldap.LDAPError, error: return {'bool' : False, 'message' : error } - + class LoginPassword(): """ Class to handle login and password generation, using custom login generation @@ -107,7 +107,7 @@ class LoginPassword(): def __init__(self): """ Sets password and login maximum length, and defines the characters - that can be found in a random generated password. + that can be found in a random generated password. """ self.login_max_length = 8 self.length_password = 8 @@ -120,10 +120,10 @@ class LoginPassword(): 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p' ,'q', \ 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', \ '\''] - - - - + + + + @staticmethod def clean_user_names(record): """ @@ -144,15 +144,15 @@ class LoginPassword(): lower_last_name = record['last_name'].replace('-','')\ .replace('_','').replace('[','')\ .replace(']','').replace(' ','')\ - .lower() + .lower() return lower_first_name, lower_last_name else: return None, None - + @staticmethod def extract_name_from_email(record): """ - When there is no valid first name and last name in the record, + When there is no valid first name and last name in the record, the email is used to generate the login. Here, we assume the email is firstname.lastname@something.smthg. The first name and last names are extracted from the email, special charcaters are removed and @@ -177,13 +177,13 @@ class LoginPassword(): lower_first_name = mail[0] lower_last_name = mail[1] break - - #Otherwise just take the part before the @ as the + + #Otherwise just take the part before the @ as the #lower_first_name and lower_last_name if lower_first_name is None: lower_first_name = email - lower_last_name = email - + lower_last_name = email + return lower_first_name, lower_last_name def get_user_firstname_lastname(self, record): @@ -191,24 +191,24 @@ class LoginPassword(): we have in the record. :param record: user's information :type record: dict - :return: the user's first name and last name. + :return: the user's first name and last name. ..seealso: clean_user_names ..seealso: extract_name_from_email """ lower_first_name, lower_last_name = self.clean_user_names(record) - - #No first name and last name check email + + #No first name and last name check email if lower_first_name is None and lower_last_name is None: lower_first_name, lower_last_name = \ self.extract_name_from_email(record) - + return lower_first_name, lower_last_name - - + + def choose_sets_chars_for_login(self, lower_first_name, lower_last_name): """ - Algorithm to select sets of characters from the first name and + Algorithm to select sets of characters from the first name and last name, depending on the lenght of the last name and the maximum login length which in our case is set to 8 charachetrs. :param lower_first_name: user's first name in lower case. @@ -216,11 +216,11 @@ class LoginPassword(): :return: user's login :rtype:string """ - length_last_name = len(lower_last_name) + length_last_name = len(lower_last_name) self.login_max_length = 8 - + #Try generating a unique login based on first name and last name - + if length_last_name >= self.login_max_length : login = lower_last_name[0:self.login_max_length] index = 0 @@ -248,95 +248,95 @@ class LoginPassword(): impossible to generate unique login for %s %s" \ %(lower_first_name,lower_last_name)) return index, login - - + + def generate_password(self): - - """Generate a password upon adding a new user in LDAP Directory + + """Generate a password upon adding a new user in LDAP Directory (8 characters length). The generated password is composed of characters from the charsPassword list :return: the randomly generated password :rtype: string - + """ password = str() - + length = len(self.chars_password) for index in range(self.length_password): char_index = random.randint(0, length-1) password += self.chars_password[char_index] return password - + @staticmethod def encrypt_password(password): """ Use passlib library to make a RFC2307 LDAP encrypted password - salt size = 8, use sha-1 algorithm. + salt size = 8, use sha-1 algorithm. :param password: password not encrypted. :type password: string :return: Returns encrypted password. :rtype:string """ - #Keep consistency with Java Senslab's LDAP API + #Keep consistency with Java Senslab's LDAP API #RFC2307SSHAPasswordEncryptor so set the salt size to 8 bytes - return lssha.encrypt(password, salt_size = 8) - - - + return lssha.encrypt(password, salt_size = 8) + + + class LDAPapi : def __init__(self): - logger.setLevelDebug() - + logger.setLevelDebug() + #SFA related config config = Config() - self.login_pwd = LoginPassword() + self.login_pwd = LoginPassword() self.authname = config.SFA_REGISTRY_ROOT_AUTH - self.conn = ldap_co() - self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS - self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN - self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER - self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH - + self.conn = ldap_co() + self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS + self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN + self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER + self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH + self.baseDN = self.conn.ldapPeopleDN - + self.ldapShell = '/bin/bash' - - - + + + def LdapGenerateUniqueLogin(self, record): """ - Generate login for adding a new user in LDAP Directory + Generate login for adding a new user in LDAP Directory (four characters minimum length). Get proper last name and - first name so that the user's login can be generated. + first name so that the user's login can be generated. :param record: Record must contain first_name and last_name. :param record: dict - :return: the generated login for the user described with record if the + :return: the generated login for the user described with record if the login generation is successful, None if it fails. :rtype: string or None - """ + """ #For compatibility with other ldap func if 'mail' in record and 'email' not in record: record['email'] = record['mail'] - + lower_first_name, lower_last_name = \ - self.login_pwd.get_user_firstname_lastname(record) - - + self.login_pwd.get_user_firstname_lastname(record) + + index, login = self.login_pwd.choose_sets_chars_for_login( \ lower_first_name, \ - lower_last_name) + lower_last_name) login_filter = '(uid=' + login + ')' get_attrs = ['uid'] try : #Check if login already in use - + while (len(self.LdapSearch(login_filter, get_attrs)) is not 0 ): - + index += 1 if index >= 9: logger.error("LoginException : Generation login error \ @@ -349,19 +349,19 @@ class LDAPapi : login_filter = '(uid='+ login+ ')' except KeyError: print "lower_first_name - lower_last_name too short" - + logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s"%(login)) return login - + except ldap.LDAPError, error : logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" %error) return None def find_max_uidNumber(self): - + """Find the LDAP max uidNumber (POSIX uid attribute) . - Used when adding a new user in LDAP Directory + Used when adding a new user in LDAP Directory :return: max uidNumber + 1 :rtype:string """ @@ -369,37 +369,37 @@ class LDAPapi : get_attrs = "(uidNumber=*)" login_filter = ['uidNumber'] - result_data = self.LdapSearch(get_attrs, login_filter) + result_data = self.LdapSearch(get_attrs, login_filter) #It there is no user in LDAP yet, First LDAP user if result_data == []: max_uidnumber = self.ldapUserUidNumberMin #Otherwise, get the highest uidNumber else: - + uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data ] logger.debug("LDAPapi.py \tfind_max_uidNumber \ uidNumberList %s " %(uidNumberList)) max_uidnumber = max(uidNumberList) + 1 - + return str(max_uidnumber) - - + + def get_ssh_pkey(self, record): - """TODO ; Get ssh public key from sfa record + """TODO ; Get ssh public key from sfa record To be filled by N. Turro ? or using GID pl way? - + """ return 'A REMPLIR ' - + @staticmethod - #TODO Handle OR filtering in the ldap query when + #TODO Handle OR filtering in the ldap query when #dealing with a list of records instead of doing a for loop in GetPersons def make_ldap_filters_from_record( record=None): - """ + """ Helper function to make LDAP filter requests out of SFA records. :param record: user's sfa record. Should contain first_name,last_name, email or mail, and if the record is enabled or not. If the dict - record does not have all of these, must at least contain the user's + record does not have all of these, must at least contain the user's email. :type record: dict :return: LDAP request @@ -420,19 +420,19 @@ class LDAPapi : req_ldapdict['shadowExpire'] = '-1' else: req_ldapdict['shadowExpire'] = '0' - - #Hrn should not be part of the filter because the hrn - #presented by a certificate of a SFA user not imported in - #Senslab does not include the senslab login in it + + #Hrn should not be part of the filter because the hrn + #presented by a certificate of a SFA user not imported in + #Senslab does not include the senslab login in it #Plus, the SFA user may already have an account with senslab #using another login. - - + + logger.debug("\r\n \t LDAP.PY make_ldap_filters_from_record \ record %s req_ldapdict %s" \ %(record, req_ldapdict)) - + for k in req_ldapdict: req_ldap += '('+ str(k)+ '=' + str(req_ldapdict[k]) + ')' if len(req_ldapdict.keys()) >1 : @@ -441,23 +441,23 @@ class LDAPapi : req_ldap = req_ldap[:(size-1)] +')'+ req_ldap[(size-1):] else: req_ldap = "(cn=*)" - + return req_ldap - + def make_ldap_attributes_from_record(self, record): - """When adding a new user to Senslab's LDAP, creates an attributes + """When adding a new user to Senslab's LDAP, creates an attributes dictionnary from the SFA record understandable by LDAP. - Generates the user's LDAP login. + Generates the user's LDAP login. User is automatically validated (account enabled) and described - as a SFA USER FROM OUTSIDE SENSLAB'. + as a SFA USER FROM OUTSIDE SENSLAB'. :param record: must contain the following keys and values: first_name, last_name, mail, pkey (ssh key). :type record: dict - + :return: dictionary of attributes using LDAP data structure model. :rtype: dict - + """ attrs = {} @@ -465,23 +465,23 @@ class LDAPapi : "organizationalPerson", "posixAccount", \ "shadowAccount", "systemQuotas", \ "ldapPublicKey"] - - - attrs['uid'] = self.LdapGenerateUniqueLogin(record) + + + attrs['uid'] = self.LdapGenerateUniqueLogin(record) try: attrs['givenName'] = str(record['first_name']).lower().capitalize() attrs['sn'] = str(record['last_name']).lower().capitalize() attrs['cn'] = attrs['givenName'] + ' ' + attrs['sn'] attrs['gecos'] = attrs['givenName'] + ' ' + attrs['sn'] - - except KeyError: + + except KeyError: attrs['givenName'] = attrs['uid'] attrs['sn'] = attrs['uid'] attrs['cn'] = attrs['uid'] attrs['gecos'] = attrs['uid'] - - - attrs['quota'] = self.ldapUserQuotaNFS + + + attrs['quota'] = self.ldapUserQuotaNFS attrs['homeDirectory'] = self.ldapUserHomePath + attrs['uid'] attrs['loginShell'] = self.ldapShell attrs['gidNumber'] = self.ldapUserGidNumber @@ -490,18 +490,18 @@ class LDAPapi : try: attrs['sshPublicKey'] = record['pkey'] except KeyError: - attrs['sshPublicKey'] = self.get_ssh_pkey(record) - + attrs['sshPublicKey'] = self.get_ssh_pkey(record) + - #Password is automatically generated because SFA user don't go - #through the Senslab website used to register new users, + #Password is automatically generated because SFA user don't go + #through the Senslab website used to register new users, #There is no place in SFA where users can enter such information #yet. - #If the user wants to set his own password , he must go to the Senslab + #If the user wants to set his own password , he must go to the Senslab #website. password = self.login_pwd.generate_password() attrs['userPassword'] = self.login_pwd.encrypt_password(password) - + #Account automatically validated (no mail request to admins) #Set to 0 to disable the account, -1 to enable it, attrs['shadowExpire'] = '-1' @@ -511,7 +511,7 @@ class LDAPapi : attrs['ou'] = 'SFA' #Optional: organizational unit #No info about those here: - attrs['l'] = 'To be defined'#Optional: Locality. + attrs['l'] = 'To be defined'#Optional: Locality. attrs['st'] = 'To be defined' #Optional: state or province (country). return attrs @@ -519,63 +519,63 @@ class LDAPapi : def LdapAddUser(self, record) : - """Add SFA user to LDAP if it is not in LDAP yet. - :param record: dictionnary with the user's data. - + """Add SFA user to LDAP if it is not in LDAP yet. + :param record: dictionnary with the user's data. + :return: a dictionary with the status (Fail= False, Success= True) and the uid of the newly added user if successful, or the error meassage it is not. Dict has keys bool and message in case of failure, and bool uid in case of success. :rtype: dict - + ..seealso: make_ldap_filters_from_record - + """ logger.debug(" \r\n \t LDAP LdapAddUser \r\n\r\n ================\r\n ") user_ldap_attrs = self.make_ldap_attributes_from_record(record) - + #Check if user already in LDAP wih email, first name and last name filter_by = self.make_ldap_filters_from_record(user_ldap_attrs) user_exist = self.LdapSearch(filter_by) if user_exist: logger.warning(" \r\n \t LDAP LdapAddUser user %s %s \ already exists" %(user_ldap_attrs['sn'], \ - user_ldap_attrs['mail'])) + user_ldap_attrs['mail'])) return {'bool': False} - + #Bind to the server result = self.conn.connect() - + if(result['bool']): - + # A dict to help build the "body" of the object - + logger.debug(" \r\n \t LDAP LdapAddUser attrs %s " %user_ldap_attrs) # The dn of our new entry/object - dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN + dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN try: ldif = modlist.addModlist(user_ldap_attrs) logger.debug("LDAPapi.py add attrs %s \r\n ldif %s"\ %(user_ldap_attrs, ldif) ) self.conn.ldapserv.add_s(dn, ldif) - + logger.info("Adding user %s login %s in LDAP" \ %(user_ldap_attrs['cn'] , user_ldap_attrs['uid'])) - - + + except ldap.LDAPError, error: logger.log_exc("LDAP Add Error %s" %error) return {'bool' : False, 'message' : error } - + self.conn.close() - return {'bool': True, 'uid':user_ldap_attrs['uid']} - else: + return {'bool': True, 'uid':user_ldap_attrs['uid']} + else: return result - + def LdapDelete(self, person_dn): """ Deletes a person in LDAP. Uses the dn of the user. @@ -585,23 +585,23 @@ class LDAPapi : and the error if not. :rtype:dict """ - #Connect and bind + #Connect and bind result = self.conn.connect() if(result['bool']): try: self.conn.ldapserv.delete_s(person_dn) self.conn.close() return {'bool': True} - + except ldap.LDAPError, error: logger.log_exc("LDAP Delete Error %s" %error) return {'bool': False, 'message': error} - - - def LdapDeleteUser(self, record_filter): + + + def LdapDeleteUser(self, record_filter): """ Deletes a SFA person in LDAP, based on the user's hrn. - :param record_filter: Filter to find the user to be deleted. Must + :param record_filter: Filter to find the user to be deleted. Must contain at least the user's email. :type record_filter: dict :return: dict with bool True if successful, bool False and error message @@ -610,24 +610,24 @@ class LDAPapi : ..seealso: LdapFindUser docstring for more info on record filter. ..seealso: LdapDelete for user deletion """ - #Find uid of the person + #Find uid of the person person = self.LdapFindUser(record_filter, []) logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s" \ %(record_filter, person)) if person: - dn = 'uid=' + person['uid'] + "," + self.baseDN + dn = 'uid=' + person['uid'] + "," + self.baseDN else: return {'bool': False} - + result = self.LdapDelete(dn) return result - - def LdapModify(self, dn, old_attributes_dict, new_attributes_dict): + + def LdapModify(self, dn, old_attributes_dict, new_attributes_dict): """ Modifies a LDAP entry, replaces user's old attributes with the new ones given. - :param dn: user's absolute name in the LDAP hierarchy. + :param dn: user's absolute name in the LDAP hierarchy. :param old_attributes_dict: old user's attributes. Keys must match the ones used in the LDAP model. :param new_attributes_dict: new user's attributes. Keys must match @@ -636,13 +636,13 @@ class LDAPapi : :type old_attributes_dict: dict :type new_attributes_dict: dict :return: dict bool True if Successful, bool False if not. - :rtype:dict + :rtype:dict """ - + ldif = modlist.modifyModlist(old_attributes_dict, new_attributes_dict) - # Connect and bind/authenticate - result = self.conn.connect() - if (result['bool']): + # Connect and bind/authenticate + result = self.conn.connect() + if (result['bool']): try: self.conn.ldapserv.modify_s(dn, ldif) self.conn.close() @@ -650,17 +650,17 @@ class LDAPapi : except ldap.LDAPError, error: logger.log_exc("LDAP LdapModify Error %s" %error) return {'bool' : False } - - + + def LdapModifyUser(self, user_record, new_attributes_dict): """ - Gets the record from one user based on the user sfa record + Gets the record from one user based on the user sfa record and changes the attributes according to the specified new_attributes. - Do not use this if we need to modify the uid. Use a ModRDN + Do not use this if we need to modify the uid. Use a ModRDN #operation instead ( modify relative DN ) :param user_record: sfa user record. :param new_attributes_dict: new user attributes, keys must be the - same as the LDAP model. + same as the LDAP model. :type user_record: dict :type new_attributes_dict: dict :return: bool True if successful, bool False if not. @@ -671,9 +671,9 @@ class LDAPapi : """ if user_record is None: logger.error("LDAP \t LdapModifyUser Need user record ") - return {'bool': False} - - #Get all the attributes of the user_uid_login + return {'bool': False} + + #Get all the attributes of the user_uid_login #person = self.LdapFindUser(record_filter,[]) req_ldap = self.make_ldap_filters_from_record(user_record) person_list = self.LdapSearch(req_ldap, []) @@ -685,13 +685,13 @@ class LDAPapi : if person_list is None : logger.error("LDAP \t LdapModifyUser User %s doesn't exist "\ %(user_record)) - return {'bool': False} - + return {'bool': False} + # The dn of our existing entry/object #One result only from ldapSearch person = person_list[0][1] - dn = 'uid=' + person['uid'][0] + "," + self.baseDN - + dn = 'uid=' + person['uid'][0] + "," + self.baseDN + if new_attributes_dict: old = {} for k in new_attributes_dict: @@ -700,17 +700,17 @@ class LDAPapi : else : old[k] = person[k] logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s"\ - %( new_attributes_dict)) + %( new_attributes_dict)) result = self.LdapModify(dn, old, new_attributes_dict) return result else: logger.error("LDAP \t LdapModifyUser No new attributes given. ") - return {'bool': False} - - - - - def LdapMarkUserAsDeleted(self, record): + return {'bool': False} + + + + + def LdapMarkUserAsDeleted(self, record): """ Sets shadowExpire to 0, disabling the user in LDAP. Calls LdapModifyUser to change the shadowExpire of the user. @@ -720,7 +720,7 @@ class LDAPapi : :rtype:dict ..seealso: LdapModifyUser """ - + new_attrs = {} #Disable account new_attrs['shadowExpire'] = '0' @@ -728,12 +728,12 @@ class LDAPapi : ret = self.LdapModifyUser(record, new_attrs) return ret - + def LdapResetPassword(self, record): """ Resets password for the user whose record is the parameter and changes the corresponding entry in the LDAP. - + """ password = self.login_pwd.generate_password() attrs = {} @@ -742,28 +742,28 @@ class LDAPapi : %(attrs['userPassword'])) result = self.LdapModifyUser(record, attrs) return result - - + + def LdapSearch (self, req_ldap = None, expected_fields = None ): """ Used to search directly in LDAP, by using ldap filters and - return fields. + return fields. When req_ldap is None, returns all the entries in the LDAP. - + """ result = self.conn.connect(bind = False) if (result['bool']) : - + return_fields_list = [] - if expected_fields == None : + if expected_fields == None : return_fields_list = ['mail', 'givenName', 'sn', 'uid', \ 'sshPublicKey', 'shadowExpire'] - else : + else : return_fields_list = expected_fields - #No specifc request specified, get the whole LDAP + #No specifc request specified, get the whole LDAP if req_ldap == None: req_ldap = '(cn=*)' - + logger.debug("LDAP.PY \t LdapSearch req_ldap %s \ return_fields_list %s" \ %(req_ldap, return_fields_list)) @@ -771,8 +771,8 @@ class LDAPapi : try: msg_id = self.conn.ldapserv.search( self.baseDN,ldap.SCOPE_SUBTREE,\ - req_ldap, return_fields_list) - #Get all the results matching the search from ldap in one + req_ldap, return_fields_list) + #Get all the results matching the search from ldap in one #shot (1 value) result_type, result_data = \ self.conn.ldapserv.result(msg_id, 1) @@ -783,38 +783,40 @@ class LDAPapi : %(result_data)) return result_data - + except ldap.LDAPError, error : logger.log_exc("LDAP LdapSearch Error %s" %error) return [] - + else: logger.error("LDAP.PY \t Connection Failed" ) - return - - + return + + def _process_ldap_info_for_all_users(self, result_data): """ Process the data of all enabled users in LDAP. :param result_data: Contains information of all enabled users in LDAP and is coming from LdapSearch. :param result_data: list - ..seealso: LdapSearch + ..seealso: LdapSearch """ results = [] + logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s " \ + %(result_data)) for ldapentry in result_data: - logger.debug(" LDAP.py LdapFindUser ldapentry name : %s " \ + logger.debug(" LDAP.py _process_ldap_info_for_all_users ldapentry name : %s " \ %(ldapentry[1]['uid'][0])) tmpname = ldapentry[1]['uid'][0] hrn = self.authname + "." + tmpname - + tmpemail = ldapentry[1]['mail'][0] if ldapentry[1]['mail'][0] == "unknown": tmpemail = None try: - results.append( { + results.append( { 'type': 'user', 'pkey': ldapentry[1]['sshPublicKey'][0], #'uid': ldapentry[1]['uid'][0], @@ -829,26 +831,26 @@ class LDAPapi : 'peer_authority': '', 'pointer' : -1, 'hrn': hrn, - } ) + } ) except KeyError, error: logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s" \ %(error)) return - + return results - + def _process_ldap_info_for_one_user(self, record, result_data): - """ + """ Put the user's ldap data into shape. Only deals with one user - record and one user data from ldap. - :param record: user record + record and one user data from ldap. + :param record: user record :param result_data: Raw ldap data coming from LdapSearch :return: user's data dict with 'type','pkey','uid', 'email', - 'first_name' 'last_name''serial''authority''peer_authority' + 'first_name' 'last_name''serial''authority''peer_authority' 'pointer''hrn' :type record: dict :type result_data: list - :rtype :dict + :rtype :dict """ #One entry only in the ldap data because we used a filter #to find one user only @@ -859,26 +861,26 @@ class LDAPapi : tmpemail = ldapentry['mail'][0] if ldapentry['mail'][0] == "unknown": tmpemail = None - + parent_hrn = None - peer_authority = None + peer_authority = None if 'hrn' in record: hrn = record['hrn'] parent_hrn = get_authority(hrn) if parent_hrn != self.authname: peer_authority = parent_hrn #In case the user was not imported from Senslab LDAP - #but from another federated site, has an account in + #but from another federated site, has an account in #senslab but currently using his hrn from federated site - #then the login is different from the one found in its hrn + #then the login is different from the one found in its hrn if tmpname != hrn.split('.')[1]: hrn = None else: hrn = None - - - - results = { + + + + results = { 'type': 'user', 'pkey': ldapentry['sshPublicKey'], #'uid': ldapentry[1]['uid'][0], @@ -895,50 +897,50 @@ class LDAPapi : 'hrn': hrn, } return results - - - def LdapFindUser(self, record = None, is_user_enabled=None, \ - expected_fields = None): + + + def LdapFindUser(self, record=None, is_user_enabled=None, + expected_fields=None): """ - Search a SFA user with a hrn. User should be already registered - in Senslab LDAP. + Search a SFA user with a hrn. User should be already registered + in Senslab LDAP. :param record: sfa user's record. Should contain first_name,last_name, email or mail. If no record is provided, returns all the users found in LDAP. :type record: dict :param is_user_enabled: is the user's senslab account already valid. - :type is_user_enabled: Boolean. + :type is_user_enabled: Boolean. :return: LDAP entries from ldap matching the filter provided. Returns a single entry if one filter has been given and a list of entries otherwise. :rtype: dict or list - """ + """ custom_record = {} - if is_user_enabled: + if is_user_enabled: custom_record['enabled'] = is_user_enabled - if record: + if record: custom_record.update(record) - req_ldap = self.make_ldap_filters_from_record(custom_record) + req_ldap = self.make_ldap_filters_from_record(custom_record) return_fields_list = [] - if expected_fields == None : + if expected_fields == None : return_fields_list = ['mail', 'givenName', 'sn', 'uid', \ 'sshPublicKey'] - else : + else : return_fields_list = expected_fields - + result_data = self.LdapSearch(req_ldap, return_fields_list ) logger.debug("LDAP.PY \t LdapFindUser result_data %s" %(result_data)) - + if len(result_data) is 0: return None #Asked for a specific user - if record : + if record is not None: results = self._process_ldap_info_for_one_user(record, result_data) - + else: #Asked for all users in ldap results = self._process_ldap_info_for_all_users(result_data) - return results - + return results + diff --git a/sfa/senslab/slabaggregate.py b/sfa/senslab/slabaggregate.py index 474092bc..208a565b 100644 --- a/sfa/senslab/slabaggregate.py +++ b/sfa/senslab/slabaggregate.py @@ -24,9 +24,9 @@ def slab_xrn_to_hostname(xrn): def slab_xrn_object(root_auth, hostname): """Attributes are urn and hrn. Get the hostname using slab_xrn_to_hostname on the urn. - + :return: the senslab node's xrn - :rtype: Xrn + :rtype: Xrn """ return Xrn('.'.join( [root_auth, Xrn.escape(hostname)]), type='node') @@ -38,11 +38,11 @@ class SlabAggregate: interfaces = {} links = {} node_tags = {} - + prepared = False user_options = {} - + def __init__(self, driver): self.driver = driver @@ -54,18 +54,18 @@ class SlabAggregate: dictionary, keyed on the node hostnames. Returns a dict of slivers based on the sliver's node_id. Called by get_rspec. - - + + :param slice_xrn: xrn of the slice :param login: user's login on senslab ldap - + :type slice_xrn: string :type login: string :reutnr : a list of slices dict and a dictionary of Sliver object :rtype: (list, dict) - - ..note: There is no slivers in senslab, only leases. - + + ..note: There is no slivers in senslab, only leases. + """ slivers = {} sfa_slice = None @@ -78,7 +78,7 @@ class SlabAggregate: slices = self.driver.slab_api.GetSlices(slice_filter= str(slice_name), \ slice_filter_type = 'slice_hrn', \ login=login) - + logger.debug("Slabaggregate api \tget_slice_and_slivers \ sfa_slice %s \r\n slices %s self.driver.hrn %s" \ %(sfa_slice, slices, self.driver.hrn)) @@ -90,44 +90,44 @@ class SlabAggregate: #and therefore, node allocated to this slice for sfa_slice in slices: try: - node_ids_list = sfa_slice['node_ids'] + node_ids_list = sfa_slice['node_ids'] except KeyError: logger.log_exc("SLABAGGREGATE \t \ - get_slice_and_slivers KeyError ") + get_slice_and_slivers No nodes in the slice - KeyError ") continue - + for node in node_ids_list: sliver_xrn = Xrn(slice_urn, type='sliver', id=node) sliver_xrn.set_authority(self.driver.hrn) - sliver = Sliver({'sliver_id':sliver_xrn.urn, + sliver = Sliver({'sliver_id':sliver_xrn.urn, 'name': sfa_slice['hrn'], - 'type': 'slab-node', + 'type': 'slab-node', 'tags': []}) - + slivers[node] = sliver - - + + #Add default sliver attribute : #connection information for senslab - if get_authority (sfa_slice['hrn']) == self.driver.slab_api.root_auth: + if get_authority (sfa_slice['hrn']) == self.driver.slab_api.root_auth: tmp = sfa_slice['hrn'].split('.') ldap_username = tmp[1].split('_')[0] ssh_access = None slivers['default_sliver'] = {'ssh': ssh_access , \ 'login': ldap_username} - + #TODO get_slice_and_slivers Find the login of the external user logger.debug("SLABAGGREGATE api get_slice_and_slivers slivers %s "\ %(slivers)) return (slices, slivers) - - + + def get_nodes(self, slices=None, slivers=[], options=None): # NT: the semantic of this function is not clear to me : # if slice is not defined, then all the nodes should be returned - # if slice is defined, we should return only the nodes that + # if slice is defined, we should return only the nodes that # are part of this slice # but what is the role of the slivers parameter ? # So i assume that slice['node_ids'] will be the same as slivers for us @@ -136,43 +136,43 @@ class SlabAggregate: #if not slices or not slices['node_ids']: #return ([],[]) #tags_filter = {} - + # get the granularity in second for the reservation system grain = self.driver.slab_api.GetLeaseGranularity() - - + + nodes = self.driver.slab_api.GetNodes() - #geni_available = options.get('geni_available') + #geni_available = options.get('geni_available') #if geni_available: - #filter['boot_state'] = 'boot' - + #filter['boot_state'] = 'boot' + #filter.update({'peer_id': None}) #nodes = self.driver.slab_api.GetNodes(filter['hostname']) - + #site_ids = [] #interface_ids = [] #tag_ids = [] nodes_dict = {} - + #for node in nodes: - + #nodes_dict[node['node_id']] = node #logger.debug("SLABAGGREGATE api get_nodes nodes %s "\ #%(nodes )) # get sites - #sites_dict = self.get_sites({'site_id': site_ids}) + #sites_dict = self.get_sites({'site_id': site_ids}) # get interfaces - #interfaces = self.get_interfaces({'interface_id':interface_ids}) + #interfaces = self.get_interfaces({'interface_id':interface_ids}) # get tags #node_tags = self.get_node_tags(tags_filter) - + #if slices, this means we got to list all the nodes given to this slice - # Make a list of all the nodes in the slice before getting their + # Make a list of all the nodes in the slice before getting their #attributes rspec_nodes = [] slice_nodes_list = [] logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slices )) + %(slices )) if slices is not None: for one_slice in slices: try: @@ -181,14 +181,14 @@ class SlabAggregate: pass #for node in one_slice['node_ids']: #slice_nodes_list.append(node) - + reserved_nodes = self.driver.slab_api.GetNodesCurrentlyInUse() logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slice_nodes_list)) + %(slice_nodes_list)) for node in nodes: nodes_dict[node['node_id']] = node if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: - + rspec_node = SlabNode() # xxx how to retrieve site['login_base'] #site_id=node['site_id'] @@ -197,25 +197,25 @@ class SlabAggregate: rspec_node['mobile'] = node['mobile'] rspec_node['archi'] = node['archi'] rspec_node['radio'] = node['radio'] - + slab_xrn = slab_xrn_object(self.driver.slab_api.root_auth, \ node['hostname']) rspec_node['component_id'] = slab_xrn.urn - rspec_node['component_name'] = node['hostname'] + rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = \ hrn_to_urn(self.driver.slab_api.root_auth, \ 'authority+sa') - - # Senslab's nodes are federated : there is only one authority + + # Senslab's nodes are federated : there is only one authority # for all Senslab sites, registered in SFA. - # Removing the part including the site + # Removing the part including the site # in authority_id SA 27/07/12 - rspec_node['authority_id'] = rspec_node['component_manager_id'] - + rspec_node['authority_id'] = rspec_node['component_manager_id'] + # do not include boot state ( element) #in the manifest rspec - - + + rspec_node['boot_state'] = node['boot_state'] if node['hostname'] in reserved_nodes: rspec_node['boot_state'] = "Reserved" @@ -228,7 +228,7 @@ class SlabAggregate: node['site']}) rspec_node['location'] = location - + position = SlabPosition() for field in position : try: @@ -236,10 +236,10 @@ class SlabAggregate: except KeyError, error : logger.log_exc("SLABAGGREGATE\t get_nodes \ position %s "%(error)) - + rspec_node['position'] = position #rspec_node['interfaces'] = [] - + # Granularity granularity = Granularity({'grain': grain}) rspec_node['granularity'] = granularity @@ -250,7 +250,7 @@ class SlabAggregate: rspec_node['sliver_id'] = sliver['sliver_id'] rspec_node['client_id'] = node['hostname'] rspec_node['slivers'] = [sliver] - + # slivers always provide the ssh service login = Login({'authentication': 'ssh-keys', \ 'hostname': node['hostname'], 'port':'22', \ @@ -259,22 +259,22 @@ class SlabAggregate: rspec_node['services'] = [service] rspec_nodes.append(rspec_node) - return (rspec_nodes) + return (rspec_nodes) #def get_all_leases(self, slice_record = None): def get_all_leases(self): """ - Get list of lease dictionaries which all have the mandatory keys + Get list of lease dictionaries which all have the mandatory keys ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration'). - All the leases running or scheduled are returned. - - + All the leases running or scheduled are returned. + + ..note::There is no filtering of leases within a given time frame. - All the running or scheduled leases are returned. options + All the running or scheduled leases are returned. options removed SA 15/05/2013 - - + + """ - + #now = int(time.time()) #lease_filter = {'clip': now } @@ -287,8 +287,8 @@ class SlabAggregate: site_ids = [] rspec_leases = [] for lease in leases: - #as many leases as there are nodes in the job - for node in lease['reserved_nodes']: + #as many leases as there are nodes in the job + for node in lease['reserved_nodes']: rspec_lease = Lease() rspec_lease['lease_id'] = lease['lease_id'] #site = node['site_id'] @@ -303,18 +303,18 @@ class SlabAggregate: pass rspec_lease['start_time'] = lease['t_from'] rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ - / grain + / grain rspec_leases.append(rspec_lease) - return rspec_leases - - - -#from plc/aggregate.py + return rspec_leases + + + +#from plc/aggregate.py def get_rspec(self, slice_xrn=None, login=None, version = None, \ options=None): rspec = None - version_manager = VersionManager() + version_manager = VersionManager() version = version_manager.get_version(version) logger.debug("SlabAggregate \t get_rspec ***version %s \ version.type %s version.version %s options %s \r\n" \ @@ -327,13 +327,13 @@ class SlabAggregate: else: rspec_version = version_manager._get_version(version.type, \ version.version, 'manifest') - + slices, slivers = self.get_slice_and_slivers(slice_xrn, login) - #at this point sliver may be empty if no senslab job + #at this point sliver may be empty if no senslab job #is running for this user/slice. rspec = RSpec(version=rspec_version, user_options=options) - + #if slice and 'expires' in slice: #rspec.xml.set('expires',\ #datetime_to_string(utcparse(slice['expires'])) @@ -341,26 +341,26 @@ class SlabAggregate: #nodes, links = self.get_nodes(slice, slivers) logger.debug("\r\n \r\n SlabAggregate \tget_rspec *** \ slice_xrn %s slices %s\r\n \r\n"\ - %(slice_xrn, slices)) - - if options is not None: + %(slice_xrn, slices)) + + if options is not None: lease_option = options['list_leases'] else: #If no options are specified, at least print the resources lease_option = 'all' #if slice_xrn : #lease_option = 'all' - - + + if lease_option in ['all', 'resources']: - #if not options.get('list_leases') or options.get('list_leases') + #if not options.get('list_leases') or options.get('list_leases') #and options['list_leases'] != 'leases': - nodes = self.get_nodes(slices, slivers) + nodes = self.get_nodes(slices, slivers) logger.debug("\r\n \r\n SlabAggregate \ lease_option %s \ get rspec ******* nodes %s"\ %(lease_option, nodes[0])) - sites_set = set([node['location']['site'] for node in nodes] ) + sites_set = set([node['location']['site'] for node in nodes] ) #In case creating a job, slice_xrn is not set to None rspec.version.add_nodes(nodes) @@ -370,13 +370,13 @@ class SlabAggregate: #slices['record_id_user']).first() #ldap_username = (user.hrn).split('.')[1] - - + + #for one_slice in slices : ldap_username = slices[0]['hrn'] tmp = ldap_username.split('.') ldap_username = tmp[1].split('_')[0] - + if version.type == "Slab": rspec.version.add_connection_information(ldap_username, \ sites_set) @@ -388,12 +388,12 @@ class SlabAggregate: default_sliver%s \r\n" %(default_sliver)) for attrib in default_sliver: rspec.version.add_default_sliver_attribute(attrib, \ - default_sliver[attrib]) + default_sliver[attrib]) if lease_option in ['all','leases']: #leases = self.get_all_leases(slices) leases = self.get_all_leases() rspec.version.add_leases(leases) - + #logger.debug("SlabAggregate \tget_rspec ******* rspec_toxml %s \r\n"\ - #%(rspec.toxml())) - return rspec.toxml() + #%(rspec.toxml())) + return rspec.toxml() diff --git a/sfa/senslab/slabapi.py b/sfa/senslab/slabapi.py index 839a4278..8a0ed4a7 100644 --- a/sfa/senslab/slabapi.py +++ b/sfa/senslab/slabapi.py @@ -15,20 +15,20 @@ from sfa.util.xrn import Xrn, hrn_to_urn, get_authority from sfa.trust.certificate import Keypair, convert_public_key from sfa.trust.gid import create_uuid from sfa.trust.hierarchy import Hierarchy - + from sfa.senslab.slabaggregate import slab_xrn_object class SlabTestbedAPI(): """ Class enabled to use LDAP and OAR api calls. """ - + def __init__(self, config): """Creates an instance of OARrestapi and LDAPapi which will be used to issue calls to OAR or LDAP methods. - Set the time format and the testbed granularity used for OAR - reservation and leases. - + Set the time format and the testbed granularity used for OAR + reservation and leases. + :param config: configuration object from sfa.util.config - :type config: Config object + :type config: Config object """ self.oar = OARrestapi() self.ldap = LDAPapi() @@ -40,20 +40,20 @@ class SlabTestbedAPI(): #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \ #level=logging.DEBUG) return - - + + @staticmethod def GetMinExperimentDurationInSec(): return 600 - - @staticmethod + + @staticmethod def GetPeers (peer_filter=None ): """ Gathers registered authorities in SFA DB and looks for specific peer - if peer_filter is specified. + if peer_filter is specified. :param peer_filter: name of the site authority looked for. :type peer_filter: string :return: list of records. - + """ existing_records = {} @@ -61,7 +61,7 @@ class SlabTestbedAPI(): logger.debug("SLABDRIVER \tGetPeers peer_filter %s, \ " %(peer_filter)) all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all() - + for record in all_records: existing_records[(record.hrn, record.type)] = record if record.type not in existing_hrns_by_types: @@ -69,46 +69,46 @@ class SlabTestbedAPI(): else: existing_hrns_by_types[record.type].append(record.hrn) - + logger.debug("SLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\ %( existing_hrns_by_types)) - records_list = [] - - try: + records_list = [] + + try: if peer_filter: records_list.append(existing_records[(peer_filter,'authority')]) else : for hrn in existing_hrns_by_types['authority']: records_list.append(existing_records[(hrn,'authority')]) - + logger.debug("SLABDRIVER \tGetPeer \trecords_list %s " \ %(records_list)) except KeyError: pass - + return_records = records_list logger.debug("SLABDRIVER \tGetPeer return_records %s " \ %(return_records)) return return_records - - - #TODO : Handling OR request in make_ldap_filters_from_records - #instead of the for loop + + + #TODO : Handling OR request in make_ldap_filters_from_records + #instead of the for loop #over the records' list def GetPersons(self, person_filter=None): """ - Get the enabled users and their properties from Senslab LDAP. + Get the enabled users and their properties from Senslab LDAP. If a filter is specified, looks for the user whose properties match the filter, otherwise returns the whole enabled users'list. - :param person_filter: Must be a list of dictionnaries + :param person_filter: Must be a list of dictionnaries with users properties when not set to None. :param person_filter: list of dict - :return:Returns a list of users whose accounts are enabled + :return:Returns a list of users whose accounts are enabled found in ldap. :rtype: list of dicts - + """ logger.debug("SLABDRIVER \tGetPersons person_filter %s" \ %(person_filter)) @@ -117,23 +117,23 @@ class SlabTestbedAPI(): #If we are looking for a list of users (list of dict records) #Usually the list contains only one user record for searched_attributes in person_filter: - - #Get only enabled user accounts in senslab LDAP : + + #Get only enabled user accounts in senslab LDAP : #add a filter for make_ldap_filters_from_record person = self.ldap.LdapFindUser(searched_attributes, \ is_user_enabled=True) #If a person was found, append it to the list if person: person_list.append(person) - + #If the list is empty, return None if len(person_list) is 0: person_list = None - + else: - #Get only enabled user accounts in senslab LDAP : + #Get only enabled user accounts in senslab LDAP : #add a filter for make_ldap_filters_from_record - person_list = self.ldap.LdapFindUser(is_user_enabled=True) + person_list = self.ldap.LdapFindUser(is_user_enabled=True) return person_list @@ -145,20 +145,20 @@ class SlabTestbedAPI(): #SendRequest("GET_timezone") #return server_timestamp, server_tz - def DeleteJobs(self, job_id, username): - + def DeleteJobs(self, job_id, username): + """ Deletes the job with the specified job_id and username on OAR by posting a delete request to OAR. - + :param job_id: job id in OAR. - :param username: user's senslab login in LDAP. + :param username: user's senslab login in LDAP. :type job_id:integer :type username: string - + :return: dictionary with the job id and if delete has been successful (True) or no (False) :rtype: dict - """ + """ logger.debug("SLABDRIVER \tDeleteJobs jobid %s username %s "\ %(job_id, username)) if not job_id or job_id is -1: @@ -167,7 +167,7 @@ class SlabTestbedAPI(): reqdict = {} reqdict['method'] = "delete" reqdict['strval'] = str(job_id) - + answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', \ reqdict,username) @@ -179,19 +179,19 @@ class SlabTestbedAPI(): username %s" %(job_id, answer, username)) return ret - - + + ##TODO : Unused GetJobsId ? SA 05/07/12 #def GetJobsId(self, job_id, username = None ): #""" - #Details about a specific job. - #Includes details about submission time, jot type, state, events, + #Details about a specific job. + #Includes details about submission time, jot type, state, events, #owner, assigned ressources, walltime etc... - + #""" #req = "GET_jobs_id" #node_list_k = 'assigned_network_address' - ##Get job info from OAR + ##Get job info from OAR #job_info = self.oar.parser.SendRequest(req, job_id, username) #logger.debug("SLABDRIVER \t GetJobsId %s " %(job_info)) @@ -204,14 +204,14 @@ class SlabTestbedAPI(): #logger.debug("SLABDRIVER \t GetJobsId ERROR message %s "\ #%(job_info)) #return None - + #except KeyError: #logger.error("SLABDRIVER \tGetJobsId KeyError") - #return None - + #return None + #parsed_job_info = self.get_info_on_reserved_nodes(job_info, \ #node_list_k) - ##Replaces the previous entry + ##Replaces the previous entry ##"assigned_network_address" / "reserved_resources" ##with "node_ids" #job_info.update({'node_ids':parsed_job_info[node_list_k]}) @@ -219,74 +219,74 @@ class SlabTestbedAPI(): #logger.debug(" \r\nSLABDRIVER \t GetJobsId job_info %s " %(job_info)) #return job_info - + def GetJobsResources(self, job_id, username = None): - """ Gets the list of nodes associated with the job_id and username - if provided. + """ Gets the list of nodes associated with the job_id and username + if provided. Transforms the senslab hostnames to the corresponding SFA nodes hrns. - Rertuns dict key :'node_ids' , value : hostnames list + Rertuns dict key :'node_ids' , value : hostnames list :param username: user's LDAP login :paran job_id: job's OAR identifier. :type username: string :type job_id: integer - + :return: dicionary with nodes' hostnames belonging to the job. :rtype: dict """ req = "GET_jobs_id_resources" - - - #Get job resources list from OAR + + + #Get job resources list from OAR node_id_list = self.oar.parser.SendRequest(req, job_id, username) logger.debug("SLABDRIVER \t GetJobsResources %s " %(node_id_list)) - + hostname_list = \ self.__get_hostnames_from_oar_node_ids(node_id_list) - - #Replaces the previous entry "assigned_network_address" / + + #Replaces the previous entry "assigned_network_address" / #"reserved_resources" with "node_ids" job_info = {'node_ids': hostname_list} return job_info - + #def get_info_on_reserved_nodes(self, job_info, node_list_name): #""" #..warning:unused SA 23/05/13 #""" - ##Get the list of the testbed nodes records and make a + ##Get the list of the testbed nodes records and make a ##dictionnary keyed on the hostname out of it - #node_list_dict = self.GetNodes() + #node_list_dict = self.GetNodes() ##node_hostname_list = [] - #node_hostname_list = [node['hostname'] for node in node_list_dict] + #node_hostname_list = [node['hostname'] for node in node_list_dict] ##for node in node_list_dict: ##node_hostname_list.append(node['hostname']) #node_dict = dict(zip(node_hostname_list, node_list_dict)) #try : #reserved_node_hostname_list = [] #for index in range(len(job_info[node_list_name])): - ##job_info[node_list_name][k] = + ##job_info[node_list_name][k] = #reserved_node_hostname_list[index] = \ #node_dict[job_info[node_list_name][index]]['hostname'] - + #logger.debug("SLABDRIVER \t get_info_on_reserved_nodes \ #reserved_node_hostname_list %s" \ #%(reserved_node_hostname_list)) #except KeyError: #logger.error("SLABDRIVER \t get_info_on_reserved_nodes KEYERROR " ) - - #return reserved_node_hostname_list - + + #return reserved_node_hostname_list + def GetNodesCurrentlyInUse(self): - """Returns a list of all the nodes already involved in an oar running + """Returns a list of all the nodes already involved in an oar running job. - :rtype: list of nodes hostnames. + :rtype: list of nodes hostnames. """ - return self.oar.parser.SendRequest("GET_running_jobs") - + return self.oar.parser.SendRequest("GET_running_jobs") + def __get_hostnames_from_oar_node_ids(self, resource_id_list ): """Get the hostnames of the nodes from their OAR identifiers. Get the list of nodes dict using GetNodes and find the hostname @@ -299,17 +299,17 @@ class SlabTestbedAPI(): oar_id_node_dict = {} for node in full_nodes_dict_list: oar_id_node_dict[node['oar_id']] = node - - hostname_list = [] + + hostname_list = [] for resource_id in resource_id_list: #Because jobs requested "asap" do not have defined resources if resource_id is not "Undefined": hostname_list.append(\ oar_id_node_dict[resource_id]['hostname']) - + #hostname_list.append(oar_id_node_dict[resource_id]['hostname']) - return hostname_list - + return hostname_list + def GetReservedNodes(self, username = None): """ Get list of leases. Get the leases for the username if specified, otherwise get all the leases. Finds the nodes hostnames for each @@ -319,39 +319,39 @@ class SlabTestbedAPI(): :return: list of reservations dict :rtype: dict list """ - + #Get the nodes in use and the reserved nodes reservation_dict_list = \ self.oar.parser.SendRequest("GET_reserved_nodes", \ username = username) - - + + for resa in reservation_dict_list: logger.debug ("GetReservedNodes resa %s"%(resa)) #dict list of hostnames and their site resa['reserved_nodes'] = \ self.__get_hostnames_from_oar_node_ids(resa['resource_ids']) - + #del resa['resource_ids'] return reservation_dict_list - + def GetNodes(self, node_filter_dict = None, return_fields_list = None): """ - + Make a list of senslab nodes and their properties from information - given by OAR. Search for specific nodes if some filters are specified. + given by OAR. Search for specific nodes if some filters are specified. Nodes properties returned if no return_fields_list given: - 'hrn','archi','mobile','hostname','site','boot_state','node_id', - 'radio','posx','posy','oar_id','posz'. - + 'hrn','archi','mobile','hostname','site','boot_state','node_id', + 'radio','posx','posy','oar_id','posz'. + :param node_filter_dict: dictionnary of lists with node properties :type node_filter_dict: dict - :param return_fields_list: list of specific fields the user wants to be + :param return_fields_list: list of specific fields the user wants to be returned. :type return_fields_list: list :return: list of dictionaries with node properties :rtype: list - + """ node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full") node_dict_list = node_dict_by_id.values() @@ -360,12 +360,12 @@ class SlabTestbedAPI(): #No filtering needed return the list directly if not (node_filter_dict or return_fields_list): return node_dict_list - + return_node_list = [] if node_filter_dict: for filter_key in node_filter_dict: try: - #Filter the node_dict_list by each value contained in the + #Filter the node_dict_list by each value contained in the #list node_filter_dict[filter_key] for value in node_filter_dict[filter_key]: for node in node_dict_list: @@ -373,7 +373,7 @@ class SlabTestbedAPI(): if return_fields_list : tmp = {} for k in return_fields_list: - tmp[k] = node[k] + tmp[k] = node[k] return_node_list.append(tmp) else: return_node_list.append(node) @@ -383,9 +383,9 @@ class SlabTestbedAPI(): return return_node_list - - - + + + @staticmethod def AddSlice(slice_record, user_record): """Add slice to the local senslab sfa tables if the slice comes @@ -393,28 +393,27 @@ class SlabTestbedAPI(): although the user has already a LDAP login. Called by verify_slice during lease/sliver creation. :param slice_record: record of slice, must contain hrn, gid, slice_id - and authority of the slice. + and authority of the slice. :type slice_record: dictionary :param user_record: record of the user :type user_record: RegUser """ - - sfa_record = RegSlice(hrn=slice_record['hrn'], - gid=slice_record['gid'], - pointer=slice_record['slice_id'], - authority=slice_record['authority']) - - logger.debug("SLABDRIVER.PY AddSlice sfa_record %s user_record %s" \ - %(sfa_record, user_record)) + + sfa_record = RegSlice(hrn=slice_record['hrn'], + gid=slice_record['gid'], + pointer=slice_record['slice_id'], + authority=slice_record['authority']) + logger.debug("SLABDRIVER.PY AddSlice sfa_record %s user_record %s" + % (sfa_record, user_record)) sfa_record.just_created() dbsession.add(sfa_record) - dbsession.commit() + dbsession.commit() #Update the reg-researcher dependance table sfa_record.reg_researchers = [user_record] - dbsession.commit() - + dbsession.commit() + return - + def GetSites(self, site_filter_name_list = None, return_fields_list = None): site_dict = self.oar.parser.SendRequest("GET_sites") @@ -423,7 +422,7 @@ class SlabTestbedAPI(): if not ( site_filter_name_list or return_fields_list): return_site_list = site_dict.values() return return_site_list - + for site_filter_name in site_filter_name_list: if site_filter_name in site_dict: if return_fields_list: @@ -437,40 +436,40 @@ class SlabTestbedAPI(): return_site_list.append(tmp) else: return_site_list.append( site_dict[site_filter_name]) - + return return_site_list - - - - #TODO : Check rights to delete person + + + + #TODO : Check rights to delete person def DeletePerson(self, person_record): """ Disable an existing account in senslab LDAP. - Users and techs can only delete themselves. PIs can only - delete themselves and other non-PIs at their sites. + Users and techs can only delete themselves. PIs can only + delete themselves and other non-PIs at their sites. ins can delete anyone. :param person_record: user's record :type person_record: dict :return: True if successful, False otherwise. :rtype: boolean - + """ #Disable user account in senslab LDAP ret = self.ldap.LdapMarkUserAsDeleted(person_record) logger.warning("SLABDRIVER DeletePerson %s " %(person_record)) return ret['bool'] - - + + def DeleteSlice(self, slice_record): - """ Deletes the specified slice and kills the jobs associated with + """ Deletes the specified slice and kills the jobs associated with the slice if any, using DeleteSliceFromNodes. - + :return: True if all the jobs in the slice have been deleted, or the list of jobs that could not be deleted otherwise. - :rtype: list or boolean - + :rtype: list or boolean + """ ret = self.DeleteSliceFromNodes(slice_record) delete_failed = None @@ -479,15 +478,15 @@ class SlabTestbedAPI(): if delete_failed is None: delete_failed = [] delete_failed.append(job_id) - + logger.info("SLABDRIVER DeleteSlice %s answer %s"%(slice_record, \ delete_failed)) return delete_failed or True - + @staticmethod def __add_person_to_db(user_dict): """ - Add a federated user straight to db when the user issues a lease + Add a federated user straight to db when the user issues a lease request with senslab nodes and that he has not registered with senslab yet (that is he does not have a LDAP entry yet). Uses parts of the routines in SlabImport when importing user from LDAP. @@ -495,15 +494,15 @@ class SlabTestbedAPI(): :param user_dict: Must contain email, hrn and pkey to get a GID and be added to the SFA db. :type user_dict: dict - + """ check_if_exists = \ dbsession.query(RegUser).filter_by(email = user_dict['email']).first() #user doesn't exists if not check_if_exists: logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \ - " %(user_dict)) - hrn = user_dict['hrn'] + " %(user_dict)) + hrn = user_dict['hrn'] person_urn = hrn_to_urn(hrn, 'user') pubkey = user_dict['pkey'] try: @@ -513,8 +512,8 @@ class SlabTestbedAPI(): logger.warn('__add_person_to_db: unable to convert public \ key for %s' %(hrn )) pkey = Keypair(create=True) - - + + if pubkey is not None and pkey is not None : hierarchy = Hierarchy() person_gid = hierarchy.create_gid(person_urn, create_uuid(), \ @@ -524,7 +523,7 @@ class SlabTestbedAPI(): SLAB IMPORTER PERSON EMAIL OK email %s "\ %(user_dict['email'])) person_gid.set_email(user_dict['email']) - + user_record = RegUser(hrn=hrn , pointer= '-1', \ authority=get_authority(hrn), \ email=user_dict['email'], gid = person_gid) @@ -532,20 +531,20 @@ class SlabTestbedAPI(): user_record.just_created() dbsession.add (user_record) dbsession.commit() - return - + return + def AddPerson(self, record): - """Adds a new account. Any fields specified in records are used, - otherwise defaults are used. Creates an appropriate login by calling + """Adds a new account. Any fields specified in records are used, + otherwise defaults are used. Creates an appropriate login by calling LdapAddUser. :param record: dictionary with the sfa user's properties. - :return: The uid of the added person if sucessful, otherwise returns + :return: The uid of the added person if sucessful, otherwise returns the error message from LDAP. :rtype: interger or string """ ret = self.ldap.LdapAddUser(record) - + if ret['bool'] is True: record['hrn'] = self.root_auth + '.' + ret['uid'] logger.debug("SLABDRIVER AddPerson return code %s record %s \r\n "\ @@ -554,64 +553,64 @@ class SlabTestbedAPI(): return ret['uid'] else: return ret['message'] - - - + + + #TODO AddPersonKey 04/07/2012 SA def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict): """Adds a new key to the specified account. Adds the key to the senslab ldap, provided that the person_uid is valid. Non-admins can only modify their own keys. - + :param person_uid: user's senslab login in LDAP - :param old_attributes_dict: dict with the user's old sshPublicKey + :param old_attributes_dict: dict with the user's old sshPublicKey :param new_key_dict:dict with the user's new sshPublicKey :type person_uid: string - + :rtype: Boolean :return: True if the key has been modified, False otherwise. - + """ ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \ new_key_dict) logger.warning("SLABDRIVER AddPersonKey EMPTY - DO NOTHING \r\n ") return ret['bool'] - - def DeleteLeases(self, leases_id_list, slice_hrn ): + + def DeleteLeases(self, leases_id_list, slice_hrn ): """ Deletes several leases, based on their job ids and the slice they are associated with. Uses DeleteJobs to delete the jobs on OAR. Note that one slice can contain multiple jobs, and in this case all the jobs in the leases_id_list MUST belong to ONE slice, - since there is only one slice hrn provided here. + since there is only one slice hrn provided here. :param leases_id_list: list of job ids that belong to the slice whose slice hrn is provided. - :param slice_hrn: the slice hrn . + :param slice_hrn: the slice hrn . ..warning: Does not have a return value since there was no easy way to handle failure when dealing with multiple job delete. Plus, there was no easy way to report it to the user. - """ + """ logger.debug("SLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \ \r\n " %(leases_id_list, slice_hrn)) for job_id in leases_id_list: self.DeleteJobs(job_id, slice_hrn) - return + return @staticmethod def _process_walltime(duration): """ Calculates the walltime in seconds from the duration in H:M:S specified in the RSpec. - + """ if duration: - # Fixing the walltime by adding a few delays. + # Fixing the walltime by adding a few delays. # First put the walltime in seconds oarAdditionalDelay = 20; # additional delay for /bin/sleep command to # take in account prologue and epilogue scripts execution # int walltimeAdditionalDelay = 240; additional delay - #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT + #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT #in oar.conf # Put the duration in seconds first #desired_walltime = duration * 60 @@ -631,30 +630,30 @@ class SlabTestbedAPI(): else: logger.log_exc(" __process_walltime duration null") - - return walltime, sleep_walltime - - @staticmethod + + return walltime, sleep_walltime + + @staticmethod def _create_job_structure_request_for_OAR(lease_dict): """ Creates the structure needed for a correct POST on OAR. Makes the timestamp transformation into the appropriate format. - Sends the POST request to create the job with the resources in + Sends the POST request to create the job with the resources in added_nodes. - + """ nodeid_list = [] reqdict = {} - - reqdict['workdir'] = '/tmp' - reqdict['resource'] = "{network_address in (" - for node in lease_dict['added_nodes']: + reqdict['workdir'] = '/tmp' + reqdict['resource'] = "{network_address in (" + + for node in lease_dict['added_nodes']: logger.debug("\r\n \r\n OARrestapi \t \ __create_job_structure_request_for_OAR node %s" %(node)) - # Get the ID of the node + # Get the ID of the node nodeid = node reqdict['resource'] += "'" + nodeid + "', " nodeid_list.append(nodeid) @@ -674,7 +673,7 @@ class SlabTestbedAPI(): reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime) #In case of a scheduled experiment (not immediate) - #To run an XP immediately, don't specify date and time in RSpec + #To run an XP immediately, don't specify date and time in RSpec #They will be set to None. if lease_dict['lease_start_time'] is not '0': #Readable time accepted by OAR @@ -682,28 +681,28 @@ class SlabTestbedAPI(): int(lease_dict['lease_start_time'])).\ strftime(lease_dict['time_format']) reqdict['reservation'] = start_time - #If there is not start time, Immediate XP. No need to add special + #If there is not start time, Immediate XP. No need to add special # OAR parameters - reqdict['type'] = "deploy" + reqdict['type'] = "deploy" reqdict['directory'] = "" reqdict['name'] = "SFA_" + lease_dict['slice_user'] return reqdict - - + + def LaunchExperimentOnOAR(self, added_nodes, slice_name, \ lease_start_time, lease_duration, slice_user=None): - + """ - Create a job request structure based on the information provided - and post the job on OAR. + Create a job request structure based on the information provided + and post the job on OAR. :param added_nodes: list of nodes that belong to the described lease. :param slice_name: the slice hrn associated to the lease. :param lease_start_time: timestamp of the lease startting time. :param lease_duration: lease durationin minutes - + """ lease_dict = {} lease_dict['lease_start_time'] = lease_start_time @@ -714,54 +713,54 @@ class SlabTestbedAPI(): lease_dict['grain'] = self.GetLeaseGranularity() lease_dict['time_format'] = self.time_format - + logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR slice_user %s\ - \r\n " %(slice_user)) + \r\n " %(slice_user)) #Create the request for OAR reqdict = self._create_job_structure_request_for_OAR(lease_dict) - # first step : start the OAR job and update the job + # first step : start the OAR job and update the job logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\ - \r\n " %(reqdict)) - + \r\n " %(reqdict)) + answer = self.oar.POSTRequestToOARRestAPI('POST_job', \ reqdict, slice_user) logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s " %(answer)) - try: + try: jobid = answer['id'] except KeyError: logger.log_exc("SLABDRIVER \tLaunchExperimentOnOAR \ Impossible to create job %s " %(answer)) return None - - - + + + if jobid : logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s \ added_nodes %s slice_user %s" %(jobid, added_nodes, \ slice_user)) - - + + return jobid - - + + def AddLeases(self, hostname_list, slice_record, \ lease_start_time, lease_duration): - + """Creates a job in OAR corresponding to the information provided - as parameters. Adds the job id and the slice hrn in the senslab + as parameters. Adds the job id and the slice hrn in the senslab database so that we are able to know which slice has which nodes. - - :param hostname_list: list of nodes' OAR hostnames. + + :param hostname_list: list of nodes' OAR hostnames. :param slice_record: sfa slice record, must contain login and hrn. :param lease_start_time: starting time , unix timestamp format :param lease_duration: duration in minutes - + :type hostname_list: list :type slice_record: dict :type lease_start_time: integer :type lease_duration: integer - + """ logger.debug("SLABDRIVER \r\n \r\n \t AddLeases hostname_list %s \ slice_record %s lease_start_time %s lease_duration %s "\ @@ -780,73 +779,73 @@ class SlabTestbedAPI(): strftime(self.time_format) end_time = lease_start_time + lease_duration - + logger.debug("SLABDRIVER \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ %s %s %s "%(slice_record['hrn'], job_id, end_time)) - - + + logger.debug("SLABDRIVER \r\n \r\n \t AddLeases %s %s %s " \ %(type(slice_record['hrn']), type(job_id), type(end_time))) - + slab_ex_row = SenslabXP(slice_hrn = slice_record['hrn'], \ job_id = job_id, end_time= end_time) - + logger.debug("SLABDRIVER \r\n \r\n \t AddLeases slab_ex_row %s" \ %(slab_ex_row)) slab_dbsession.add(slab_ex_row) slab_dbsession.commit() - + logger.debug("SLABDRIVER \t AddLeases hostname_list start_time %s " \ %(start_time)) - + return - - + + #Delete the jobs from job_senslab table def DeleteSliceFromNodes(self, slice_record): - """ Deletes all the running or scheduled jobs of a given slice - given its record. + """ Deletes all the running or scheduled jobs of a given slice + given its record. :param slice_record: record of the slice :type slice_record: dict - + :return: dict of the jobs'deletion status. Success= True, Failure= False, for each job id. :rtype: dict """ logger.debug("SLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record)) - - if isinstance(slice_record['oar_job_id'], list): + + if isinstance(slice_record['oar_job_id'], list): oar_bool_answer = {} for job_id in slice_record['oar_job_id']: ret = self.DeleteJobs(job_id, slice_record['user']) - + oar_bool_answer.update(ret) else: oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], \ slice_record['user'])] - + return oar_bool_answer - - - + + + def GetLeaseGranularity(self): """ Returns the granularity of an experiment in the Senslab testbed. - OAR uses seconds for experiments duration , the granulaity is also - defined in seconds. + OAR uses seconds for experiments duration , the granulaity is also + defined in seconds. Experiments which last less than 10 min (600 sec) are invalid""" return self.grain - - + + @staticmethod def update_jobs_in_slabdb( job_oar_list, jobs_psql): """ Cleans the slab db by deleting expired and cancelled jobs. Compares the list of job ids given by OAR with the job ids that - are already in the database, deletes the jobs that are no longer in + are already in the database, deletes the jobs that are no longer in the OAR job id list. :param job_oar_list: list of job ids coming from OAR :type job_oar_list: list - :param job_psql: list of job ids cfrom the database. + :param job_psql: list of job ids cfrom the database. type job_psql: list """ #Turn the list into a set @@ -860,11 +859,11 @@ class SlabTestbedAPI(): if len(deleted_jobs) > 0: slab_dbsession.query(SenslabXP).filter(SenslabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') slab_dbsession.commit() - + return - - + + def GetLeases(self, lease_filter_dict=None, login=None): """ Get the list of leases from OAR with complete information about which slice owns which jobs and nodes. @@ -879,7 +878,7 @@ class SlabTestbedAPI(): 'slice_hrn', 'resource_ids', 't_from', 't_until' :rtype: list """ - + unfiltered_reservation_list = self.GetReservedNodes(login) reservation_list = [] @@ -889,62 +888,62 @@ class SlabTestbedAPI(): #Create user dict first to avoid looking several times for #the same user in LDAP SA 27/07/12 job_oar_list = [] - + jobs_psql_query = slab_dbsession.query(SenslabXP).all() jobs_psql_dict = dict([(row.job_id, row.__dict__ ) for row in jobs_psql_query ]) #jobs_psql_dict = jobs_psql_dict) logger.debug("SLABDRIVER \tGetLeases jobs_psql_dict %s"\ %(jobs_psql_dict)) jobs_psql_id_list = [ row.job_id for row in jobs_psql_query ] - - - + + + for resa in unfiltered_reservation_list: logger.debug("SLABDRIVER \tGetLeases USER %s"\ - %(resa['user'])) - #Construct list of jobs (runing, waiting..) in oar - job_oar_list.append(resa['lease_id']) + %(resa['user'])) + #Construct list of jobs (runing, waiting..) in oar + job_oar_list.append(resa['lease_id']) #If there is information on the job in SLAB DB ] - #(slice used and job id) + #(slice used and job id) if resa['lease_id'] in jobs_psql_dict: job_info = jobs_psql_dict[resa['lease_id']] logger.debug("SLABDRIVER \tGetLeases job_info %s"\ - %(job_info)) + %(job_info)) resa['slice_hrn'] = job_info['slice_hrn'] resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') - - #otherwise, assume it is a senslab slice: + + #otherwise, assume it is a senslab slice: else: resa['slice_id'] = hrn_to_urn(self.root_auth+'.'+ \ - resa['user'] +"_slice" , 'slice') + resa['user'] +"_slice" , 'slice') resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn() - resa['component_id_list'] = [] + resa['component_id_list'] = [] #Transform the hostnames into urns (component ids) for node in resa['reserved_nodes']: - + slab_xrn = slab_xrn_object(self.root_auth, node) resa['component_id_list'].append(slab_xrn.urn) - + if lease_filter_dict: logger.debug("SLABDRIVER \tGetLeases resa_ %s \ - \r\n leasefilter %s" %(resa, lease_filter_dict)) - + \r\n leasefilter %s" %(resa, lease_filter_dict)) + if lease_filter_dict['name'] == resa['slice_hrn']: reservation_list.append(resa) - + if lease_filter_dict is None: reservation_list = unfiltered_reservation_list - - + + self.update_jobs_in_slabdb(job_oar_list, jobs_psql_id_list) - + logger.debug(" SLABDRIVER.PY \tGetLeases reservation_list %s"\ %(reservation_list)) return reservation_list - - - + + + #TODO FUNCTIONS SECTION 04/07/2012 SA @@ -953,88 +952,88 @@ class SlabTestbedAPI(): #@staticmethod #def UnBindObjectFromPeer( auth, object_type, object_id, shortname): #""" This method is a hopefully temporary hack to let the sfa correctly - #detach the objects it creates from a remote peer object. This is - #needed so that the sfa federation link can work in parallel with - #RefreshPeer, as RefreshPeer depends on remote objects being correctly + #detach the objects it creates from a remote peer object. This is + #needed so that the sfa federation link can work in parallel with + #RefreshPeer, as RefreshPeer depends on remote objects being correctly #marked. #Parameters: #auth : struct, API authentication structure - #AuthMethod : string, Authentication method to use + #AuthMethod : string, Authentication method to use #object_type : string, Object type, among 'site','person','slice', #'node','key' #object_id : int, object_id - #shortname : string, peer shortname + #shortname : string, peer shortname #FROM PLC DOC - + #""" #logger.warning("SLABDRIVER \tUnBindObjectFromPeer EMPTY-\ #DO NOTHING \r\n ") - #return - - ##TODO Is BindObjectToPeer still necessary ? Currently does nothing + #return + + ##TODO Is BindObjectToPeer still necessary ? Currently does nothing ##04/07/2012 SA #|| Commented out 28/05/13 SA #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \ #remote_object_id=None): - #"""This method is a hopefully temporary hack to let the sfa correctly - #attach the objects it creates to a remote peer object. This is needed - #so that the sfa federation link can work in parallel with RefreshPeer, + #"""This method is a hopefully temporary hack to let the sfa correctly + #attach the objects it creates to a remote peer object. This is needed + #so that the sfa federation link can work in parallel with RefreshPeer, #as RefreshPeer depends on remote objects being correctly marked. #Parameters: - #shortname : string, peer shortname - #remote_object_id : int, remote object_id, set to 0 if unknown + #shortname : string, peer shortname + #remote_object_id : int, remote object_id, set to 0 if unknown #FROM PLC API DOC - + #""" #logger.warning("SLABDRIVER \tBindObjectToPeer EMPTY - DO NOTHING \r\n ") #return - + ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA ##Funciton should delete and create another job since oin senslab slice=job - #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None): - #"""Updates the parameters of an existing slice with the values in + #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None): + #"""Updates the parameters of an existing slice with the values in #slice_fields. - #Users may only update slices of which they are members. - #PIs may update any of the slices at their sites, or any slices of + #Users may only update slices of which they are members. + #PIs may update any of the slices at their sites, or any slices of #which they are members. Admins may update any slice. #Only PIs and admins may update max_nodes. Slices cannot be renewed #(by updating the expires parameter) more than 8 weeks into the future. #Returns 1 if successful, faults otherwise. #FROM PLC API DOC - - #""" + + #""" #logger.warning("SLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ") #return - + #Unused SA 30/05/13, we only update the user's key or we delete it. ##TODO UpdatePerson 04/07/2012 SA #def UpdatePerson(self, slab_hrn, federated_hrn, person_fields=None): - #"""Updates a person. Only the fields specified in person_fields + #"""Updates a person. Only the fields specified in person_fields #are updated, all other fields are left untouched. #Users and techs can only update themselves. PIs can only update #themselves and other non-PIs at their sites. #Returns 1 if successful, faults otherwise. #FROM PLC API DOC - + #""" ##new_row = FederatedToSenslab(slab_hrn, federated_hrn) ##slab_dbsession.add(new_row) ##slab_dbsession.commit() - + #logger.debug("SLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ") #return - + @staticmethod def GetKeys(key_filter=None): """Returns a dict of dict based on the key string. Each dict entry - contains the key id, the ssh key, the user's email and the + contains the key id, the ssh key, the user's email and the user's hrn. - If key_filter is specified and is an array of key identifiers, - only keys matching the filter will be returned. + If key_filter is specified and is an array of key identifiers, + only keys matching the filter will be returned. Admin may query all keys. Non-admins may only query their own keys. FROM PLC API DOC - + :return: dict with ssh key as key and dicts as value. :rtype: dict """ @@ -1042,19 +1041,19 @@ class SlabTestbedAPI(): keys = dbsession.query(RegKey).options(joinedload('reg_user')).all() else : keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all() - + key_dict = {} for key in keys: key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, \ 'email': key.reg_user.email, 'hrn':key.reg_user.hrn} - + #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ #for user in ldap_rslt) - + logger.debug("SLABDRIVER GetKeys -key_dict %s \r\n " %(key_dict)) return key_dict - + #TODO : test def DeleteKey(self, user_record, key_string): """ Deletes a key in the LDAP entry of the specified user. @@ -1066,7 +1065,7 @@ class SlabTestbedAPI(): :type user_record: dict :return: True if sucessful, False if not. :rtype: Boolean - + """ all_user_keys = user_record['keys'] all_user_keys.remove(key_string) @@ -1075,30 +1074,30 @@ class SlabTestbedAPI(): logger.debug("SLABDRIVER DeleteKey %s- "%(ret)) return ret['bool'] - - - - @staticmethod + + + + @staticmethod def _sql_get_slice_info( slice_filter ): """ - Get the slice record based on the slice hrn. Fetch the record of the + Get the slice record based on the slice hrn. Fetch the record of the user associated with the slice by usingjoinedload based on t he reg_researcher relationship. :param slice_filter: the slice hrn we are looking for :type slice_filter: string :return: the slice record enhanced with the user's information if the - slice was found, None it wasn't. + slice was found, None it wasn't. :rtype: dict or None. """ - #DO NOT USE RegSlice - reg_researchers to get the hrn - #of the user otherwise will mess up the RegRecord in + #DO NOT USE RegSlice - reg_researchers to get the hrn + #of the user otherwise will mess up the RegRecord in #Resolve, don't know why - SA 08/08/2012 - + #Only one entry for one user = one slice in slab_xp table #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn = slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() - if raw_slicerec: + if raw_slicerec: #load_reg_researcher #raw_slicerec.reg_researchers raw_slicerec = raw_slicerec.__dict__ @@ -1109,14 +1108,14 @@ class SlabTestbedAPI(): #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] #del slicerec['reg_researchers']['_sa_instance_state'] return slicerec - + else : return None - - @staticmethod - def _sql_get_slice_info_from_user(slice_filter ): + + @staticmethod + def _sql_get_slice_info_from_user(slice_filter ): """ - Get the slice record based on the user recordid by using a joinedload + Get the slice record based on the user recordid by using a joinedload on the relationship reg_slices_as_researcher. Format the sql record into a dict with the mandatory fields for user and slice. :return: dict with slice record and user record if the record was found @@ -1126,7 +1125,7 @@ class SlabTestbedAPI(): #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id = slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() - #Put it in correct order + #Put it in correct order user_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'email', 'pointer'] slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'pointer'] if raw_slicerec: @@ -1144,12 +1143,12 @@ class SlabTestbedAPI(): #del raw_slicerec['reg_slices_as_researcher'] #slicerec['reg_researchers'] = raw_slicerec ##del slicerec['_sa_instance_state'] - + return slicerec - + else: return None - + def _get_slice_records(self, slice_filter = None, \ slice_filter_type = None): """ @@ -1159,94 +1158,94 @@ class SlabTestbedAPI(): :type slice_filter: string :param slice_filter_type: describes the slice filter type used, can be slice_hrn or record_id_user - :type: string + :type: string :return: the slice record - :rtype:dict + :rtype:dict ..seealso:_sql_get_slice_info_from_user ..seealso: _sql_get_slice_info """ - + #Get list of slices based on the slice hrn if slice_filter_type == 'slice_hrn': - + #if get_authority(slice_filter) == self.root_auth: - #login = slice_filter.split(".")[1].split("_")[0] - + #login = slice_filter.split(".")[1].split("_")[0] + slicerec = self._sql_get_slice_info(slice_filter) - + if slicerec is None: - return None - #return login, None - - #Get slice based on user id - if slice_filter_type == 'record_id_user': - + return None + #return login, None + + #Get slice based on user id + if slice_filter_type == 'record_id_user': + slicerec = self._sql_get_slice_info_from_user(slice_filter) - + if slicerec: fixed_slicerec_dict = slicerec - #At this point if there is no login it means + #At this point if there is no login it means #record_id_user filter has been used for filtering #if login is None : ##If theslice record is from senslab #if fixed_slicerec_dict['peer_authority'] is None: - #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] #return login, fixed_slicerec_dict - return fixed_slicerec_dict - - - + return fixed_slicerec_dict + + + def GetSlices(self, slice_filter = None, slice_filter_type = None, \ login=None): """ Get the slice records from the slab db and add lease information - if any. - + if any. + :param slice_filter: can be the slice hrn or slice record id in the db depending on the slice_filter_type. - :param slice_filter_type: defines the type of the filtering used, Can be + :param slice_filter_type: defines the type of the filtering used, Can be either 'slice_hrn' or "record_id'. :type slice_filter: string :type slice_filter_type: string - :return: a slice dict if slice_filter and slice_filter_type + :return: a slice dict if slice_filter and slice_filter_type are specified and a matching entry is found in the db. The result is put into a list.Or a list of slice dictionnaries if no filters are - specified. - + specified. + :rtype: list """ #login = None authorized_filter_types_list = ['slice_hrn', 'record_id_user'] return_slicerec_dictlist = [] - - #First try to get information on the slice based on the filter provided + + #First try to get information on the slice based on the filter provided if slice_filter_type in authorized_filter_types_list: fixed_slicerec_dict = \ self._get_slice_records(slice_filter, slice_filter_type) slice_hrn = fixed_slicerec_dict['hrn'] - + logger.debug(" SLABDRIVER \tGetSlices login %s \ slice record %s slice_filter %s \ slice_filter_type %s " %(login, \ fixed_slicerec_dict, slice_filter, \ slice_filter_type)) - - - #Now we have the slice record fixed_slicerec_dict, get the + + + #Now we have the slice record fixed_slicerec_dict, get the #jobs associated to this slice leases_list = [] - + leases_list = self.GetLeases(login = login) - #If no job is running or no job scheduled - #return only the slice record + #If no job is running or no job scheduled + #return only the slice record if leases_list == [] and fixed_slicerec_dict: return_slicerec_dictlist.append(fixed_slicerec_dict) - - #If several jobs for one slice , put the slice record into + + #If several jobs for one slice , put the slice record into # each lease information dict - - - for lease in leases_list : - slicerec_dict = {} + + + for lease in leases_list : + slicerec_dict = {} logger.debug("SLABDRIVER.PY \tGetSlices slice_filter %s \ \ lease['slice_hrn'] %s" \ %(slice_filter, lease['slice_hrn'])) @@ -1255,9 +1254,9 @@ class SlabTestbedAPI(): slicerec_dict['hrn'] = lease['slice_hrn'] slicerec_dict['user'] = lease['user'] slicerec_dict['oar_job_id'] = lease['lease_id'] - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) + slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) slicerec_dict.update({'node_ids':lease['reserved_nodes']}) - + #Update lease dict with the slice record if fixed_slicerec_dict: fixed_slicerec_dict['oar_job_id'] = [] @@ -1265,49 +1264,49 @@ class SlabTestbedAPI(): slicerec_dict.update(fixed_slicerec_dict) #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) - - return_slicerec_dictlist.append(slicerec_dict) + + return_slicerec_dictlist.append(slicerec_dict) logger.debug("SLABDRIVER.PY \tGetSlices \ OHOHOHOH %s" %(return_slicerec_dictlist )) - + logger.debug("SLABDRIVER.PY \tGetSlices \ slicerec_dict %s return_slicerec_dictlist %s \ lease['reserved_nodes'] \ %s" %(slicerec_dict, return_slicerec_dictlist, \ lease['reserved_nodes'] )) - + logger.debug("SLABDRIVER.PY \tGetSlices RETURN \ return_slicerec_dictlist %s" \ %(return_slicerec_dictlist)) - + return return_slicerec_dictlist - - + + else: #Get all slices from the senslab sfa database , - #put them in dict format - #query_slice_list = dbsession.query(RegRecord).all() - query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() + #put them in dict format + #query_slice_list = dbsession.query(RegRecord).all() + query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() - for record in query_slice_list: + for record in query_slice_list: tmp = record.__dict__ tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__ #del tmp['reg_researchers']['_sa_instance_state'] return_slicerec_dictlist.append(tmp) #return_slicerec_dictlist.append(record.__dict__) - + #Get all the jobs reserved nodes leases_list = self.GetReservedNodes() - - + + for fixed_slicerec_dict in return_slicerec_dictlist: - slicerec_dict = {} + slicerec_dict = {} #Check if the slice belongs to a senslab user if fixed_slicerec_dict['peer_authority'] is None: - owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] else: owner = None - for lease in leases_list: + for lease in leases_list: if owner == lease['user']: slicerec_dict['oar_job_id'] = lease['lease_id'] @@ -1316,19 +1315,19 @@ class SlabTestbedAPI(): %(lease )) slicerec_dict.update({'node_ids':lease['reserved_nodes']}) - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) + slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) slicerec_dict.update(fixed_slicerec_dict) #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) #return_slicerec_dictlist.append(slicerec_dict) fixed_slicerec_dict.update(slicerec_dict) - + logger.debug("SLABDRIVER.PY \tGetSlices RETURN \ return_slicerec_dictlist %s \slice_filter %s " \ %(return_slicerec_dictlist, slice_filter)) return return_slicerec_dictlist - + #Update slice unused, therefore sfa_fields_to_slab_fields unused @@ -1341,15 +1340,15 @@ class SlabTestbedAPI(): #slab_record = {} ##for field in record: ## slab_record[field] = record[field] - + #if sfa_type == "slice": - ##instantion used in get_slivers ? + ##instantion used in get_slivers ? #if not "instantiation" in slab_record: #slab_record["instantiation"] = "senslab-instantiated" - ##slab_record["hrn"] = hrn_to_pl_slicename(hrn) - ##Unused hrn_to_pl_slicename because Slab's hrn already + ##slab_record["hrn"] = hrn_to_pl_slicename(hrn) + ##Unused hrn_to_pl_slicename because Slab's hrn already ##in the appropriate form SA 23/07/12 - #slab_record["hrn"] = hrn + #slab_record["hrn"] = hrn #logger.debug("SLABDRIVER.PY sfa_fields_to_slab_fields \ #slab_record %s " %(slab_record['hrn'])) #if "url" in record: @@ -1358,7 +1357,7 @@ class SlabTestbedAPI(): #slab_record["description"] = record["description"] #if "expires" in record: #slab_record["expires"] = int(record["expires"]) - + ##nodes added by OAR only and then imported to SFA ##elif type == "node": ##if not "hostname" in slab_record: @@ -1367,8 +1366,8 @@ class SlabTestbedAPI(): ##slab_record["hostname"] = record["hostname"] ##if not "model" in slab_record: ##slab_record["model"] = "geni" - - ##One authority only + + ##One authority only ##elif type == "authority": ##slab_record["login_base"] = hrn_to_slab_login_base(hrn) @@ -1387,12 +1386,11 @@ class SlabTestbedAPI(): #return slab_record - - - - - - - - \ No newline at end of file + + + + + + + diff --git a/sfa/senslab/slabdriver.py b/sfa/senslab/slabdriver.py index c80f762d..b37f8b3b 100644 --- a/sfa/senslab/slabdriver.py +++ b/sfa/senslab/slabdriver.py @@ -13,32 +13,31 @@ from sfa.util.xrn import Xrn, hrn_to_urn, get_authority from sfa.senslab.slabpostgres import SlabDB - - + + from sfa.senslab.slabaggregate import SlabAggregate, slab_xrn_to_hostname - + from sfa.senslab.slabslices import SlabSlices from sfa.senslab.slabapi import SlabTestbedAPI - class SlabDriver(Driver): """ Senslab Driver class inherited from Driver generic class. - + Contains methods compliant with the SFA standard and the testbed infrastructure (calls to LDAP and OAR). - - .. seealso:: Driver class - + + ..seealso:: Driver class + """ def __init__(self, config): - """ - + """ + Sets the senslab SFA config parameters , instanciates the testbed api and the senslab database. - + :param config: senslab SFA configuration object :type config: Config object """ @@ -48,43 +47,43 @@ class SlabDriver(Driver): self.db = SlabDB(config, debug = False) self.slab_api = SlabTestbedAPI(config) self.cache = None - + def augment_records_with_testbed_info (self, record_list ): - """ - - Adds specific testbed info to the records. - + """ + + Adds specific testbed info to the records. + :param record_list: list of sfa dictionaries records :type record_list: list :return: list of records with extended information in each record :rtype: list """ return self.fill_record_info (record_list) - + def fill_record_info(self, record_list): """ For each SFA record, fill in the senslab specific and SFA specific - fields in the record. - + fields in the record. + :param record_list: list of sfa dictionaries records :type record_list: list :return: list of records with extended information in each record :rtype: list - + .. warnings:: Should not be modifying record_list directly because modi - fication are kept outside the method's scope. Howerver, there is no + fication are kept outside the method's scope. Howerver, there is no other way to do it given the way it's called in registry manager. """ - + logger.debug("SLABDRIVER \tfill_record_info records %s " %(record_list)) if not isinstance(record_list, list): record_list = [record_list] - + try: for record in record_list: - #If the record is a SFA slice record, then add information - #about the user of this slice. This kind of + #If the record is a SFA slice record, then add information + #about the user of this slice. This kind of #information is in the Senslab's DB. if str(record['type']) == 'slice': if 'reg_researchers' in record and \ @@ -93,41 +92,41 @@ class SlabDriver(Driver): record['reg_researchers'][0].__dict__ record.update({'PI':[record['reg_researchers']['hrn']], 'researcher': [record['reg_researchers']['hrn']], - 'name':record['hrn'], + 'name':record['hrn'], 'oar_job_id':[], 'node_ids': [], 'person_ids':[record['reg_researchers']['record_id']], 'geni_urn':'', #For client_helper.py compatibility 'keys':'', #For client_helper.py compatibility 'key_ids':''}) #For client_helper.py compatibility - - + + #Get slab slice record and oar job id if any. recslice_list = self.slab_api.GetSlices(slice_filter = \ str(record['hrn']),\ slice_filter_type = 'slice_hrn') - - + + logger.debug("SLABDRIVER \tfill_record_info \ TYPE SLICE RECUSER record['hrn'] %s ecord['oar_job_id']\ %s " %(record['hrn'], record['oar_job_id'])) del record['reg_researchers'] try: - for rec in recslice_list: + for rec in recslice_list: logger.debug("SLABDRIVER\r\n \t \ fill_record_info oar_job_id %s " \ %(rec['oar_job_id'])) - + record['node_ids'] = [ self.slab_api.root_auth + \ hostname for hostname in rec['node_ids']] except KeyError: pass - - + + logger.debug( "SLABDRIVER.PY \t fill_record_info SLICE \ recslice_list %s \r\n \t RECORD %s \r\n \ - \r\n" %(recslice_list, record)) - + \r\n" %(recslice_list, record)) + if str(record['type']) == 'user': #The record is a SFA user record. #Get the information about his slice from Senslab's DB @@ -135,84 +134,84 @@ class SlabDriver(Driver): recslice_list = self.slab_api.GetSlices(\ slice_filter = record['record_id'],\ slice_filter_type = 'record_id_user') - + logger.debug( "SLABDRIVER.PY \t fill_record_info TYPE USER \ recslice_list %s \r\n \t RECORD %s \r\n" \ - %(recslice_list , record)) - #Append slice record in records list, + %(recslice_list , record)) + #Append slice record in records list, #therefore fetches user and slice info again(one more loop) #Will update PIs and researcher for the slice - + recuser = recslice_list[0]['reg_researchers'] logger.debug( "SLABDRIVER.PY \t fill_record_info USER \ - recuser %s \r\n \r\n" %(recuser)) + recuser %s \r\n \r\n" %(recuser)) recslice = {} recslice = recslice_list[0] recslice.update({'PI':[recuser['hrn']], 'researcher': [recuser['hrn']], - 'name':record['hrn'], + 'name':record['hrn'], 'node_ids': [], 'oar_job_id': [], - 'person_ids':[recuser['record_id']]}) + 'person_ids':[recuser['record_id']]}) try: for rec in recslice_list: recslice['oar_job_id'].append(rec['oar_job_id']) except KeyError: pass - + recslice.update({'type':'slice', \ 'hrn':recslice_list[0]['hrn']}) - #GetPersons takes [] as filters + #GetPersons takes [] as filters user_slab = self.slab_api.GetPersons([record]) - - + + record.update(user_slab[0]) #For client_helper.py compatibility record.update( { 'geni_urn':'', 'keys':'', - 'key_ids':'' }) + 'key_ids':'' }) record_list.append(recslice) - + logger.debug("SLABDRIVER.PY \tfill_record_info ADDING SLICE\ - INFO TO USER records %s" %(record_list)) - + INFO TO USER records %s" %(record_list)) + except TypeError, error: logger.log_exc("SLABDRIVER \t fill_record_info EXCEPTION %s"\ %(error)) - + return record_list - - + + def sliver_status(self, slice_urn, slice_hrn): """ - Receive a status request for slice named urn/hrn + Receive a status request for slice named urn/hrn urn:publicid:IDN+senslab+nturro_slice hrn senslab.nturro_slice shall return a structure as described in http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus NT : not sure if we should implement this or not, but used by sface. - + :param slice_urn: slice urn :type slice_urn: string :param slice_hrn: slice hrn :type slice_hrn: string """ - - + + #First get the slice with the slice hrn slice_list = self.slab_api.GetSlices(slice_filter = slice_hrn, \ slice_filter_type = 'slice_hrn') - + if len(slice_list) is 0: raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn)) - - #Used for fetching the user info witch comes along the slice info - one_slice = slice_list[0] - + #Used for fetching the user info witch comes along the slice info + one_slice = slice_list[0] + + #Make a list of all the nodes hostnames in use for this slice slice_nodes_list = [] #for single_slice in slice_list: @@ -221,18 +220,18 @@ class SlabDriver(Driver): #for node in one_slice: #slice_nodes_list.append(node['hostname']) slice_nodes_list = one_slice['node_ids'] - #Get all the corresponding nodes details + #Get all the corresponding nodes details nodes_all = self.slab_api.GetNodes({'hostname':slice_nodes_list}, ['node_id', 'hostname','site','boot_state']) nodeall_byhostname = dict([(one_node['hostname'], one_node) \ - for one_node in nodes_all]) - - - + for one_node in nodes_all]) + + + for single_slice in slice_list: #For compatibility - top_level_status = 'empty' + top_level_status = 'empty' result = {} result.fromkeys(\ ['geni_urn','geni_error', 'pl_login','geni_status','geni_resources'], None) @@ -240,18 +239,18 @@ class SlabDriver(Driver): logger.debug("Slabdriver - sliver_status Sliver status \ urn %s hrn %s single_slice %s \r\n " \ %(slice_urn, slice_hrn, single_slice)) - + if 'node_ids' not in single_slice: #No job in the slice result['geni_status'] = top_level_status - result['geni_resources'] = [] + result['geni_resources'] = [] return result - - top_level_status = 'ready' + + top_level_status = 'ready' #A job is running on Senslab for this slice # report about the local nodes that are in the slice only - + result['geni_urn'] = slice_urn resources = [] @@ -259,7 +258,7 @@ class SlabDriver(Driver): res = {} res['slab_hostname'] = node_hostname res['slab_boot_state'] = nodeall_byhostname[node_hostname]['boot_state'] - + #res['pl_hostname'] = node['hostname'] #res['pl_boot_state'] = \ #nodeall_byhostname[node['hostname']]['boot_state'] @@ -268,55 +267,55 @@ class SlabDriver(Driver): sliver_id = Xrn(slice_urn, type='slice', \ id=nodeall_byhostname[node_hostname]['node_id'], \ authority=self.hrn).urn - - res['geni_urn'] = sliver_id + + res['geni_urn'] = sliver_id #node_name = node['hostname'] if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive': res['geni_status'] = 'ready' else: res['geni_status'] = 'failed' - top_level_status = 'failed' - + top_level_status = 'failed' + res['geni_error'] = '' - + resources.append(res) - + result['geni_status'] = top_level_status - result['geni_resources'] = resources + result['geni_resources'] = resources logger.debug("SLABDRIVER \tsliver_statusresources %s res %s "\ %(resources,res)) - return result - - @staticmethod - def get_user_record(hrn): - """ + return result + + @staticmethod + def get_user_record(hrn): + """ Returns the user record based on the hrn from the SFA DB . - + :param hrn: user's hrn :type hrn: string :return : user record from SFA database :rtype: RegUser - - """ - return dbsession.query(RegRecord).filter_by(hrn = hrn).first() - - - def testbed_name (self): - """ - Returns testbed's name. - + + """ + return dbsession.query(RegRecord).filter_by(hrn = hrn).first() + + + def testbed_name (self): + """ + Returns testbed's name. + :rtype: string """ return self.hrn - + # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory def aggregate_version (self): """ - - Returns the testbed's supported rspec advertisement and + + Returns the testbed's supported rspec advertisement and request versions. - + :rtype: dict """ version_manager = VersionManager() @@ -326,15 +325,15 @@ class SlabDriver(Driver): if rspec_version.content_type in ['*', 'ad']: ad_rspec_versions.append(rspec_version.to_dict()) if rspec_version.content_type in ['*', 'request']: - request_rspec_versions.append(rspec_version.to_dict()) + request_rspec_versions.append(rspec_version.to_dict()) return { 'testbed':self.testbed_name(), 'geni_request_rspec_versions': request_rspec_versions, 'geni_ad_rspec_versions': ad_rspec_versions, - } - - - + } + + + def _get_requested_leases_list(self, rspec): """ Process leases in rspec depending on the rspec version (format) @@ -342,8 +341,8 @@ class SlabDriver(Driver): a lease request list with the mandatory information ( nodes, start time and duration) of the valid leases (duration above or equal to the senslab experiment minimum duration). - - :param rspec: rspec request received. + + :param rspec: rspec request received. :type rspec: RSpec :return: list of lease requests found in the rspec :rtype: list @@ -351,8 +350,8 @@ class SlabDriver(Driver): requested_lease_list = [] for lease in rspec.version.get_leases(): single_requested_lease = {} - logger.debug("SLABDRIVER.PY \tcreate_sliver lease %s " %(lease)) - + logger.debug("SLABDRIVER.PY \t_get_requested_leases_list lease %s " %(lease)) + if not lease.get('lease_id'): if get_authority(lease['component_id']) == \ self.slab_api.root_auth: @@ -366,99 +365,99 @@ class SlabDriver(Driver): #the lease to the requested leases list duration_in_seconds = \ int(single_requested_lease['duration']) - if duration_in_seconds > self.slab_api.GetMinExperimentDurationInSec() : + if duration_in_seconds >= self.slab_api.GetMinExperimentDurationInSec() : requested_lease_list.append(single_requested_lease) - + return requested_lease_list - - @staticmethod + + @staticmethod def _group_leases_by_start_time(requested_lease_list): """ Create dict of leases by start_time, regrouping nodes reserved - at the same time, for the same amount of time so as to + at the same time, for the same amount of time so as to define one job on OAR. - + :param requested_lease_list: list of leases :type requested_lease_list: list :return: Dictionary with key = start time, value = list of leases - with the same start time. + with the same start time. :rtype: dictionary """ requested_job_dict = {} for lease in requested_lease_list: - + #In case it is an asap experiment start_time is empty if lease['start_time'] == '': - lease['start_time'] = '0' - + lease['start_time'] = '0' + if lease['start_time'] not in requested_job_dict: if isinstance(lease['hostname'], str): lease['hostname'] = [lease['hostname']] - + requested_job_dict[lease['start_time']] = lease - + else : job_lease = requested_job_dict[lease['start_time']] if lease['duration'] == job_lease['duration'] : - job_lease['hostname'].append(lease['hostname']) - + job_lease['hostname'].append(lease['hostname']) + return requested_job_dict - + def _process_requested_jobs(self, rspec): """ - Turns the requested leases and information into a dictionary + Turns the requested leases and information into a dictionary of requested jobs, grouped by starting time. - + :param rspec: RSpec received :type rspec : RSpec - :rtype: dictionary + :rtype: dictionary """ requested_lease_list = self._get_requested_leases_list(rspec) logger.debug("SLABDRIVER _process_requested_jobs requested_lease_list \ %s"%(requested_lease_list)) - job_dict = self._group_leases_by_start_time(requested_lease_list) + job_dict = self._group_leases_by_start_time(requested_lease_list) logger.debug("SLABDRIVER _process_requested_jobs job_dict\ %s"%(job_dict)) - + return job_dict - + def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \ users, options): - """ + """ Answer to CreateSliver. - Creates the leases and slivers for the users from the information + Creates the leases and slivers for the users from the information found in the rspec string. Launch experiment on OAR if the requested leases is valid. Delete no longer requested leases. - - :param creds: user's credentials + + :param creds: user's credentials :type creds: string :param users: user record list :type users: list :param options: :type options: - - :return: a valid Rspec for the slice which has just been + + :return: a valid Rspec for the slice which has just been modified. :rtype: RSpec - - + + """ aggregate = SlabAggregate(self) - + slices = SlabSlices(self) peer = slices.get_peer(slice_hrn) sfa_peer = slices.get_sfa_peer(slice_hrn) - slice_record = None - + slice_record = None + if not isinstance(creds, list): creds = [creds] - + if users: - slice_record = users[0].get('slice_record', {}) + slice_record = users[0].get('slice_record', {}) logger.debug("SLABDRIVER.PY \t ===============create_sliver \t\ creds %s \r\n \r\n users %s" \ %(creds, users)) @@ -470,27 +469,27 @@ class SlabDriver(Driver): logger.debug("SLABDRIVER.PY \t create_sliver \trspec.version \ %s slice_record %s users %s" \ %(rspec.version,slice_record, users)) - + # ensure site record exists? # ensure slice record exists #Removed options to verify_slice SA 14/08/12 sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \ sfa_peer) - + # ensure person records exists #verify_persons returns added persons but since the return value - #is not used + #is not used slices.verify_persons(slice_hrn, sfa_slice, users, peer, \ - sfa_peer, options=options) - #requested_attributes returned by rspec.version.get_slice_attributes() + sfa_peer, options=options) + #requested_attributes returned by rspec.version.get_slice_attributes() #unused, removed SA 13/08/12 #rspec.version.get_slice_attributes() logger.debug("SLABDRIVER.PY create_sliver slice %s " %(sfa_slice)) - # add/remove slice from nodes - + # add/remove slice from nodes + #requested_slivers = [node.get('component_id') \ #for node in rspec.version.get_nodes_with_slivers()\ #if node.get('authority_id') is self.slab_api.root_auth] @@ -499,131 +498,131 @@ class SlabDriver(Driver): #requested_slivers %s listnodes %s" \ #%(requested_slivers,l)) #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. - #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) - + #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) + requested_job_dict = self._process_requested_jobs(rspec) - - + + logger.debug("SLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\ - %(requested_job_dict)) + %(requested_job_dict)) #verify_slice_leases returns the leases , but the return value is unused - #here. Removed SA 13/08/12 + #here. Removed SA 13/08/12 slices.verify_slice_leases(sfa_slice, \ requested_job_dict, peer) - + return aggregate.get_rspec(slice_xrn=slice_urn, \ login=sfa_slice['login'], version=rspec.version) - - + + def delete_sliver (self, slice_urn, slice_hrn, creds, options): """ Deletes the lease associated with the slice hrn and the credentials if the slice belongs to senslab. Answer to DeleteSliver. - - :return: 1 if the slice to delete was not found on senslab, - True if the deletion was successful, False otherwise otherwise. - - .. note:: Should really be named delete_leases because senslab does + + :return: 1 if the slice to delete was not found on senslab, + True if the deletion was successful, False otherwise otherwise. + + .. note:: Should really be named delete_leases because senslab does not have any slivers, but only deals with leases. However, SFA api only have delete_sliver define so far. SA 13.05/2013 """ - + sfa_slice_list = self.slab_api.GetSlices(slice_filter = slice_hrn, \ slice_filter_type = 'slice_hrn') - + if not sfa_slice_list: return 1 - + #Delete all leases in the slice for sfa_slice in sfa_slice_list: - + logger.debug("SLABDRIVER.PY delete_sliver slice %s" %(sfa_slice)) slices = SlabSlices(self) # determine if this is a peer slice - - peer = slices.get_peer(slice_hrn) + + peer = slices.get_peer(slice_hrn) logger.debug("SLABDRIVER.PY delete_sliver peer %s \ \r\n \t sfa_slice %s " %(peer, sfa_slice)) try: - + self.slab_api.DeleteSliceFromNodes(sfa_slice) return True except : return False - - + + def list_resources (self, slice_urn, slice_hrn, creds, options): """ - List resources from the senslab aggregate and returns a Rspec - advertisement with resources found when slice_urn and slice_hrn are None + List resources from the senslab aggregate and returns a Rspec + advertisement with resources found when slice_urn and slice_hrn are None (in case of resource discovery). If a slice hrn and urn are provided, list experiment's slice nodes in a rspec format. Answer to ListResources. - Caching unused. + Caching unused. :param options: options used when listing resources (list_leases, info, - geni_available) + geni_available) :return: rspec string in xml - :rtype: string + :rtype: string """ - - #cached_requested = options.get('cached', True) - + + #cached_requested = options.get('cached', True) + version_manager = VersionManager() # get the rspec's return format from options rspec_version = \ version_manager.get_version(options.get('geni_rspec_version')) version_string = "rspec_%s" % (rspec_version) - + #panos adding the info option to the caching key (can be improved) if options.get('info'): version_string = version_string + "_" + \ options.get('info', 'default') - + # Adding the list_leases option to the caching key if options.get('list_leases'): version_string = version_string + "_" + \ options.get('list_leases', 'default') - + # Adding geni_available to caching key if options.get('geni_available'): version_string = version_string + "_" + \ str(options.get('geni_available')) - + # look in cache first #if cached_requested and self.cache and not slice_hrn: #rspec = self.cache.get(version_string) #if rspec: #logger.debug("SlabDriver.ListResources: \ #returning cached advertisement") - #return rspec - + #return rspec + #panos: passing user-defined options aggregate = SlabAggregate(self) - + rspec = aggregate.get_rspec(slice_xrn=slice_urn, \ version=rspec_version, options=options) - + # cache the result #if self.cache and not slice_hrn: #logger.debug("Slab.ListResources: stores advertisement in cache") #self.cache.add(version_string, rspec) - + return rspec - - + + def list_slices (self, creds, options): """ Answer to ListSlices. - List slices belonging to senslab, returns slice urns list. + List slices belonging to senslab, returns slice urns list. No caching used. Options unused but are defined in the SFA method - api prototype. - + api prototype. + :return: slice urns list :rtype: list - + """ # look in cache first #if self.cache: @@ -631,11 +630,11 @@ class SlabDriver(Driver): #if slices: #logger.debug("PlDriver.list_slices returns from cache") #return slices - - # get data from db - slices = self.slab_api.GetSlices() - logger.debug("SLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices)) + # get data from db + + slices = self.slab_api.GetSlices() + logger.debug("SLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices)) slice_hrns = [slab_slice['hrn'] for slab_slice in slices] slice_urns = [hrn_to_urn(slice_hrn, 'slice') \ @@ -644,62 +643,62 @@ class SlabDriver(Driver): # cache the result #if self.cache: #logger.debug ("SlabDriver.list_slices stores value in cache") - #self.cache.add('slices', slice_urns) - + #self.cache.add('slices', slice_urns) + return slice_urns - - + + def register (self, sfa_record, hrn, pub_key): - """ + """ Adding new user, slice, node or site should not be handled by SFA. - - ..warnings:: should not be used. Different components are in charge of + + ..warnings:: should not be used. Different components are in charge of doing this task. Adding nodes = OAR Adding users = LDAP Senslab Adding slice = Import from LDAP users Adding site = OAR - - :param sfa_record: record provided by the client of the - Register API call. + + :param sfa_record: record provided by the client of the + Register API call. :type sfa_record: dict """ return -1 - - + + def update (self, old_sfa_record, new_sfa_record, hrn, new_key): """No site or node record update allowed in Senslab. - The only modifications authorized here are key deletion/addition + The only modifications authorized here are key deletion/addition on an existing user and password change. On an existing user, CAN NOT BE MODIFIED: 'first_name', 'last_name', 'email' DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio','title', 'accepted_aup', A slice is bound to its user, so modifying the user's ssh key should - modify the slice's GID after an import procedure. - + modify the slice's GID after an import procedure. + :param old_sfa_record: what is in the db for this hrn :param new_sfa_record: what was passed to the Update call - - ..seealso:: update in driver.py. + + ..seealso:: update in driver.py. """ - + pointer = old_sfa_record['pointer'] old_sfa_record_type = old_sfa_record['type'] # new_key implemented for users only if new_key and old_sfa_record_type not in [ 'user' ]: raise UnknownSfaType(old_sfa_record_type) - - + + if old_sfa_record_type == "user": update_fields = {} all_fields = new_sfa_record for key in all_fields.keys(): if key in ['key', 'password']: update_fields[key] = all_fields[key] - - + + if new_key: # must check this key against the previous one if it exists persons = self.slab_api.GetPersons([old_sfa_record]) @@ -707,11 +706,11 @@ class SlabDriver(Driver): keys = [person['pkey']] #Get all the person's keys keys_dict = self.slab_api.GetKeys(keys) - + # Delete all stale keys, meaning the user has only one key #at a time #TODO: do we really want to delete all the other keys? - #Is this a problem with the GID generation to have multiple + #Is this a problem with the GID generation to have multiple #keys? SA 30/05/13 key_exists = False if key in keys_dict: @@ -725,31 +724,31 @@ class SlabDriver(Driver): #self.slab_api.AddPersonKey(person, {'key_type': 'ssh', \ #'key': new_key}) return True - + def remove (self, sfa_record): """ Removes users only. Mark the user as disabled in - LDAP. The user and his slice are then deleted from the db by running an + LDAP. The user and his slice are then deleted from the db by running an import on the registry. - - - - :param sfa_record: record is the existing sfa record in the db + + + + :param sfa_record: record is the existing sfa record in the db :type sfa_record: dict - + ..warning::As fas as the slice is concerned, here only the leases are - removed from the slice. The slice is record itself is not removed from - the db. - TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, - - TODO: return boolean for the slice part + removed from the slice. The slice is record itself is not removed from + the db. + TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, + + TODO: return boolean for the slice part """ sfa_record_type = sfa_record['type'] hrn = sfa_record['hrn'] if sfa_record_type == 'user': - #get user from senslab ldap + #get user from senslab ldap person = self.slab_api.GetPersons(sfa_record) #No registering at a given site in Senslab. #Once registered to the LDAP, all senslab sites are @@ -766,5 +765,5 @@ class SlabDriver(Driver): return True - - + +