- """
- logger.debug("IOTLAB_API \tDeleteJobs jobid %s username %s "
- % (job_id, username))
- if not job_id or job_id is -1:
- return
-
- reqdict = {}
- reqdict['method'] = "delete"
- reqdict['strval'] = str(job_id)
-
- answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id',
- reqdict, username)
- if answer['status'] == 'Delete request registered':
- ret = {job_id: True}
- else:
- ret = {job_id: False}
- logger.debug("IOTLAB_API \tDeleteJobs jobid %s \r\n answer %s \
- username %s" % (job_id, answer, username))
- return ret
-
-
-
- ##TODO : Unused GetJobsId ? SA 05/07/12
- #def GetJobsId(self, job_id, username = None ):
- #"""
- #Details about a specific job.
- #Includes details about submission time, jot type, state, events,
- #owner, assigned ressources, walltime etc...
-
- #"""
- #req = "GET_jobs_id"
- #node_list_k = 'assigned_network_address'
- ##Get job info from OAR
- #job_info = self.oar.parser.SendRequest(req, job_id, username)
-
- #logger.debug("IOTLAB_API \t GetJobsId %s " %(job_info))
- #try:
- #if job_info['state'] == 'Terminated':
- #logger.debug("IOTLAB_API \t GetJobsId job %s TERMINATED"\
- #%(job_id))
- #return None
- #if job_info['state'] == 'Error':
- #logger.debug("IOTLAB_API \t GetJobsId ERROR message %s "\
- #%(job_info))
- #return None
-
- #except KeyError:
- #logger.error("IOTLAB_API \tGetJobsId KeyError")
- #return None
-
- #parsed_job_info = self.get_info_on_reserved_nodes(job_info, \
- #node_list_k)
- ##Replaces the previous entry
- ##"assigned_network_address" / "reserved_resources"
- ##with "node_ids"
- #job_info.update({'node_ids':parsed_job_info[node_list_k]})
- #del job_info[node_list_k]
- #logger.debug(" \r\nIOTLAB_API \t GetJobsId job_info %s " %(job_info))
- #return job_info
-
-
- def GetJobsResources(self, job_id, username = None):
- """ Gets the list of nodes associated with the job_id and username
- if provided.
- Transforms the iotlab hostnames to the corresponding
- SFA nodes hrns.
- Rertuns dict key :'node_ids' , value : hostnames list
- :param username: user's LDAP login
- :paran job_id: job's OAR identifier.
- :type username: string
- :type job_id: integer
-
- :returns: dicionary with nodes' hostnames belonging to the job.
- :rtype: dict
- .. warning: Unused. SA 16/10/13
- """
-
- req = "GET_jobs_id_resources"
-
-
- #Get job resources list from OAR
- node_id_list = self.oar.parser.SendRequest(req, job_id, username)
- logger.debug("IOTLAB_API \t GetJobsResources %s " %(node_id_list))
-
- hostname_list = \
- self.__get_hostnames_from_oar_node_ids(node_id_list)
-
-
- #Replaces the previous entry "assigned_network_address" /
- #"reserved_resources" with "node_ids"
- job_info = {'node_ids': hostname_list}
-
- return job_info
-
-
- #def get_info_on_reserved_nodes(self, job_info, node_list_name):
- #"""
- #..warning:unused SA 23/05/13
- #"""
- ##Get the list of the testbed nodes records and make a
- ##dictionnary keyed on the hostname out of it
- #node_list_dict = self.GetNodes()
- ##node_hostname_list = []
- #node_hostname_list = [node['hostname'] for node in node_list_dict]
- ##for node in node_list_dict:
- ##node_hostname_list.append(node['hostname'])
- #node_dict = dict(zip(node_hostname_list, node_list_dict))
- #try :
- #reserved_node_hostname_list = []
- #for index in range(len(job_info[node_list_name])):
- ##job_info[node_list_name][k] =
- #reserved_node_hostname_list[index] = \
- #node_dict[job_info[node_list_name][index]]['hostname']
-
- #logger.debug("IOTLAB_API \t get_info_on_reserved_nodes \
- #reserved_node_hostname_list %s" \
- #%(reserved_node_hostname_list))
- #except KeyError:
- #logger.error("IOTLAB_API \t get_info_on_reserved_nodes KEYERROR " )
-
- #return reserved_node_hostname_list
-
- def GetNodesCurrentlyInUse(self):
- """Returns a list of all the nodes already involved in an oar running
- job.
- :rtype: list of nodes hostnames.
- """
- return self.oar.parser.SendRequest("GET_running_jobs")
-
- def __get_hostnames_from_oar_node_ids(self, oar_id_node_dict,
- resource_id_list ):
- """Get the hostnames of the nodes from their OAR identifiers.
- Get the list of nodes dict using GetNodes and find the hostname
- associated with the identifier.
- :param oar_id_node_dict: full node dictionary list keyed by oar node id
- :param resource_id_list: list of nodes identifiers
- :returns: list of node hostnames.
- """
-
- hostname_list = []
- for resource_id in resource_id_list:
- #Because jobs requested "asap" do not have defined resources
- if resource_id is not "Undefined":
- hostname_list.append(\
- oar_id_node_dict[resource_id]['hostname'])
-
- #hostname_list.append(oar_id_node_dict[resource_id]['hostname'])
- return hostname_list
-
- def GetReservedNodes(self, username=None):
- """ Get list of leases. Get the leases for the username if specified,
- otherwise get all the leases. Finds the nodes hostnames for each
- OAR node identifier.
- :param username: user's LDAP login
- :type username: string
- :returns: list of reservations dict
- :rtype: dict list
- """
-
- #Get the nodes in use and the reserved nodes
- reservation_dict_list = \
- self.oar.parser.SendRequest("GET_reserved_nodes", \
- username = username)
-
- # Get the full node dict list once for all
- # so that we can get the hostnames given their oar node id afterwards
- # when the reservations are checked.
- full_nodes_dict_list = self.GetNodes()
- #Put the full node list into a dictionary keyed by oar node id
- oar_id_node_dict = {}
- for node in full_nodes_dict_list:
- oar_id_node_dict[node['oar_id']] = node
-
- for resa in reservation_dict_list:
- logger.debug ("GetReservedNodes resa %s"%(resa))
- #dict list of hostnames and their site
- resa['reserved_nodes'] = \
- self.__get_hostnames_from_oar_node_ids(oar_id_node_dict,
- resa['resource_ids'])
-
- #del resa['resource_ids']
- return reservation_dict_list
-
- def GetNodes(self, node_filter_dict=None, return_fields_list=None):
- """
-
- Make a list of iotlab nodes and their properties from information
- given by OAR. Search for specific nodes if some filters are
- specified. Nodes properties returned if no return_fields_list given:
- 'hrn','archi','mobile','hostname','site','boot_state','node_id',
- 'radio','posx','posy','oar_id','posz'.
-
- :param node_filter_dict: dictionnary of lists with node properties. For
- instance, if you want to look for a specific node with its hrn,
- the node_filter_dict should be {'hrn': [hrn_of_the_node]}
- :type node_filter_dict: dict
- :param return_fields_list: list of specific fields the user wants to be
- returned.
- :type return_fields_list: list
- :returns: list of dictionaries with node properties
- :rtype: list
-
- """
- node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
- node_dict_list = node_dict_by_id.values()
- logger.debug (" IOTLAB_API GetNodes node_filter_dict %s \
- return_fields_list %s " % (node_filter_dict, return_fields_list))
- #No filtering needed return the list directly
- if not (node_filter_dict or return_fields_list):
- return node_dict_list
-
- return_node_list = []
- if node_filter_dict:
- for filter_key in node_filter_dict:
- try:
- #Filter the node_dict_list by each value contained in the
- #list node_filter_dict[filter_key]
- for value in node_filter_dict[filter_key]:
- for node in node_dict_list:
- if node[filter_key] == value:
- if return_fields_list:
- tmp = {}
- for k in return_fields_list:
- tmp[k] = node[k]
- return_node_list.append(tmp)
- else:
- return_node_list.append(node)
- except KeyError:
- logger.log_exc("GetNodes KeyError")
- return
-
-
- return return_node_list
-
-
- def AddSlice(self, slice_record, user_record):
- """
-
- Add slice to the local iotlab sfa tables if the slice comes
- from a federated site and is not yet in the iotlab sfa DB,
- although the user has already a LDAP login.
- Called by verify_slice during lease/sliver creation.
-
- :param slice_record: record of slice, must contain hrn, gid, slice_id
- and authority of the slice.
- :type slice_record: dictionary
- :param user_record: record of the user
- :type user_record: RegUser
-
- """
-
- sfa_record = RegSlice(hrn=slice_record['hrn'],
- gid=slice_record['gid'],
- pointer=slice_record['slice_id'],
- authority=slice_record['authority'])
- logger.debug("IOTLAB_API.PY AddSlice sfa_record %s user_record %s"
- % (sfa_record, user_record))
- sfa_record.just_created()
- self.api.dbsession().add(sfa_record)
- self.api.dbsession().commit()
- #Update the reg-researcher dependance table
- sfa_record.reg_researchers = [user_record]
- self.api.dbsession().commit()
-
- return
-
-
- def GetSites(self, site_filter_name_list=None, return_fields_list=None):
- """Returns the list of Iotlab's sites with the associated nodes and
- the sites' properties as dictionaries.
-
- Site properties:
- ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids',
- 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids',
- 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created',
- 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name']
- Uses the OAR request GET_sites to find the Iotlab's sites.
-
- :param site_filter_name_list: used to specify specific sites
- :param return_fields_list: field that has to be returned
- :type site_filter_name_list: list
- :type return_fields_list: list
-
-
- """
- site_dict = self.oar.parser.SendRequest("GET_sites")
- #site_dict : dict where the key is the sit ename
- return_site_list = []
- if not (site_filter_name_list or return_fields_list):
- return_site_list = site_dict.values()
- return return_site_list
-
- for site_filter_name in site_filter_name_list:
- if site_filter_name in site_dict:
- if return_fields_list:
- for field in return_fields_list:
- tmp = {}
- try:
- tmp[field] = site_dict[site_filter_name][field]
- except KeyError:
- logger.error("GetSites KeyError %s " % (field))
- return None
- return_site_list.append(tmp)
- else:
- return_site_list.append(site_dict[site_filter_name])
-
- return return_site_list
-
-
- #TODO : Check rights to delete person
- def DeletePerson(self, person_record):
- """Disable an existing account in iotlab LDAP.
-
- Users and techs can only delete themselves. PIs can only
- delete themselves and other non-PIs at their sites.
- ins can delete anyone.
-
- :param person_record: user's record
- :type person_record: dict
- :returns: True if successful, False otherwise.
- :rtype: boolean
-
- .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF.
- """
- #Disable user account in iotlab LDAP
- ret = self.ldap.LdapMarkUserAsDeleted(person_record)
- logger.warning("IOTLAB_API DeletePerson %s " % (person_record))
- return ret['bool']
-
- def DeleteSlice(self, slice_record):
- """Deletes the specified slice and kills the jobs associated with
- the slice if any, using DeleteSliceFromNodes.
-
- :param slice_record: record of the slice, must contain oar_job_id, user
- :type slice_record: dict
- :returns: True if all the jobs in the slice have been deleted,
- or the list of jobs that could not be deleted otherwise.
- :rtype: list or boolean
-
- .. seealso:: DeleteSliceFromNodes
-
- """
- ret = self.DeleteSliceFromNodes(slice_record)
- delete_failed = None
- for job_id in ret:
- if False in ret[job_id]:
- if delete_failed is None:
- delete_failed = []
- delete_failed.append(job_id)
-
- logger.info("IOTLAB_API DeleteSlice %s answer %s"%(slice_record, \
- delete_failed))
- return delete_failed or True
-
-
- def __add_person_to_db(self, user_dict):
- """
- Add a federated user straight to db when the user issues a lease
- request with iotlab nodes and that he has not registered with iotlab
- yet (that is he does not have a LDAP entry yet).
- Uses parts of the routines in IotlabImport when importing user from LDAP.
- Called by AddPerson, right after LdapAddUser.
- :param user_dict: Must contain email, hrn and pkey to get a GID
- and be added to the SFA db.
- :type user_dict: dict
-
- """
- check_if_exists = \
- self.api.dbsession().query(RegUser).filter_by(email = user_dict['email']).first()
- #user doesn't exists
- if not check_if_exists:
- logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
- " %(user_dict))
- hrn = user_dict['hrn']
- person_urn = hrn_to_urn(hrn, 'user')
- pubkey = user_dict['pkey']
- try:
- pkey = convert_public_key(pubkey)
- except TypeError:
- #key not good. create another pkey
- logger.warn('__add_person_to_db: unable to convert public \
- key for %s' %(hrn ))
- pkey = Keypair(create=True)
-
-
- if pubkey is not None and pkey is not None :
- hierarchy = Hierarchy()
- person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
- pkey)
- if user_dict['email']:
- logger.debug("__add_person_to_db \r\n \r\n \
- IOTLAB IMPORTER PERSON EMAIL OK email %s "\
- %(user_dict['email']))
- person_gid.set_email(user_dict['email'])
-
- user_record = RegUser(hrn=hrn , pointer= '-1', \
- authority=get_authority(hrn), \
- email=user_dict['email'], gid = person_gid)
- user_record.reg_keys = [RegKey(user_dict['pkey'])]
- user_record.just_created()
- self.api.dbsession().add (user_record)
- self.api.dbsession().commit()
- return
-
-
- def AddPerson(self, record):
- """
-
- Adds a new account. Any fields specified in records are used,
- otherwise defaults are used. Creates an appropriate login by calling
- LdapAddUser.
-
- :param record: dictionary with the sfa user's properties.
- :returns: a dicitonary with the status. If successful, the dictionary
- boolean is set to True and there is a 'uid' key with the new login
- added to LDAP, otherwise the bool is set to False and a key
- 'message' is in the dictionary, with the error message.
- :rtype: dict
-
- """
- ret = self.ldap.LdapAddUser(record)
-
- if ret['bool'] is True:
- record['hrn'] = self.root_auth + '.' + ret['uid']
- logger.debug("IOTLAB_API AddPerson return code %s record %s "
- % (ret, record))
- self.__add_person_to_db(record)
- return ret
-
-
-
-
-
- #TODO AddPersonKey 04/07/2012 SA
- def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
- """Adds a new key to the specified account. Adds the key to the
- iotlab ldap, provided that the person_uid is valid.
-
- Non-admins can only modify their own keys.
-
- :param person_uid: user's iotlab login in LDAP
- :param old_attributes_dict: dict with the user's old sshPublicKey
- :param new_key_dict: dict with the user's new sshPublicKey
- :type person_uid: string
-
-
- :rtype: Boolean
- :returns: True if the key has been modified, False otherwise.
-
- """
- ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \
- new_key_dict)
- logger.warning("IOTLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ")
- return ret['bool']
-
- def DeleteLeases(self, leases_id_list, slice_hrn):
- """
-
- Deletes several leases, based on their job ids and the slice
- they are associated with. Uses DeleteJobs to delete the jobs
- on OAR. Note that one slice can contain multiple jobs, and in this
- case all the jobs in the leases_id_list MUST belong to ONE slice,
- since there is only one slice hrn provided here.
-
- :param leases_id_list: list of job ids that belong to the slice whose
- slice hrn is provided.
- :param slice_hrn: the slice hrn.
- :type slice_hrn: string
-
- .. warning:: Does not have a return value since there was no easy
- way to handle failure when dealing with multiple job delete. Plus,
- there was no easy way to report it to the user.
-
- """
- logger.debug("IOTLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
- \r\n " %(leases_id_list, slice_hrn))
- for job_id in leases_id_list:
- self.DeleteJobs(job_id, slice_hrn)
-
- return
-
- @staticmethod
- def _process_walltime(duration):
- """ Calculates the walltime in seconds from the duration in H:M:S
- specified in the RSpec.
-
- """
- if duration:
- # Fixing the walltime by adding a few delays.
- # First put the walltime in seconds oarAdditionalDelay = 20;
- # additional delay for /bin/sleep command to
- # take in account prologue and epilogue scripts execution
- # int walltimeAdditionalDelay = 240; additional delay
- #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
- #in oar.conf
- # Put the duration in seconds first
- #desired_walltime = duration * 60
- desired_walltime = duration
- total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
- sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
- walltime = []
- #Put the walltime back in str form
- #First get the hours
- walltime.append(str(total_walltime / 3600))
- total_walltime = total_walltime - 3600 * int(walltime[0])
- #Get the remaining minutes
- walltime.append(str(total_walltime / 60))
- total_walltime = total_walltime - 60 * int(walltime[1])
- #Get the seconds
- walltime.append(str(total_walltime))
-
- else:
- logger.log_exc(" __process_walltime duration null")
-
- return walltime, sleep_walltime
-
- @staticmethod
- def _create_job_structure_request_for_OAR(lease_dict):
- """ Creates the structure needed for a correct POST on OAR.
- Makes the timestamp transformation into the appropriate format.
- Sends the POST request to create the job with the resources in
- added_nodes.
-
- """
-
- nodeid_list = []
- reqdict = {}
-
-
- reqdict['workdir'] = '/tmp'
- reqdict['resource'] = "{network_address in ("
-
- for node in lease_dict['added_nodes']:
- logger.debug("\r\n \r\n OARrestapi \t \
- __create_job_structure_request_for_OAR node %s" %(node))
-
- # Get the ID of the node
- nodeid = node
- reqdict['resource'] += "'" + nodeid + "', "
- nodeid_list.append(nodeid)
-
- custom_length = len(reqdict['resource'])- 2
- reqdict['resource'] = reqdict['resource'][0:custom_length] + \
- ")}/nodes=" + str(len(nodeid_list))
-
-
- walltime, sleep_walltime = \
- IotlabShell._process_walltime(\
- int(lease_dict['lease_duration']))
-
-
- reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
- ":" + str(walltime[1]) + ":" + str(walltime[2])
- reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
-
- #In case of a scheduled experiment (not immediate)
- #To run an XP immediately, don't specify date and time in RSpec
- #They will be set to None.
- if lease_dict['lease_start_time'] is not '0':
- #Readable time accepted by OAR
- start_time = datetime.fromtimestamp( \
- int(lease_dict['lease_start_time'])).\
- strftime(lease_dict['time_format'])
- reqdict['reservation'] = start_time
- #If there is not start time, Immediate XP. No need to add special
- # OAR parameters
-
-
- reqdict['type'] = "deploy"
- reqdict['directory'] = ""
- reqdict['name'] = "SFA_" + lease_dict['slice_user']
-
- return reqdict
-
-
- def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
- lease_start_time, lease_duration, slice_user=None):
-
- """
- Create a job request structure based on the information provided
- and post the job on OAR.
- :param added_nodes: list of nodes that belong to the described lease.
- :param slice_name: the slice hrn associated to the lease.
- :param lease_start_time: timestamp of the lease startting time.
- :param lease_duration: lease durationin minutes
-
- """
- lease_dict = {}
- lease_dict['lease_start_time'] = lease_start_time
- lease_dict['lease_duration'] = lease_duration
- lease_dict['added_nodes'] = added_nodes
- lease_dict['slice_name'] = slice_name
- lease_dict['slice_user'] = slice_user
- lease_dict['grain'] = self.GetLeaseGranularity()
- lease_dict['time_format'] = self.time_format
-
-
- logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\
- \r\n " %(slice_user))
- #Create the request for OAR
- reqdict = self._create_job_structure_request_for_OAR(lease_dict)
- # first step : start the OAR job and update the job
- logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\
- \r\n " %(reqdict))
-
- answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
- reqdict, slice_user)
- logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s " %(answer))