2 File containing the IotlabTestbedAPI, used to interact with nodes, users,
3 slices, leases and keys, as well as the dedicated iotlab database and table,
4 holding information about which slice is running which job.
7 from datetime import datetime
9 from sfa.util.sfalogging import logger
11 from sfa.storage.alchemy import dbsession
12 from sqlalchemy.orm import joinedload
13 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
14 from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP
15 from sfa.iotlab.OARrestapi import OARrestapi
16 from sfa.iotlab.LDAPapi import LDAPapi
18 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
20 from sfa.trust.certificate import Keypair, convert_public_key
21 from sfa.trust.gid import create_uuid
22 from sfa.trust.hierarchy import Hierarchy
24 from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
26 class CortexlabTestbedAPI():
27 """ Class enabled to use LDAP and OAR api calls. """
29 _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins
31 def __init__(self, config):
32 """Creates an instance of OARrestapi and LDAPapi which will be used to
33 issue calls to OAR or LDAP methods.
34 Set the time format and the testbed granularity used for OAR
35 reservation and leases.
37 :param config: configuration object from sfa.util.config
38 :type config: Config object
40 self.iotlab_db = IotlabDB(config)
41 self.oar = OARrestapi()
43 self.time_format = "%Y-%m-%d %H:%M:%S"
44 self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
45 self.grain = 60 # 10 mins lease minimum, 60 sec granularity
46 #import logging, logging.handlers
47 #from sfa.util.sfalogging import _SfaLogger
48 #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \
53 def GetMinExperimentDurationInGranularity():
54 """ Returns the minimum allowed duration for an experiment on the
58 return IotlabTestbedAPI._MINIMUM_DURATION
61 def GetPeers(peer_filter=None ):
62 """ Gathers registered authorities in SFA DB and looks for specific peer
63 if peer_filter is specified.
64 :param peer_filter: name of the site authority looked for.
65 :type peer_filter: string
66 :returns: list of records.
71 existing_hrns_by_types = {}
72 logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter))
73 all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
75 for record in all_records:
76 existing_records[(record.hrn, record.type)] = record
77 if record.type not in existing_hrns_by_types:
78 existing_hrns_by_types[record.type] = [record.hrn]
80 existing_hrns_by_types[record.type].append(record.hrn)
82 logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s "
83 % (existing_hrns_by_types))
88 records_list.append(existing_records[(peer_filter,
91 for hrn in existing_hrns_by_types['authority']:
92 records_list.append(existing_records[(hrn, 'authority')])
94 logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s "
100 return_records = records_list
101 logger.debug("CORTEXLAB_API \tGetPeer return_records %s "
103 return return_records
105 #TODO : Handling OR request in make_ldap_filters_from_records
106 #instead of the for loop
107 #over the records' list
108 def GetPersons(self, person_filter=None):
110 Get the enabled users and their properties from Iotlab LDAP.
111 If a filter is specified, looks for the user whose properties match
112 the filter, otherwise returns the whole enabled users'list.
114 :param person_filter: Must be a list of dictionnaries with users
115 properties when not set to None.
116 :type person_filter: list of dict
118 :returns: Returns a list of users whose accounts are enabled
120 :rtype: list of dicts
123 logger.debug("CORTEXLAB_API \tGetPersons person_filter %s"
126 if person_filter and isinstance(person_filter, list):
127 #If we are looking for a list of users (list of dict records)
128 #Usually the list contains only one user record
129 for searched_attributes in person_filter:
131 #Get only enabled user accounts in iotlab LDAP :
132 #add a filter for make_ldap_filters_from_record
133 person = self.ldap.LdapFindUser(searched_attributes,
134 is_user_enabled=True)
135 #If a person was found, append it to the list
137 person_list.append(person)
139 #If the list is empty, return None
140 if len(person_list) is 0:
144 #Get only enabled user accounts in iotlab LDAP :
145 #add a filter for make_ldap_filters_from_record
146 person_list = self.ldap.LdapFindUser(is_user_enabled=True)
151 #def GetTimezone(self):
152 #""" Returns the OAR server time and timezone.
153 #Unused SA 30/05/13"""
154 #server_timestamp, server_tz = self.oar.parser.\
155 #SendRequest("GET_timezone")
156 #return server_timestamp, server_tz
158 def DeleteLeases(self, lease_id, username):
161 Deletes the lease with the specified lease_id and username on OAR by
162 posting a delete request to OAR.
164 :param lease_id: Reservation identifier.
165 :param username: user's iotlab login in LDAP.
166 :type lease_id: Depends on what tou are using, could be integer or
168 :type username: string
170 :returns: dictionary with the lease id and if delete has been successful
176 # Here delete the lease specified
178 # If the username is not necessary to delete the lease, then you can
179 # remove it from the parameters, given that you propagate the changes
182 # Return delete status so that you know if the delete has been
184 if answer['status'] == 'Delete request registered':
185 ret = {lease_id: True}
187 ret = {lease_id: False}
188 logger.debug("CORTEXLAB_API \DeleteLeases lease_id %s \r\n answer %s \
189 username %s" % (lease_id, answer, username))
197 def GetJobsResources(self, job_id, username = None):
198 """ Gets the list of nodes associated with the job_id and username
200 Transforms the iotlab hostnames to the corresponding
202 Rertuns dict key :'node_ids' , value : hostnames list
203 :param username: user's LDAP login
204 :paran job_id: job's OAR identifier.
205 :type username: string
206 :type job_id: integer
208 :returns: dicionary with nodes' hostnames belonging to the job.
212 req = "GET_jobs_id_resources"
215 #Get job resources list from OAR
216 node_id_list = self.oar.parser.SendRequest(req, job_id, username)
217 logger.debug("CORTEXLAB_API \t GetJobsResources %s " %(node_id_list))
220 self.__get_hostnames_from_oar_node_ids(node_id_list)
223 #Replaces the previous entry "assigned_network_address" /
224 #"reserved_resources" with "node_ids"
225 job_info = {'node_ids': hostname_list}
231 def GetNodesCurrentlyInUse(self):
232 """Returns a list of all the nodes already involved in an oar running
234 :rtype: list of nodes hostnames.
236 return self.oar.parser.SendRequest("GET_running_jobs")
238 def __get_hostnames_from_oar_node_ids(self, resource_id_list ):
239 """Get the hostnames of the nodes from their OAR identifiers.
240 Get the list of nodes dict using GetNodes and find the hostname
241 associated with the identifier.
242 :param resource_id_list: list of nodes identifiers
243 :returns: list of node hostnames.
245 full_nodes_dict_list = self.GetNodes()
246 #Put the full node list into a dictionary keyed by oar node id
247 oar_id_node_dict = {}
248 for node in full_nodes_dict_list:
249 oar_id_node_dict[node['oar_id']] = node
252 for resource_id in resource_id_list:
253 #Because jobs requested "asap" do not have defined resources
254 if resource_id is not "Undefined":
255 hostname_list.append(\
256 oar_id_node_dict[resource_id]['hostname'])
258 #hostname_list.append(oar_id_node_dict[resource_id]['hostname'])
261 def GetReservedNodes(self, username=None):
262 """ Get list of leases. Get the leases for the username if specified,
263 otherwise get all the leases. Finds the nodes hostnames for each
265 :param username: user's LDAP login
266 :type username: string
267 :returns: list of reservations dict
271 #Get the nodes in use and the reserved nodes
272 reservation_dict_list = \
273 self.oar.parser.SendRequest("GET_reserved_nodes", \
277 for resa in reservation_dict_list:
278 logger.debug ("GetReservedNodes resa %s"%(resa))
279 #dict list of hostnames and their site
280 resa['reserved_nodes'] = \
281 self.__get_hostnames_from_oar_node_ids(resa['resource_ids'])
283 #del resa['resource_ids']
284 return reservation_dict_list
286 def GetNodes(self, node_filter_dict=None, return_fields_list=None):
289 Make a list of iotlab nodes and their properties from information
290 given by OAR. Search for specific nodes if some filters are
291 specified. Nodes properties returned if no return_fields_list given:
292 'hrn','archi','mobile','hostname','site','boot_state','node_id',
293 'radio','posx','posy','oar_id','posz'.
295 :param node_filter_dict: dictionnary of lists with node properties. For
296 instance, if you want to look for a specific node with its hrn,
297 the node_filter_dict should be {'hrn': [hrn_of_the_node]}
298 :type node_filter_dict: dict
299 :param return_fields_list: list of specific fields the user wants to be
301 :type return_fields_list: list
302 :returns: list of dictionaries with node properties
306 node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
307 node_dict_list = node_dict_by_id.values()
308 logger.debug (" CORTEXLAB_API GetNodes node_filter_dict %s \
309 return_fields_list %s " % (node_filter_dict, return_fields_list))
310 #No filtering needed return the list directly
311 if not (node_filter_dict or return_fields_list):
312 return node_dict_list
314 return_node_list = []
316 for filter_key in node_filter_dict:
318 #Filter the node_dict_list by each value contained in the
319 #list node_filter_dict[filter_key]
320 for value in node_filter_dict[filter_key]:
321 for node in node_dict_list:
322 if node[filter_key] == value:
323 if return_fields_list:
325 for k in return_fields_list:
327 return_node_list.append(tmp)
329 return_node_list.append(node)
331 logger.log_exc("GetNodes KeyError")
335 return return_node_list
340 def AddSlice(slice_record, user_record):
343 Add slice to the local iotlab sfa tables if the slice comes
344 from a federated site and is not yet in the iotlab sfa DB,
345 although the user has already a LDAP login.
346 Called by verify_slice during lease/sliver creation.
348 :param slice_record: record of slice, must contain hrn, gid, slice_id
349 and authority of the slice.
350 :type slice_record: dictionary
351 :param user_record: record of the user
352 :type user_record: RegUser
356 sfa_record = RegSlice(hrn=slice_record['hrn'],
357 gid=slice_record['gid'],
358 pointer=slice_record['slice_id'],
359 authority=slice_record['authority'])
360 logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s"
361 % (sfa_record, user_record))
362 sfa_record.just_created()
363 dbsession.add(sfa_record)
365 #Update the reg-researcher dependance table
366 sfa_record.reg_researchers = [user_record]
372 def GetSites(self, site_filter_name_list=None, return_fields_list=None):
373 """Returns the list of Iotlab's sites with the associated nodes and
374 their properties as dictionaries.
376 Uses the OAR request GET_sites to find the Iotlab's sites.
378 :param site_filter_name_list: used to specify specific sites
379 :param return_fields_list: field that has to be returned
380 :type site_filter_name_list: list
381 :type return_fields_list: list
385 site_dict = self.oar.parser.SendRequest("GET_sites")
386 #site_dict : dict where the key is the sit ename
387 return_site_list = []
388 if not (site_filter_name_list or return_fields_list):
389 return_site_list = site_dict.values()
390 return return_site_list
392 for site_filter_name in site_filter_name_list:
393 if site_filter_name in site_dict:
394 if return_fields_list:
395 for field in return_fields_list:
398 tmp[field] = site_dict[site_filter_name][field]
400 logger.error("GetSites KeyError %s " % (field))
402 return_site_list.append(tmp)
404 return_site_list.append(site_dict[site_filter_name])
406 return return_site_list
409 #TODO : Check rights to delete person
410 def DeletePerson(self, person_record):
411 """Disable an existing account in iotlab LDAP.
413 Users and techs can only delete themselves. PIs can only
414 delete themselves and other non-PIs at their sites.
415 ins can delete anyone.
417 :param person_record: user's record
418 :type person_record: dict
419 :returns: True if successful, False otherwise.
422 .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF.
424 #Disable user account in iotlab LDAP
425 ret = self.ldap.LdapMarkUserAsDeleted(person_record)
426 logger.warning("CORTEXLAB_API DeletePerson %s " % (person_record))
429 def DeleteSlice(self, slice_record):
430 """Deletes the specified slice and kills the jobs associated with
431 the slice if any, using DeleteSliceFromNodes.
433 :param slice_record: record of the slice, must contain oar_job_id, user
434 :type slice_record: dict
435 :returns: True if all the jobs in the slice have been deleted,
436 or the list of jobs that could not be deleted otherwise.
437 :rtype: list or boolean
439 .. seealso:: DeleteSliceFromNodes
442 ret = self.DeleteSliceFromNodes(slice_record)
445 if False in ret[job_id]:
446 if delete_failed is None:
448 delete_failed.append(job_id)
450 logger.info("CORTEXLAB_API DeleteSlice %s answer %s"%(slice_record, \
452 return delete_failed or True
455 def __add_person_to_db(user_dict):
457 Add a federated user straight to db when the user issues a lease
458 request with iotlab nodes and that he has not registered with iotlab
459 yet (that is he does not have a LDAP entry yet).
460 Uses parts of the routines in SlabImport when importing user from LDAP.
461 Called by AddPerson, right after LdapAddUser.
462 :param user_dict: Must contain email, hrn and pkey to get a GID
463 and be added to the SFA db.
464 :type user_dict: dict
468 dbsession.query(RegUser).filter_by(email = user_dict['email']).first()
470 if not check_if_exists:
471 logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
473 hrn = user_dict['hrn']
474 person_urn = hrn_to_urn(hrn, 'user')
475 pubkey = user_dict['pkey']
477 pkey = convert_public_key(pubkey)
479 #key not good. create another pkey
480 logger.warn('__add_person_to_db: unable to convert public \
482 pkey = Keypair(create=True)
485 if pubkey is not None and pkey is not None :
486 hierarchy = Hierarchy()
487 person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
489 if user_dict['email']:
490 logger.debug("__add_person_to_db \r\n \r\n \
491 IOTLAB IMPORTER PERSON EMAIL OK email %s "\
492 %(user_dict['email']))
493 person_gid.set_email(user_dict['email'])
495 user_record = RegUser(hrn=hrn , pointer= '-1', \
496 authority=get_authority(hrn), \
497 email=user_dict['email'], gid = person_gid)
498 user_record.reg_keys = [RegKey(user_dict['pkey'])]
499 user_record.just_created()
500 dbsession.add (user_record)
505 def AddPerson(self, record):
508 Adds a new account. Any fields specified in records are used,
509 otherwise defaults are used. Creates an appropriate login by calling
512 :param record: dictionary with the sfa user's properties.
513 :returns: a dicitonary with the status. If successful, the dictionary
514 boolean is set to True and there is a 'uid' key with the new login
515 added to LDAP, otherwise the bool is set to False and a key
516 'message' is in the dictionary, with the error message.
520 ret = self.ldap.LdapAddUser(record)
522 if ret['bool'] is True:
523 record['hrn'] = self.root_auth + '.' + ret['uid']
524 logger.debug("CORTEXLAB_API AddPerson return code %s record %s "
526 self.__add_person_to_db(record)
533 #TODO AddPersonKey 04/07/2012 SA
534 def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
535 """Adds a new key to the specified account. Adds the key to the
536 iotlab ldap, provided that the person_uid is valid.
538 Non-admins can only modify their own keys.
540 :param person_uid: user's iotlab login in LDAP
541 :param old_attributes_dict: dict with the user's old sshPublicKey
542 :param new_key_dict: dict with the user's new sshPublicKey
543 :type person_uid: string
547 :returns: True if the key has been modified, False otherwise.
550 ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \
552 logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ")
555 def DeleteLeases(self, leases_id_list, slice_hrn):
558 Deletes several leases, based on their job ids and the slice
559 they are associated with. Uses DeleteJobs to delete the jobs
560 on OAR. Note that one slice can contain multiple jobs, and in this
561 case all the jobs in the leases_id_list MUST belong to ONE slice,
562 since there is only one slice hrn provided here.
564 :param leases_id_list: list of job ids that belong to the slice whose
565 slice hrn is provided.
566 :param slice_hrn: the slice hrn.
567 :type slice_hrn: string
569 .. warning:: Does not have a return value since there was no easy
570 way to handle failure when dealing with multiple job delete. Plus,
571 there was no easy way to report it to the user.
574 logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
575 \r\n " %(leases_id_list, slice_hrn))
576 for job_id in leases_id_list:
577 self.DeleteJobs(job_id, slice_hrn)
582 def _process_walltime(duration):
583 """ Calculates the walltime in seconds from the duration in H:M:S
584 specified in the RSpec.
588 # Fixing the walltime by adding a few delays.
589 # First put the walltime in seconds oarAdditionalDelay = 20;
590 # additional delay for /bin/sleep command to
591 # take in account prologue and epilogue scripts execution
592 # int walltimeAdditionalDelay = 240; additional delay
593 #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
595 # Put the duration in seconds first
596 #desired_walltime = duration * 60
597 desired_walltime = duration
598 total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
599 sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
601 #Put the walltime back in str form
603 walltime.append(str(total_walltime / 3600))
604 total_walltime = total_walltime - 3600 * int(walltime[0])
605 #Get the remaining minutes
606 walltime.append(str(total_walltime / 60))
607 total_walltime = total_walltime - 60 * int(walltime[1])
609 walltime.append(str(total_walltime))
612 logger.log_exc(" __process_walltime duration null")
614 return walltime, sleep_walltime
617 def _create_job_structure_request_for_OAR(lease_dict):
618 """ Creates the structure needed for a correct POST on OAR.
619 Makes the timestamp transformation into the appropriate format.
620 Sends the POST request to create the job with the resources in
629 reqdict['workdir'] = '/tmp'
630 reqdict['resource'] = "{network_address in ("
632 for node in lease_dict['added_nodes']:
633 logger.debug("\r\n \r\n OARrestapi \t \
634 __create_job_structure_request_for_OAR node %s" %(node))
636 # Get the ID of the node
638 reqdict['resource'] += "'" + nodeid + "', "
639 nodeid_list.append(nodeid)
641 custom_length = len(reqdict['resource'])- 2
642 reqdict['resource'] = reqdict['resource'][0:custom_length] + \
643 ")}/nodes=" + str(len(nodeid_list))
646 walltime, sleep_walltime = \
647 IotlabTestbedAPI._process_walltime(\
648 int(lease_dict['lease_duration']))
651 reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
652 ":" + str(walltime[1]) + ":" + str(walltime[2])
653 reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
655 #In case of a scheduled experiment (not immediate)
656 #To run an XP immediately, don't specify date and time in RSpec
657 #They will be set to None.
658 if lease_dict['lease_start_time'] is not '0':
659 #Readable time accepted by OAR
660 start_time = datetime.fromtimestamp( \
661 int(lease_dict['lease_start_time'])).\
662 strftime(lease_dict['time_format'])
663 reqdict['reservation'] = start_time
664 #If there is not start time, Immediate XP. No need to add special
668 reqdict['type'] = "deploy"
669 reqdict['directory'] = ""
670 reqdict['name'] = "SFA_" + lease_dict['slice_user']
675 def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
676 lease_start_time, lease_duration, slice_user=None):
679 Create a job request structure based on the information provided
680 and post the job on OAR.
681 :param added_nodes: list of nodes that belong to the described lease.
682 :param slice_name: the slice hrn associated to the lease.
683 :param lease_start_time: timestamp of the lease startting time.
684 :param lease_duration: lease durationin minutes
688 lease_dict['lease_start_time'] = lease_start_time
689 lease_dict['lease_duration'] = lease_duration
690 lease_dict['added_nodes'] = added_nodes
691 lease_dict['slice_name'] = slice_name
692 lease_dict['slice_user'] = slice_user
693 lease_dict['grain'] = self.GetLeaseGranularity()
694 lease_dict['time_format'] = self.time_format
697 logger.debug("CORTEXLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\
698 \r\n " %(slice_user))
699 #Create the request for OAR
700 reqdict = self._create_job_structure_request_for_OAR(lease_dict)
701 # first step : start the OAR job and update the job
702 logger.debug("CORTEXLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\
705 answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
707 logger.debug("CORTEXLAB_API \tLaunchExperimentOnOAR jobid %s " %(answer))
711 logger.log_exc("CORTEXLAB_API \tLaunchExperimentOnOAR \
712 Impossible to create job %s " %(answer))
719 logger.debug("CORTEXLAB_API \tLaunchExperimentOnOAR jobid %s \
720 added_nodes %s slice_user %s" %(jobid, added_nodes, \
727 def AddLeases(self, hostname_list, slice_record,
728 lease_start_time, lease_duration):
730 """Creates a job in OAR corresponding to the information provided
731 as parameters. Adds the job id and the slice hrn in the iotlab
732 database so that we are able to know which slice has which nodes.
734 :param hostname_list: list of nodes' OAR hostnames.
735 :param slice_record: sfa slice record, must contain login and hrn.
736 :param lease_start_time: starting time , unix timestamp format
737 :param lease_duration: duration in minutes
739 :type hostname_list: list
740 :type slice_record: dict
741 :type lease_start_time: integer
742 :type lease_duration: integer
745 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \
746 slice_record %s lease_start_time %s lease_duration %s "\
747 %( hostname_list, slice_record , lease_start_time, \
750 #tmp = slice_record['reg-researchers'][0].split(".")
751 username = slice_record['login']
752 #username = tmp[(len(tmp)-1)]
753 job_id = self.LaunchExperimentOnOAR(hostname_list, \
754 slice_record['hrn'], \
755 lease_start_time, lease_duration, \
758 datetime.fromtimestamp(int(lease_start_time)).\
759 strftime(self.time_format)
760 end_time = lease_start_time + lease_duration
763 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
764 %s %s %s "%(slice_record['hrn'], job_id, end_time))
767 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \
768 %(type(slice_record['hrn']), type(job_id), type(end_time)))
770 iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id,
773 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \
775 self.iotlab_db.iotlab_session.add(iotlab_ex_row)
776 self.iotlab_db.iotlab_session.commit()
778 logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \
784 #Delete the jobs from job_iotlab table
785 def DeleteSliceFromNodes(self, slice_record):
788 Deletes all the running or scheduled jobs of a given slice
791 :param slice_record: record of the slice, must contain oar_job_id, user
792 :type slice_record: dict
794 :returns: dict of the jobs'deletion status. Success= True, Failure=
795 False, for each job id.
799 logger.debug("CORTEXLAB_API \t DeleteSliceFromNodes %s "
802 if isinstance(slice_record['oar_job_id'], list):
804 for job_id in slice_record['oar_job_id']:
805 ret = self.DeleteJobs(job_id, slice_record['user'])
807 oar_bool_answer.update(ret)
810 oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'],
811 slice_record['user'])]
813 return oar_bool_answer
817 def GetLeaseGranularity(self):
818 """ Returns the granularity of an experiment in the Iotlab testbed.
819 OAR uses seconds for experiments duration , the granulaity is also
821 Experiments which last less than 10 min (600 sec) are invalid"""
826 # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql):
827 # """ Cleans the iotlab db by deleting expired and cancelled jobs.
828 # Compares the list of job ids given by OAR with the job ids that
829 # are already in the database, deletes the jobs that are no longer in
830 # the OAR job id list.
831 # :param job_oar_list: list of job ids coming from OAR
832 # :type job_oar_list: list
833 # :param job_psql: list of job ids cfrom the database.
834 # type job_psql: list
836 # #Turn the list into a set
837 # set_jobs_psql = set(jobs_psql)
839 # kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
840 # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \
841 # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
842 # deleted_jobs = set_jobs_psql.difference(kept_jobs)
843 # deleted_jobs = list(deleted_jobs)
844 # if len(deleted_jobs) > 0:
845 # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
846 # self.iotlab_db.iotlab_session.commit()
851 def filter_lease_name(reservation_list, filter_value):
852 filtered_reservation_list = list(reservation_list)
853 logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \
854 % (reservation_list))
855 for reservation in reservation_list:
856 if 'slice_hrn' in reservation and \
857 reservation['slice_hrn'] != filter_value:
858 filtered_reservation_list.remove(reservation)
860 logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \
861 % (filtered_reservation_list))
862 return filtered_reservation_list
865 def filter_lease_start_time(reservation_list, filter_value):
866 filtered_reservation_list = list(reservation_list)
868 for reservation in reservation_list:
869 if 't_from' in reservation and \
870 reservation['t_from'] > filter_value:
871 filtered_reservation_list.remove(reservation)
873 return filtered_reservation_list
875 def GetLeases(self, lease_filter_dict=None, login=None):
878 Get the list of leases from OAR with complete information
879 about which slice owns which jobs and nodes.
881 -Fetch all the jobs from OAR (running, waiting..)
882 complete the reservation information with slice hrn
883 found in iotlab_xp table. If not available in the table,
884 assume it is a iotlab slice.
885 -Updates the iotlab table, deleting jobs when necessary.
887 :returns: reservation_list, list of dictionaries with 'lease_id',
888 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list',
889 'slice_hrn', 'resource_ids', 't_from', 't_until'
894 unfiltered_reservation_list = self.GetReservedNodes(login)
896 reservation_list = []
897 #Find the slice associated with this user iotlab ldap uid
898 logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\
899 unfiltered_reservation_list %s "
900 % (login, unfiltered_reservation_list))
901 #Create user dict first to avoid looking several times for
902 #the same user in LDAP SA 27/07/12
905 jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all()
906 jobs_psql_dict = dict([(row.job_id, row.__dict__)
907 for row in jobs_psql_query])
908 #jobs_psql_dict = jobs_psql_dict)
909 logger.debug("CORTEXLAB_API \tGetLeases jobs_psql_dict %s"
911 jobs_psql_id_list = [row.job_id for row in jobs_psql_query]
913 for resa in unfiltered_reservation_list:
914 logger.debug("CORTEXLAB_API \tGetLeases USER %s"
916 #Construct list of jobs (runing, waiting..) in oar
917 job_oar_list.append(resa['lease_id'])
918 #If there is information on the job in IOTLAB DB ]
919 #(slice used and job id)
920 if resa['lease_id'] in jobs_psql_dict:
921 job_info = jobs_psql_dict[resa['lease_id']]
922 logger.debug("CORTEXLAB_API \tGetLeases job_info %s"
924 resa['slice_hrn'] = job_info['slice_hrn']
925 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
927 #otherwise, assume it is a iotlab slice:
929 resa['slice_id'] = hrn_to_urn(self.root_auth + '.' +
930 resa['user'] + "_slice", 'slice')
931 resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
933 resa['component_id_list'] = []
934 #Transform the hostnames into urns (component ids)
935 for node in resa['reserved_nodes']:
937 iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
938 resa['component_id_list'].append(iotlab_xrn.urn)
940 if lease_filter_dict:
941 logger.debug("CORTEXLAB_API \tGetLeases \
942 \r\n leasefilter %s" % ( lease_filter_dict))
944 filter_dict_functions = {
945 'slice_hrn' : IotlabTestbedAPI.filter_lease_name,
946 't_from' : IotlabTestbedAPI.filter_lease_start_time
948 reservation_list = list(unfiltered_reservation_list)
949 for filter_type in lease_filter_dict:
950 logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \
951 % (reservation_list))
952 reservation_list = filter_dict_functions[filter_type](\
953 reservation_list,lease_filter_dict[filter_type] )
955 # Filter the reservation list with a maximum timespan so that the
956 # leases and jobs running after this timestamp do not appear
957 # in the result leases.
958 # if 'start_time' in :
959 # if resa['start_time'] < lease_filter_dict['start_time']:
960 # reservation_list.append(resa)
963 # if 'name' in lease_filter_dict and \
964 # lease_filter_dict['name'] == resa['slice_hrn']:
965 # reservation_list.append(resa)
968 if lease_filter_dict is None:
969 reservation_list = unfiltered_reservation_list
971 self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list)
973 logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s"
974 % (reservation_list))
975 return reservation_list
980 #TODO FUNCTIONS SECTION 04/07/2012 SA
982 ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
985 #def UnBindObjectFromPeer( auth, object_type, object_id, shortname):
986 #""" This method is a hopefully temporary hack to let the sfa correctly
987 #detach the objects it creates from a remote peer object. This is
988 #needed so that the sfa federation link can work in parallel with
989 #RefreshPeer, as RefreshPeer depends on remote objects being correctly
992 #auth : struct, API authentication structure
993 #AuthMethod : string, Authentication method to use
994 #object_type : string, Object type, among 'site','person','slice',
996 #object_id : int, object_id
997 #shortname : string, peer shortname
1001 #logger.warning("CORTEXLAB_API \tUnBindObjectFromPeer EMPTY-\
1005 ##TODO Is BindObjectToPeer still necessary ? Currently does nothing
1007 #|| Commented out 28/05/13 SA
1008 #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
1009 #remote_object_id=None):
1010 #"""This method is a hopefully temporary hack to let the sfa correctly
1011 #attach the objects it creates to a remote peer object. This is needed
1012 #so that the sfa federation link can work in parallel with RefreshPeer,
1013 #as RefreshPeer depends on remote objects being correctly marked.
1015 #shortname : string, peer shortname
1016 #remote_object_id : int, remote object_id, set to 0 if unknown
1020 #logger.warning("CORTEXLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
1023 ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA
1024 ##Funciton should delete and create another job since oin iotlab slice=job
1025 #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
1026 #"""Updates the parameters of an existing slice with the values in
1028 #Users may only update slices of which they are members.
1029 #PIs may update any of the slices at their sites, or any slices of
1030 #which they are members. Admins may update any slice.
1031 #Only PIs and admins may update max_nodes. Slices cannot be renewed
1032 #(by updating the expires parameter) more than 8 weeks into the future.
1033 #Returns 1 if successful, faults otherwise.
1037 #logger.warning("CORTEXLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ")
1040 #Unused SA 30/05/13, we only update the user's key or we delete it.
1041 ##TODO UpdatePerson 04/07/2012 SA
1042 #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None):
1043 #"""Updates a person. Only the fields specified in person_fields
1044 #are updated, all other fields are left untouched.
1045 #Users and techs can only update themselves. PIs can only update
1046 #themselves and other non-PIs at their sites.
1047 #Returns 1 if successful, faults otherwise.
1051 ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
1052 ##self.iotlab_db.iotlab_session.add(new_row)
1053 ##self.iotlab_db.iotlab_session.commit()
1055 #logger.debug("CORTEXLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
1059 def GetKeys(key_filter=None):
1060 """Returns a dict of dict based on the key string. Each dict entry
1061 contains the key id, the ssh key, the user's email and the
1063 If key_filter is specified and is an array of key identifiers,
1064 only keys matching the filter will be returned.
1066 Admin may query all keys. Non-admins may only query their own keys.
1069 :returns: dict with ssh key as key and dicts as value.
1072 if key_filter is None:
1073 keys = dbsession.query(RegKey).options(joinedload('reg_user')).all()
1075 keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all()
1079 key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
1080 'email': key.reg_user.email,
1081 'hrn': key.reg_user.hrn}
1083 #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
1084 #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
1085 #for user in ldap_rslt)
1087 logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
1091 def DeleteKey(self, user_record, key_string):
1092 """Deletes a key in the LDAP entry of the specified user.
1094 Removes the key_string from the user's key list and updates the LDAP
1095 user's entry with the new key attributes.
1097 :param key_string: The ssh key to remove
1098 :param user_record: User's record
1099 :type key_string: string
1100 :type user_record: dict
1101 :returns: True if sucessful, False if not.
1105 all_user_keys = user_record['keys']
1106 all_user_keys.remove(key_string)
1107 new_attributes = {'sshPublicKey':all_user_keys}
1108 ret = self.ldap.LdapModifyUser(user_record, new_attributes)
1109 logger.debug("CORTEXLAB_API DeleteKey %s- " % (ret))
1116 def _sql_get_slice_info(slice_filter):
1118 Get the slice record based on the slice hrn. Fetch the record of the
1119 user associated with the slice by using joinedload based on the
1120 reg_researcher relationship.
1122 :param slice_filter: the slice hrn we are looking for
1123 :type slice_filter: string
1124 :returns: the slice record enhanced with the user's information if the
1125 slice was found, None it wasn't.
1127 :rtype: dict or None.
1129 #DO NOT USE RegSlice - reg_researchers to get the hrn
1130 #of the user otherwise will mess up the RegRecord in
1131 #Resolve, don't know why - SA 08/08/2012
1133 #Only one entry for one user = one slice in iotlab_xp table
1134 #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1135 raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
1136 #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1138 #load_reg_researcher
1139 #raw_slicerec.reg_researchers
1140 raw_slicerec = raw_slicerec.__dict__
1141 logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \
1142 raw_slicerec %s" % (slice_filter, raw_slicerec))
1143 slicerec = raw_slicerec
1144 #only one researcher per slice so take the first one
1145 #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
1146 #del slicerec['reg_researchers']['_sa_instance_state']
1153 def _sql_get_slice_info_from_user(slice_filter):
1155 Get the slice record based on the user recordid by using a joinedload
1156 on the relationship reg_slices_as_researcher. Format the sql record
1157 into a dict with the mandatory fields for user and slice.
1158 :returns: dict with slice record and user record if the record was found
1159 based on the user's id, None if not..
1160 :rtype:dict or None..
1162 #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1163 raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
1164 #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1165 #Put it in correct order
1166 user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1167 'classtype', 'authority', 'gid', 'record_id',
1168 'date_created', 'type', 'email', 'pointer']
1169 slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1170 'classtype', 'authority', 'gid', 'record_id',
1171 'date_created', 'type', 'pointer']
1173 #raw_slicerec.reg_slices_as_researcher
1174 raw_slicerec = raw_slicerec.__dict__
1177 dict([(k, raw_slicerec[
1178 'reg_slices_as_researcher'][0].__dict__[k])
1179 for k in slice_needed_fields])
1180 slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
1181 for k in user_needed_fields])
1182 #TODO Handle multiple slices for one user SA 10/12/12
1183 #for now only take the first slice record associated to the rec user
1184 ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
1185 #del raw_slicerec['reg_slices_as_researcher']
1186 #slicerec['reg_researchers'] = raw_slicerec
1187 ##del slicerec['_sa_instance_state']
1194 def _get_slice_records(self, slice_filter=None,
1195 slice_filter_type=None):
1197 Get the slice record depending on the slice filter and its type.
1198 :param slice_filter: Can be either the slice hrn or the user's record
1200 :type slice_filter: string
1201 :param slice_filter_type: describes the slice filter type used, can be
1202 slice_hrn or record_id_user
1204 :returns: the slice record
1206 .. seealso::_sql_get_slice_info_from_user
1207 .. seealso:: _sql_get_slice_info
1210 #Get list of slices based on the slice hrn
1211 if slice_filter_type == 'slice_hrn':
1213 #if get_authority(slice_filter) == self.root_auth:
1214 #login = slice_filter.split(".")[1].split("_")[0]
1216 slicerec = self._sql_get_slice_info(slice_filter)
1218 if slicerec is None:
1222 #Get slice based on user id
1223 if slice_filter_type == 'record_id_user':
1225 slicerec = self._sql_get_slice_info_from_user(slice_filter)
1228 fixed_slicerec_dict = slicerec
1229 #At this point if there is no login it means
1230 #record_id_user filter has been used for filtering
1232 ##If theslice record is from iotlab
1233 #if fixed_slicerec_dict['peer_authority'] is None:
1234 #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
1235 #return login, fixed_slicerec_dict
1236 return fixed_slicerec_dict
1241 def GetSlices(self, slice_filter=None, slice_filter_type=None,
1243 """Get the slice records from the iotlab db and add lease information
1246 :param slice_filter: can be the slice hrn or slice record id in the db
1247 depending on the slice_filter_type.
1248 :param slice_filter_type: defines the type of the filtering used, Can be
1249 either 'slice_hrn' or "record_id'.
1250 :type slice_filter: string
1251 :type slice_filter_type: string
1252 :returns: a slice dict if slice_filter and slice_filter_type
1253 are specified and a matching entry is found in the db. The result
1254 is put into a list.Or a list of slice dictionnaries if no filters
1261 authorized_filter_types_list = ['slice_hrn', 'record_id_user']
1262 return_slicerec_dictlist = []
1264 #First try to get information on the slice based on the filter provided
1265 if slice_filter_type in authorized_filter_types_list:
1266 fixed_slicerec_dict = self._get_slice_records(slice_filter,
1268 # if the slice was not found in the sfa db
1269 if fixed_slicerec_dict is None:
1270 return return_slicerec_dictlist
1272 slice_hrn = fixed_slicerec_dict['hrn']
1274 logger.debug(" CORTEXLAB_API \tGetSlices login %s \
1275 slice record %s slice_filter %s \
1276 slice_filter_type %s " % (login,
1277 fixed_slicerec_dict, slice_filter,
1281 #Now we have the slice record fixed_slicerec_dict, get the
1282 #jobs associated to this slice
1285 leases_list = self.GetLeases(login=login)
1286 #If no job is running or no job scheduled
1287 #return only the slice record
1288 if leases_list == [] and fixed_slicerec_dict:
1289 return_slicerec_dictlist.append(fixed_slicerec_dict)
1291 # if the jobs running don't belong to the user/slice we are looking
1293 leases_hrn = [lease['slice_hrn'] for lease in leases_list]
1294 if slice_hrn not in leases_hrn:
1295 return_slicerec_dictlist.append(fixed_slicerec_dict)
1296 #If several jobs for one slice , put the slice record into
1297 # each lease information dict
1298 for lease in leases_list:
1300 logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \
1301 \t lease['slice_hrn'] %s"
1302 % (slice_filter, lease['slice_hrn']))
1303 if lease['slice_hrn'] == slice_hrn:
1304 slicerec_dict['oar_job_id'] = lease['lease_id']
1305 #Update lease dict with the slice record
1306 if fixed_slicerec_dict:
1307 fixed_slicerec_dict['oar_job_id'] = []
1308 fixed_slicerec_dict['oar_job_id'].append(
1309 slicerec_dict['oar_job_id'])
1310 slicerec_dict.update(fixed_slicerec_dict)
1311 #slicerec_dict.update({'hrn':\
1312 #str(fixed_slicerec_dict['slice_hrn'])})
1313 slicerec_dict['slice_hrn'] = lease['slice_hrn']
1314 slicerec_dict['hrn'] = lease['slice_hrn']
1315 slicerec_dict['user'] = lease['user']
1316 slicerec_dict.update(
1318 {'hostname': lease['reserved_nodes']}})
1319 slicerec_dict.update({'node_ids': lease['reserved_nodes']})
1323 return_slicerec_dictlist.append(slicerec_dict)
1324 logger.debug("CORTEXLAB_API.PY \tGetSlices \
1325 OHOHOHOH %s" %(return_slicerec_dictlist))
1327 logger.debug("CORTEXLAB_API.PY \tGetSlices \
1328 slicerec_dict %s return_slicerec_dictlist %s \
1329 lease['reserved_nodes'] \
1330 %s" % (slicerec_dict, return_slicerec_dictlist,
1331 lease['reserved_nodes']))
1333 logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
1334 return_slicerec_dictlist %s"
1335 % (return_slicerec_dictlist))
1337 return return_slicerec_dictlist
1341 #Get all slices from the iotlab sfa database ,
1342 #put them in dict format
1343 #query_slice_list = dbsession.query(RegRecord).all()
1344 query_slice_list = \
1345 dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
1347 for record in query_slice_list:
1348 tmp = record.__dict__
1349 tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
1350 #del tmp['reg_researchers']['_sa_instance_state']
1351 return_slicerec_dictlist.append(tmp)
1352 #return_slicerec_dictlist.append(record.__dict__)
1354 #Get all the jobs reserved nodes
1355 leases_list = self.GetReservedNodes()
1357 for fixed_slicerec_dict in return_slicerec_dictlist:
1359 #Check if the slice belongs to a iotlab user
1360 if fixed_slicerec_dict['peer_authority'] is None:
1361 owner = fixed_slicerec_dict['hrn'].split(
1362 ".")[1].split("_")[0]
1365 for lease in leases_list:
1366 if owner == lease['user']:
1367 slicerec_dict['oar_job_id'] = lease['lease_id']
1369 #for reserved_node in lease['reserved_nodes']:
1370 logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s "
1372 slicerec_dict.update(fixed_slicerec_dict)
1373 slicerec_dict.update({'node_ids':
1374 lease['reserved_nodes']})
1375 slicerec_dict.update({'list_node_ids':
1377 lease['reserved_nodes']}})
1379 #slicerec_dict.update({'hrn':\
1380 #str(fixed_slicerec_dict['slice_hrn'])})
1381 #return_slicerec_dictlist.append(slicerec_dict)
1382 fixed_slicerec_dict.update(slicerec_dict)
1384 logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
1385 return_slicerec_dictlist %s \slice_filter %s " \
1386 %(return_slicerec_dictlist, slice_filter))
1388 return return_slicerec_dictlist
1392 #Update slice unused, therefore sfa_fields_to_iotlab_fields unused
1395 #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record):
1400 ##for field in record:
1401 ## iotlab_record[field] = record[field]
1403 #if sfa_type == "slice":
1404 ##instantion used in get_slivers ?
1405 #if not "instantiation" in iotlab_record:
1406 #iotlab_record["instantiation"] = "iotlab-instantiated"
1407 ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn)
1408 ##Unused hrn_to_pl_slicename because Iotlab's hrn already
1409 ##in the appropriate form SA 23/07/12
1410 #iotlab_record["hrn"] = hrn
1411 #logger.debug("CORTEXLAB_API.PY sfa_fields_to_iotlab_fields \
1412 #iotlab_record %s " %(iotlab_record['hrn']))
1413 #if "url" in record:
1414 #iotlab_record["url"] = record["url"]
1415 #if "description" in record:
1416 #iotlab_record["description"] = record["description"]
1417 #if "expires" in record:
1418 #iotlab_record["expires"] = int(record["expires"])
1420 ##nodes added by OAR only and then imported to SFA
1421 ##elif type == "node":
1422 ##if not "hostname" in iotlab_record:
1423 ##if not "hostname" in record:
1424 ##raise MissingSfaInfo("hostname")
1425 ##iotlab_record["hostname"] = record["hostname"]
1426 ##if not "model" in iotlab_record:
1427 ##iotlab_record["model"] = "geni"
1429 ##One authority only
1430 ##elif type == "authority":
1431 ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn)
1433 ##if not "name" in iotlab_record:
1434 ##iotlab_record["name"] = hrn
1436 ##if not "abbreviated_name" in iotlab_record:
1437 ##iotlab_record["abbreviated_name"] = hrn
1439 ##if not "enabled" in iotlab_record:
1440 ##iotlab_record["enabled"] = True
1442 ##if not "is_public" in iotlab_record:
1443 ##iotlab_record["is_public"] = True
1445 #return iotlab_record