2 File containing the IotlabTestbedAPI, used to interact with nodes, users,
3 slices, leases and keys, as well as the dedicated iotlab database and table,
4 holding information about which slice is running which job.
7 from datetime import datetime
9 from sfa.util.sfalogging import logger
11 from sfa.storage.alchemy import dbsession
12 from sqlalchemy.orm import joinedload
13 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
15 from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP
16 from sfa.cortexlab.LDAPapi import LDAPapi
18 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
20 from sfa.trust.certificate import Keypair, convert_public_key
21 from sfa.trust.gid import create_uuid
22 from sfa.trust.hierarchy import Hierarchy
24 from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
26 class CortexlabTestbedAPI():
27 """ Class enabled to use LDAP and OAR api calls. """
29 _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins
31 def __init__(self, config):
32 """Creates an instance of OARrestapi and LDAPapi which will be used to
33 issue calls to OAR or LDAP methods.
34 Set the time format and the testbed granularity used for OAR
35 reservation and leases.
37 :param config: configuration object from sfa.util.config
38 :type config: Config object
40 self.cortexlab_leases_db = TestbedAdditionalSfaDB(config)
41 self.query_sites = CortexlabQueryNodes()
43 self.time_format = "%Y-%m-%d %H:%M:%S"
44 self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
45 self.grain = 60 # 10 mins lease minimum, 60 sec granularity
46 #import logging, logging.handlers
47 #from sfa.util.sfalogging import _SfaLogger
48 #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \
53 def GetMinExperimentDurationInGranularity():
54 """ Returns the minimum allowed duration for an experiment on the
58 return CortexlabTestbedAPI._MINIMUM_DURATION
61 def GetPeers(peer_filter=None ):
62 """ Gathers registered authorities in SFA DB and looks for specific peer
63 if peer_filter is specified.
64 :param peer_filter: name of the site authority looked for.
65 :type peer_filter: string
66 :returns: list of records.
71 existing_hrns_by_types = {}
72 logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter))
73 all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
75 for record in all_records:
76 existing_records[(record.hrn, record.type)] = record
77 if record.type not in existing_hrns_by_types:
78 existing_hrns_by_types[record.type] = [record.hrn]
80 existing_hrns_by_types[record.type].append(record.hrn)
82 logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s "
83 % (existing_hrns_by_types))
88 records_list.append(existing_records[(peer_filter,
91 for hrn in existing_hrns_by_types['authority']:
92 records_list.append(existing_records[(hrn, 'authority')])
94 logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s "
100 return_records = records_list
101 logger.debug("CORTEXLAB_API \tGetPeer return_records %s "
103 return return_records
105 #TODO : Handling OR request in make_ldap_filters_from_records
106 #instead of the for loop
107 #over the records' list
108 def GetPersons(self, person_filter=None):
110 Get the enabled users and their properties from Cortexlab LDAP.
111 If a filter is specified, looks for the user whose properties match
112 the filter, otherwise returns the whole enabled users'list.
114 :param person_filter: Must be a list of dictionnaries with users
115 properties when not set to None.
116 :type person_filter: list of dict
118 :returns: Returns a list of users whose accounts are enabled
120 :rtype: list of dicts
123 logger.debug("CORTEXLAB_API \tGetPersons person_filter %s"
126 if person_filter and isinstance(person_filter, list):
127 #If we are looking for a list of users (list of dict records)
128 #Usually the list contains only one user record
129 for searched_attributes in person_filter:
131 #Get only enabled user accounts in iotlab LDAP :
132 #add a filter for make_ldap_filters_from_record
133 person = self.ldap.LdapFindUser(searched_attributes,
134 is_user_enabled=True)
135 #If a person was found, append it to the list
137 person_list.append(person)
139 #If the list is empty, return None
140 if len(person_list) is 0:
144 #Get only enabled user accounts in iotlab LDAP :
145 #add a filter for make_ldap_filters_from_record
146 person_list = self.ldap.LdapFindUser(is_user_enabled=True)
152 def DeleteOneLease(self, lease_id, username):
155 Deletes the lease with the specified lease_id and username on OAR by
156 posting a delete request to OAR.
158 :param lease_id: Reservation identifier.
159 :param username: user's iotlab login in LDAP.
160 :type lease_id: Depends on what tou are using, could be integer or
162 :type username: string
164 :returns: dictionary with the lease id and if delete has been successful
170 # Here delete the lease specified
171 answer = self.query_sites.delete_experiment(lease_id, username)
173 # If the username is not necessary to delete the lease, then you can
174 # remove it from the parameters, given that you propagate the changes
175 # Return delete status so that you know if the delete has been
179 if answer['status'] is True:
180 ret = {lease_id: True}
182 ret = {lease_id: False}
183 logger.debug("CORTEXLAB_API \DeleteOneLease lease_id %s \r\n answer %s \
184 username %s" % (lease_id, answer, username))
189 def GetNodesCurrentlyInUse(self):
190 """Returns a list of all the nodes involved in a currently running
191 experiment (and only the one not available at the moment the call to
192 this method is issued)
193 :rtype: list of nodes hostnames.
195 node_hostnames_list = []
196 return node_hostnames_list
199 def GetReservedNodes(self, username=None):
200 """ Get list of leases. Get the leases for the username if specified,
201 otherwise get all the leases. Finds the nodes hostnames for each
203 :param username: user's LDAP login
204 :type username: string
205 :returns: list of reservations dict
209 #Get the nodes in use and the reserved nodes
210 mandatory_sfa_keys = ['reserved_nodes','lease_id']
211 reservation_dict_list = \
212 self.query_sites.get_reserved_nodes(username = username)
214 if len(reservation_dict_list) == 0:
218 # Ensure mandatory keys are in the dict
219 if not self.ensure_format_is_valid(reservation_dict_list,
221 raise KeyError, "GetReservedNodes : Missing SFA mandatory keys"
224 return reservation_dict_list
227 def ensure_format_is_valid(list_dictionary_to_check, mandatory_keys_list):
228 for entry in list_dictionary_to_check:
229 if not all (key in entry for key in mandatory_keys_list):
233 def GetNodes(self, node_filter_dict=None, return_fields_list=None):
236 Make a list of cortexlab nodes and their properties from information
237 given by ?. Search for specific nodes if some filters are
238 specified. Nodes properties returned if no return_fields_list given:
239 'hrn','archi','mobile','hostname','site','boot_state','node_id',
240 'radio','posx','posy,'posz'.
242 :param node_filter_dict: dictionnary of lists with node properties. For
243 instance, if you want to look for a specific node with its hrn,
244 the node_filter_dict should be {'hrn': [hrn_of_the_node]}
245 :type node_filter_dict: dict
246 :param return_fields_list: list of specific fields the user wants to be
248 :type return_fields_list: list
249 :returns: list of dictionaries with node properties. Mandatory
250 properties hrn, site, hostname. Complete list (iotlab) ['hrn',
251 'archi', 'mobile', 'hostname', 'site', 'mobility_type',
252 'boot_state', 'node_id','radio', 'posx', 'posy', 'oar_id', 'posz']
253 Radio, archi, mobile and position are useful to help users choose
254 the appropriate nodes.
257 :TODO: FILL IN THE BLANKS
260 # Here get full dict of nodes with all their properties.
261 mandatory_sfa_keys = ['hrn', 'site', 'hostname']
262 node_list_dict = self.query_sites.get_all_nodes(node_filter_dict,
265 if len(node_list_dict) == 0:
266 return_node_list = []
269 # Ensure mandatory keys are in the dict
270 if not self.ensure_format_is_valid(node_list_dict,
272 raise KeyError, "GetNodes : Missing SFA mandatory keys"
275 return_node_list = node_list_dict
276 return return_node_list
281 def AddSlice(slice_record, user_record):
284 Add slice to the local cortexlab sfa tables if the slice comes
285 from a federated site and is not yet in the cortexlab sfa DB,
286 although the user has already a LDAP login.
287 Called by verify_slice during lease/sliver creation.
289 :param slice_record: record of slice, must contain hrn, gid, slice_id
290 and authority of the slice.
291 :type slice_record: dictionary
292 :param user_record: record of the user
293 :type user_record: RegUser
297 sfa_record = RegSlice(hrn=slice_record['hrn'],
298 gid=slice_record['gid'],
299 pointer=slice_record['slice_id'],
300 authority=slice_record['authority'])
301 logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s"
302 % (sfa_record, user_record))
303 sfa_record.just_created()
304 dbsession.add(sfa_record)
306 #Update the reg-researcher dependance table
307 sfa_record.reg_researchers = [user_record]
313 def GetSites(self, site_filter_name_list=None, return_fields_list=None):
314 """Returns the list of Cortexlab's sites with the associated nodes and
315 the sites' properties as dictionaries. Used in import.
318 ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids',
319 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids',
320 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created',
321 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name']
322 can be empty ( []): address_ids, slice_ids, pcu_ids, person_ids,
325 :param site_filter_name_list: used to specify specific sites
326 :param return_fields_list: field that has to be returned
327 :type site_filter_name_list: list
328 :type return_fields_list: list
329 :rtype: list of dicts
332 site_list_dict = self.query_sites.get_sites(site_filter_name_list,
335 mandatory_sfa_keys = ['name', 'node_ids', 'longitude','site' ]
337 if len(site_list_dict) == 0:
338 return_site_list = []
341 # Ensure mandatory keys are in the dict
342 if not self.ensure_format_is_valid(site_list_dict,
344 raise KeyError, "GetSites : Missing sfa mandatory keys"
346 return_site_list = site_list_dict
347 return return_site_list
350 #TODO : Check rights to delete person
351 def DeletePerson(self, person_record):
352 """Disable an existing account in cortexlab LDAP.
354 Users and techs can only delete themselves. PIs can only
355 delete themselves and other non-PIs at their sites.
356 ins can delete anyone.
358 :param person_record: user's record
359 :type person_record: dict
360 :returns: True if successful, False otherwise.
363 .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF.
365 #Disable user account in iotlab LDAP
366 ret = self.ldap.LdapMarkUserAsDeleted(person_record)
367 logger.warning("CORTEXLAB_API DeletePerson %s " % (person_record))
370 def DeleteSlice(self, slice_record):
371 """Deletes the specified slice and kills the jobs associated with
372 the slice if any, using DeleteSliceFromNodes.
374 :param slice_record: record of the slice, must contain experiment_id, user
375 :type slice_record: dict
376 :returns: True if all the jobs in the slice have been deleted,
377 or the list of jobs that could not be deleted otherwise.
378 :rtype: list or boolean
380 .. seealso:: DeleteSliceFromNodes
383 ret = self.DeleteSliceFromNodes(slice_record)
385 for experiment_id in ret:
386 if False in ret[experiment_id]:
387 if delete_failed is None:
389 delete_failed.append(experiment_id)
391 logger.info("CORTEXLAB_API DeleteSlice %s answer %s"%(slice_record, \
393 return delete_failed or True
396 def __add_person_to_db(user_dict):
398 Add a federated user straight to db when the user issues a lease
399 request with iotlab nodes and that he has not registered with cortexlab
400 yet (that is he does not have a LDAP entry yet).
401 Uses parts of the routines in CortexlabImport when importing user
403 Called by AddPerson, right after LdapAddUser.
404 :param user_dict: Must contain email, hrn and pkey to get a GID
405 and be added to the SFA db.
406 :type user_dict: dict
410 dbsession.query(RegUser).filter_by(email = user_dict['email']).first()
412 if not check_if_exists:
413 logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
415 hrn = user_dict['hrn']
416 person_urn = hrn_to_urn(hrn, 'user')
417 pubkey = user_dict['pkey']
419 pkey = convert_public_key(pubkey)
421 #key not good. create another pkey
422 logger.warn('__add_person_to_db: unable to convert public \
424 pkey = Keypair(create=True)
427 if pubkey is not None and pkey is not None :
428 hierarchy = Hierarchy()
429 person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
431 if user_dict['email']:
432 logger.debug("__add_person_to_db \r\n \r\n \
433 IOTLAB IMPORTER PERSON EMAIL OK email %s "\
434 %(user_dict['email']))
435 person_gid.set_email(user_dict['email'])
437 user_record = RegUser(hrn=hrn , pointer= '-1', \
438 authority=get_authority(hrn), \
439 email=user_dict['email'], gid = person_gid)
440 user_record.reg_keys = [RegKey(user_dict['pkey'])]
441 user_record.just_created()
442 dbsession.add (user_record)
447 def AddPerson(self, record):
450 Adds a new account. Any fields specified in records are used,
451 otherwise defaults are used. Creates an appropriate login by calling
454 :param record: dictionary with the sfa user's properties.
455 :returns: a dicitonary with the status. If successful, the dictionary
456 boolean is set to True and there is a 'uid' key with the new login
457 added to LDAP, otherwise the bool is set to False and a key
458 'message' is in the dictionary, with the error message.
462 ret = self.ldap.LdapAddUser(record)
464 if ret['bool'] is True:
465 record['hrn'] = self.root_auth + '.' + ret['uid']
466 logger.debug("CORTEXLAB_API AddPerson return code %s record %s "
468 self.__add_person_to_db(record)
475 #TODO AddPersonKey 04/07/2012 SA
476 def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
477 """Adds a new key to the specified account. Adds the key to the
478 iotlab ldap, provided that the person_uid is valid.
480 Non-admins can only modify their own keys.
482 :param person_uid: user's iotlab login in LDAP
483 :param old_attributes_dict: dict with the user's old sshPublicKey
484 :param new_key_dict: dict with the user's new sshPublicKey
485 :type person_uid: string
489 :returns: True if the key has been modified, False otherwise.
492 ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \
494 logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ")
499 def _process_walltime(duration):
500 """ Calculates the walltime in seconds from the duration in H:M:S
501 specified in the RSpec.
505 # Fixing the walltime by adding a few delays.
506 # First put the walltime in seconds oarAdditionalDelay = 20;
507 # additional delay for /bin/sleep command to
508 # take in account prologue and epilogue scripts execution
509 # int walltimeAdditionalDelay = 240; additional delay
510 #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
512 # Put the duration in seconds first
513 #desired_walltime = duration * 60
514 desired_walltime = duration
515 total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
516 sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
518 #Put the walltime back in str form
520 walltime.append(str(total_walltime / 3600))
521 total_walltime = total_walltime - 3600 * int(walltime[0])
522 #Get the remaining minutes
523 walltime.append(str(total_walltime / 60))
524 total_walltime = total_walltime - 60 * int(walltime[1])
526 walltime.append(str(total_walltime))
529 logger.log_exc(" __process_walltime duration null")
531 return walltime, sleep_walltime
534 def _create_job_structure_request_for_OAR(lease_dict):
535 """ Creates the structure needed for a correct POST on OAR.
536 Makes the timestamp transformation into the appropriate format.
537 Sends the POST request to create the job with the resources in
546 reqdict['workdir'] = '/tmp'
547 reqdict['resource'] = "{network_address in ("
549 for node in lease_dict['added_nodes']:
550 logger.debug("\r\n \r\n OARrestapi \t \
551 __create_job_structure_request_for_OAR node %s" %(node))
553 # Get the ID of the node
555 reqdict['resource'] += "'" + nodeid + "', "
556 nodeid_list.append(nodeid)
558 custom_length = len(reqdict['resource'])- 2
559 reqdict['resource'] = reqdict['resource'][0:custom_length] + \
560 ")}/nodes=" + str(len(nodeid_list))
563 walltime, sleep_walltime = \
564 IotlabTestbedAPI._process_walltime(\
565 int(lease_dict['lease_duration']))
568 reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
569 ":" + str(walltime[1]) + ":" + str(walltime[2])
570 reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
572 #In case of a scheduled experiment (not immediate)
573 #To run an XP immediately, don't specify date and time in RSpec
574 #They will be set to None.
575 if lease_dict['lease_start_time'] is not '0':
576 #Readable time accepted by OAR
577 start_time = datetime.fromtimestamp( \
578 int(lease_dict['lease_start_time'])).\
579 strftime(lease_dict['time_format'])
580 reqdict['reservation'] = start_time
581 #If there is not start time, Immediate XP. No need to add special
585 reqdict['type'] = "deploy"
586 reqdict['directory'] = ""
587 reqdict['name'] = "SFA_" + lease_dict['slice_user']
592 def LaunchExperimentOnTestbed(self, added_nodes, slice_name, \
593 lease_start_time, lease_duration, slice_user=None):
596 Create an experiment request structure based on the information provided
597 and schedule/run the experiment on the testbed by reserving the nodes.
598 :param added_nodes: list of nodes that belong to the described lease.
599 :param slice_name: the slice hrn associated to the lease.
600 :param lease_start_time: timestamp of the lease startting time.
601 :param lease_duration: lease duration in minutes
605 # Add in the dict whatever is necessary to create the experiment on
607 lease_dict['lease_start_time'] = lease_start_time
608 lease_dict['lease_duration'] = lease_duration
609 lease_dict['added_nodes'] = added_nodes
610 lease_dict['slice_name'] = slice_name
611 lease_dict['slice_user'] = slice_user
612 lease_dict['grain'] = self.GetLeaseGranularity()
616 answer = self.query_sites.schedule_experiment(lease_dict)
618 experiment_id = answer['id']
620 logger.log_exc("CORTEXLAB_API \tLaunchExperimentOnTestbed \
621 Impossible to create xp %s " %(answer))
625 logger.debug("CORTEXLAB_API \tLaunchExperimentOnTestbed \
626 experiment_id %s added_nodes %s slice_user %s"
627 %(experiment_id, added_nodes, slice_user))
633 def AddLeases(self, hostname_list, slice_record,
634 lease_start_time, lease_duration):
636 """Creates an experiment on the testbed corresponding to the information
637 provided as parameters. Adds the experiment id and the slice hrn in the
638 lease table on the additional sfa database so that we are able to know
639 which slice has which nodes.
641 :param hostname_list: list of nodes' OAR hostnames.
642 :param slice_record: sfa slice record, must contain login and hrn.
643 :param lease_start_time: starting time , unix timestamp format
644 :param lease_duration: duration in minutes
646 :type hostname_list: list
647 :type slice_record: dict
648 :type lease_start_time: integer
649 :type lease_duration: integer
652 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \
653 slice_record %s lease_start_time %s lease_duration %s "\
654 %( hostname_list, slice_record , lease_start_time, \
657 username = slice_record['login']
659 experiment_id = self.LaunchExperimentOnTestbed(hostname_list, \
660 slice_record['hrn'], \
661 lease_start_time, lease_duration, \
664 datetime.fromtimestamp(int(lease_start_time)).\
665 strftime(self.time_format)
666 end_time = lease_start_time + lease_duration
669 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
670 %s %s %s "%(slice_record['hrn'], experiment_id, end_time))
673 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \
674 %(type(slice_record['hrn']), type(experiment_id),
677 testbed_xp_row = LeaseTableXP(slice_hrn=slice_record['hrn'],
678 experiment_id=experiment_id, end_time=end_time)
680 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases testbed_xp_row %s" \
682 self.cortexlab_leases_db.testbed_session.add(testbed_xp_row)
683 self.cortexlab_leases_db.testbed_session.commit()
685 logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \
690 def DeleteLeases(self, leases_id_list, slice_hrn):
693 Deletes several leases, based on their experiment ids and the slice
694 they are associated with. Uses DeleteOneLease to delete the
695 experiment on the testbed. Note that one slice can contain multiple
696 experiments, and in this
697 case all the experiments in the leases_id_list MUST belong to this
698 same slice, since there is only one slice hrn provided here.
700 :param leases_id_list: list of job ids that belong to the slice whose
701 slice hrn is provided.
702 :param slice_hrn: the slice hrn.
703 :type slice_hrn: string
705 .. warning:: Does not have a return value since there was no easy
706 way to handle failure when dealing with multiple job delete. Plus,
707 there was no easy way to report it to the user.
710 logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
711 \r\n " %(leases_id_list, slice_hrn))
712 for experiment_id in leases_id_list:
713 self.DeleteOneLease(experiment_id, slice_hrn)
717 #Delete the jobs from job_iotlab table
718 def DeleteSliceFromNodes(self, slice_record):
720 Deletes all the running or scheduled jobs of a given slice
723 :param slice_record: record of the slice, must contain experiment_id,
725 :type slice_record: dict
726 :returns: dict of the jobs'deletion status. Success= True, Failure=
727 False, for each job id.
730 .. note: used in driver delete_sliver
733 logger.debug("CORTEXLAB_API \t DeleteSliceFromNodes %s "
736 if isinstance(slice_record['experiment_id'], list):
737 experiment_bool_answer = {}
738 for experiment_id in slice_record['experiment_id']:
739 ret = self.DeleteOneLease(experiment_id, slice_record['user'])
741 experiment_bool_answer.update(ret)
744 experiment_bool_answer = [self.DeleteOneLease(
745 slice_record['experiment_id'],
746 slice_record['user'])]
748 return experiment_bool_answer
752 def GetLeaseGranularity(self):
753 """ Returns the granularity of an experiment in the Iotlab testbed.
754 OAR uses seconds for experiments duration , the granulaity is also
756 Experiments which last less than 10 min (600 sec) are invalid"""
761 # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql):
762 # """ Cleans the iotlab db by deleting expired and cancelled jobs.
763 # Compares the list of job ids given by OAR with the job ids that
764 # are already in the database, deletes the jobs that are no longer in
765 # the OAR job id list.
766 # :param job_oar_list: list of job ids coming from OAR
767 # :type job_oar_list: list
768 # :param job_psql: list of job ids cfrom the database.
769 # type job_psql: list
771 # #Turn the list into a set
772 # set_jobs_psql = set(jobs_psql)
774 # kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
775 # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \
776 # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
777 # deleted_jobs = set_jobs_psql.difference(kept_jobs)
778 # deleted_jobs = list(deleted_jobs)
779 # if len(deleted_jobs) > 0:
780 # self.cortexlab_leases_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
781 # self.cortexlab_leases_db.testbed_session.commit()
786 def filter_lease_name(reservation_list, filter_value):
787 filtered_reservation_list = list(reservation_list)
788 logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \
789 % (reservation_list))
790 for reservation in reservation_list:
791 if 'slice_hrn' in reservation and \
792 reservation['slice_hrn'] != filter_value:
793 filtered_reservation_list.remove(reservation)
795 logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \
796 % (filtered_reservation_list))
797 return filtered_reservation_list
800 def filter_lease_start_time(reservation_list, filter_value):
801 filtered_reservation_list = list(reservation_list)
803 for reservation in reservation_list:
804 if 't_from' in reservation and \
805 reservation['t_from'] > filter_value:
806 filtered_reservation_list.remove(reservation)
808 return filtered_reservation_list
810 def complete_leases_info(self, unfiltered_reservation_list, db_xp_dict):
812 """Check that the leases list of dictionaries contains the appropriate
813 fields and piece of information here
814 :param unfiltered_reservation_list: list of leases to be completed.
815 :param db_xp_dict: leases information in the lease_sfa table
816 :returns local_unfiltered_reservation_list: list of leases completed.
817 list of dictionaries describing the leases, with all the needed
818 information (sfa,ldap,nodes)to identify one particular lease.
819 :returns testbed_xp_list: list of experiments'ids running or scheduled
821 :rtype local_unfiltered_reservation_list: list of dict
822 :rtype testbed_xp_list: list
826 local_unfiltered_reservation_list = list(unfiltered_reservation_list)
827 # slice_hrn and lease_id are in the lease_table,
828 # so they are in the db_xp_dict.
829 # component_id_list : list of nodes xrns
830 # reserved_nodes : list of nodes' hostnames
831 # slice_id : slice urn, can be made from the slice hrn using hrn_to_urn
832 for resa in local_unfiltered_reservation_list:
834 #Construct list of scheduled experiments (runing, waiting..)
835 testbed_xp_list.append(resa['lease_id'])
836 #If there is information on the experiment in the lease table
837 #(slice used and experiment id), meaning the experiment was created
839 if resa['lease_id'] in db_xp_dict:
840 xp_info = db_xp_dict[resa['lease_id']]
841 logger.debug("CORTEXLAB_API \tGetLeases xp_info %s"
843 resa['slice_hrn'] = xp_info['slice_hrn']
844 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
846 #otherwise, assume it is a cortexlab slice, created via the
849 resa['slice_id'] = hrn_to_urn(self.root_auth + '.' +
850 resa['user'] + "_slice", 'slice')
851 resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
853 resa['component_id_list'] = []
854 #Transform the hostnames into urns (component ids)
855 for node in resa['reserved_nodes']:
857 iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
858 resa['component_id_list'].append(iotlab_xrn.urn)
860 return local_unfiltered_reservation_list, testbed_xp_list
862 def GetLeases(self, lease_filter_dict=None, login=None):
865 Get the list of leases from the testbed with complete information
866 about in which slice is running which experiment ans which nodes are
869 -Fetch all the experiments from the testbed (running, waiting..)
870 complete the reservation information with slice hrn
871 found in testbed_xp table. If not available in the table,
872 assume it is a cortexlab slice.
873 -Updates the cortexlab table, deleting jobs when necessary.
875 :returns: reservation_list, list of dictionaries with 'lease_id',
876 'reserved_nodes','slice_id','user', 'component_id_list',
877 'slice_hrn', 'resource_ids', 't_from', 't_until'. Other
878 keys can be returned if necessary, such as the 'state' of the lease,
879 if the information has been added in GetReservedNodes.
884 unfiltered_reservation_list = self.GetReservedNodes(login)
886 reservation_list = []
887 #Find the slice associated with this user ldap uid
888 logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\
889 unfiltered_reservation_list %s "
890 % (login, unfiltered_reservation_list))
891 #Create user dict first to avoid looking several times for
892 #the same user in LDAP SA 27/07/12
895 db_xp_query = self.cortexlab_leases_db.testbed_session.query(LeaseTableXP).all()
896 db_xp_dict = dict([(row.experiment_id, row.__dict__)
897 for row in db_xp_query])
899 logger.debug("CORTEXLAB_API \tGetLeases db_xp_dict %s"
901 db_xp_id_list = [row.experiment_id for row in db_xp_query]
903 required_fiels_in_leases = ['lease_id',
904 'reserved_nodes','slice_id', 'user', 'component_id_list',
905 'slice_hrn', 'resource_ids', 't_from', 't_until']
907 # Add any missing information on the leases with complete_leases_info
908 unfiltered_reservation_list, testbed_xp_list = \
909 self.complete_leases_info(unfiltered_reservation_list,
911 # Check that the list of leases is complete and have the mandatory
913 format_status = self.ensure_format_is_valid(unfiltered_reservation_list,
914 required_fiels_in_leases)
916 if not format_status:
917 logger.log_exc("\tCortexlabapi \t GetLeases : Missing fields in \
919 raise KeyError, "GetLeases : Missing fields in reservation list "
921 if lease_filter_dict:
922 logger.debug("CORTEXLAB_API \tGetLeases \
923 \r\n leasefilter %s" % ( lease_filter_dict))
925 filter_dict_functions = {
926 'slice_hrn' : CortexlabTestbedAPI.filter_lease_name,
927 't_from' : CortexlabTestbedAPI.filter_lease_start_time
930 reservation_list = list(unfiltered_reservation_list)
931 for filter_type in lease_filter_dict:
932 logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \
933 % (reservation_list))
934 reservation_list = filter_dict_functions[filter_type](\
935 reservation_list,lease_filter_dict[filter_type] )
938 if lease_filter_dict is None:
939 reservation_list = unfiltered_reservation_list
941 self.cortexlab_leases_db.update_experiments_in_additional_sfa_db(
942 testbed_xp_list, db_xp_id_list)
944 logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s"
945 % (reservation_list))
946 return reservation_list
951 #TODO FUNCTIONS SECTION 04/07/2012 SA
953 ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
956 #def UnBindObjectFromPeer( auth, object_type, object_id, shortname):
957 #""" This method is a hopefully temporary hack to let the sfa correctly
958 #detach the objects it creates from a remote peer object. This is
959 #needed so that the sfa federation link can work in parallel with
960 #RefreshPeer, as RefreshPeer depends on remote objects being correctly
963 #auth : struct, API authentication structure
964 #AuthMethod : string, Authentication method to use
965 #object_type : string, Object type, among 'site','person','slice',
967 #object_id : int, object_id
968 #shortname : string, peer shortname
972 #logger.warning("CORTEXLAB_API \tUnBindObjectFromPeer EMPTY-\
976 ##TODO Is BindObjectToPeer still necessary ? Currently does nothing
978 #|| Commented out 28/05/13 SA
979 #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
980 #remote_object_id=None):
981 #"""This method is a hopefully temporary hack to let the sfa correctly
982 #attach the objects it creates to a remote peer object. This is needed
983 #so that the sfa federation link can work in parallel with RefreshPeer,
984 #as RefreshPeer depends on remote objects being correctly marked.
986 #shortname : string, peer shortname
987 #remote_object_id : int, remote object_id, set to 0 if unknown
991 #logger.warning("CORTEXLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
994 ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA
995 ##Funciton should delete and create another job since oin iotlab slice=job
996 #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
997 #"""Updates the parameters of an existing slice with the values in
999 #Users may only update slices of which they are members.
1000 #PIs may update any of the slices at their sites, or any slices of
1001 #which they are members. Admins may update any slice.
1002 #Only PIs and admins may update max_nodes. Slices cannot be renewed
1003 #(by updating the expires parameter) more than 8 weeks into the future.
1004 #Returns 1 if successful, faults otherwise.
1008 #logger.warning("CORTEXLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ")
1011 #Unused SA 30/05/13, we only update the user's key or we delete it.
1012 ##TODO UpdatePerson 04/07/2012 SA
1013 #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None):
1014 #"""Updates a person. Only the fields specified in person_fields
1015 #are updated, all other fields are left untouched.
1016 #Users and techs can only update themselves. PIs can only update
1017 #themselves and other non-PIs at their sites.
1018 #Returns 1 if successful, faults otherwise.
1022 ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
1023 ##self.cortexlab_leases_db.testbed_session.add(new_row)
1024 ##self.cortexlab_leases_db.testbed_session.commit()
1026 #logger.debug("CORTEXLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
1030 def GetKeys(key_filter=None):
1031 """Returns a dict of dict based on the key string. Each dict entry
1032 contains the key id, the ssh key, the user's email and the
1034 If key_filter is specified and is an array of key identifiers,
1035 only keys matching the filter will be returned.
1037 Admin may query all keys. Non-admins may only query their own keys.
1040 :returns: dict with ssh key as key and dicts as value.
1043 if key_filter is None:
1044 keys = dbsession.query(RegKey).options(joinedload('reg_user')).all()
1046 keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all()
1050 key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
1051 'email': key.reg_user.email,
1052 'hrn': key.reg_user.hrn}
1054 #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
1055 #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
1056 #for user in ldap_rslt)
1058 logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
1062 def DeleteKey(self, user_record, key_string):
1063 """Deletes a key in the LDAP entry of the specified user.
1065 Removes the key_string from the user's key list and updates the LDAP
1066 user's entry with the new key attributes.
1068 :param key_string: The ssh key to remove
1069 :param user_record: User's record
1070 :type key_string: string
1071 :type user_record: dict
1072 :returns: True if sucessful, False if not.
1076 all_user_keys = user_record['keys']
1077 all_user_keys.remove(key_string)
1078 new_attributes = {'sshPublicKey':all_user_keys}
1079 ret = self.ldap.LdapModifyUser(user_record, new_attributes)
1080 logger.debug("CORTEXLAB_API DeleteKey %s- " % (ret))
1087 def _sql_get_slice_info(slice_filter):
1089 Get the slice record based on the slice hrn. Fetch the record of the
1090 user associated with the slice by using joinedload based on the
1091 reg_researcher relationship.
1093 :param slice_filter: the slice hrn we are looking for
1094 :type slice_filter: string
1095 :returns: the slice record enhanced with the user's information if the
1096 slice was found, None it wasn't.
1098 :rtype: dict or None.
1100 #DO NOT USE RegSlice - reg_researchers to get the hrn
1101 #of the user otherwise will mess up the RegRecord in
1102 #Resolve, don't know why - SA 08/08/2012
1104 #Only one entry for one user = one slice in testbed_xp table
1105 #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1106 raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
1107 #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1109 #load_reg_researcher
1110 #raw_slicerec.reg_researchers
1111 raw_slicerec = raw_slicerec.__dict__
1112 logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \
1113 raw_slicerec %s" % (slice_filter, raw_slicerec))
1114 slicerec = raw_slicerec
1115 #only one researcher per slice so take the first one
1116 #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
1117 #del slicerec['reg_researchers']['_sa_instance_state']
1124 def _sql_get_slice_info_from_user(slice_filter):
1126 Get the slice record based on the user recordid by using a joinedload
1127 on the relationship reg_slices_as_researcher. Format the sql record
1128 into a dict with the mandatory fields for user and slice.
1129 :returns: dict with slice record and user record if the record was found
1130 based on the user's id, None if not..
1131 :rtype:dict or None..
1133 #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1134 raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
1135 #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1136 #Put it in correct order
1137 user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1138 'classtype', 'authority', 'gid', 'record_id',
1139 'date_created', 'type', 'email', 'pointer']
1140 slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1141 'classtype', 'authority', 'gid', 'record_id',
1142 'date_created', 'type', 'pointer']
1144 #raw_slicerec.reg_slices_as_researcher
1145 raw_slicerec = raw_slicerec.__dict__
1148 dict([(k, raw_slicerec[
1149 'reg_slices_as_researcher'][0].__dict__[k])
1150 for k in slice_needed_fields])
1151 slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
1152 for k in user_needed_fields])
1153 #TODO Handle multiple slices for one user SA 10/12/12
1154 #for now only take the first slice record associated to the rec user
1155 ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
1156 #del raw_slicerec['reg_slices_as_researcher']
1157 #slicerec['reg_researchers'] = raw_slicerec
1158 ##del slicerec['_sa_instance_state']
1165 def _get_slice_records(self, slice_filter=None,
1166 slice_filter_type=None):
1168 Get the slice record depending on the slice filter and its type.
1169 :param slice_filter: Can be either the slice hrn or the user's record
1171 :type slice_filter: string
1172 :param slice_filter_type: describes the slice filter type used, can be
1173 slice_hrn or record_id_user
1175 :returns: the slice record
1177 .. seealso::_sql_get_slice_info_from_user
1178 .. seealso:: _sql_get_slice_info
1181 #Get list of slices based on the slice hrn
1182 if slice_filter_type == 'slice_hrn':
1184 #if get_authority(slice_filter) == self.root_auth:
1185 #login = slice_filter.split(".")[1].split("_")[0]
1187 slicerec = self._sql_get_slice_info(slice_filter)
1189 if slicerec is None:
1193 #Get slice based on user id
1194 if slice_filter_type == 'record_id_user':
1196 slicerec = self._sql_get_slice_info_from_user(slice_filter)
1199 fixed_slicerec_dict = slicerec
1200 #At this point if there is no login it means
1201 #record_id_user filter has been used for filtering
1203 ##If theslice record is from iotlab
1204 #if fixed_slicerec_dict['peer_authority'] is None:
1205 #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
1206 #return login, fixed_slicerec_dict
1207 return fixed_slicerec_dict
1212 def GetSlices(self, slice_filter=None, slice_filter_type=None,
1214 """Get the slice records from the sfa db and add lease information
1217 :param slice_filter: can be the slice hrn or slice record id in the db
1218 depending on the slice_filter_type.
1219 :param slice_filter_type: defines the type of the filtering used, Can be
1220 either 'slice_hrn' or 'record_id'.
1221 :type slice_filter: string
1222 :type slice_filter_type: string
1223 :returns: a slice dict if slice_filter and slice_filter_type
1224 are specified and a matching entry is found in the db. The result
1225 is put into a list.Or a list of slice dictionnaries if no filters
1232 authorized_filter_types_list = ['slice_hrn', 'record_id_user']
1233 return_slicerec_dictlist = []
1235 #First try to get information on the slice based on the filter provided
1236 if slice_filter_type in authorized_filter_types_list:
1237 fixed_slicerec_dict = self._get_slice_records(slice_filter,
1239 # if the slice was not found in the sfa db
1240 if fixed_slicerec_dict is None:
1241 return return_slicerec_dictlist
1243 slice_hrn = fixed_slicerec_dict['hrn']
1245 logger.debug(" CORTEXLAB_API \tGetSlices login %s \
1246 slice record %s slice_filter %s \
1247 slice_filter_type %s " % (login,
1248 fixed_slicerec_dict, slice_filter,
1252 #Now we have the slice record fixed_slicerec_dict, get the
1253 #jobs associated to this slice
1256 leases_list = self.GetLeases(login=login)
1257 #If no job is running or no job scheduled
1258 #return only the slice record
1259 if leases_list == [] and fixed_slicerec_dict:
1260 return_slicerec_dictlist.append(fixed_slicerec_dict)
1262 # if the jobs running don't belong to the user/slice we are looking
1264 leases_hrn = [lease['slice_hrn'] for lease in leases_list]
1265 if slice_hrn not in leases_hrn:
1266 return_slicerec_dictlist.append(fixed_slicerec_dict)
1267 #If several experiments for one slice , put the slice record into
1268 # each lease information dict
1269 for lease in leases_list:
1271 logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \
1272 \t lease['slice_hrn'] %s"
1273 % (slice_filter, lease['slice_hrn']))
1274 if lease['slice_hrn'] == slice_hrn:
1275 slicerec_dict['experiment_id'] = lease['lease_id']
1276 #Update lease dict with the slice record
1277 if fixed_slicerec_dict:
1278 fixed_slicerec_dict['experiment_id'] = []
1279 fixed_slicerec_dict['experiment_id'].append(
1280 slicerec_dict['experiment_id'])
1281 slicerec_dict.update(fixed_slicerec_dict)
1283 slicerec_dict['slice_hrn'] = lease['slice_hrn']
1284 slicerec_dict['hrn'] = lease['slice_hrn']
1285 slicerec_dict['user'] = lease['user']
1286 slicerec_dict.update(
1288 {'hostname': lease['reserved_nodes']}})
1289 slicerec_dict.update({'node_ids': lease['reserved_nodes']})
1292 return_slicerec_dictlist.append(slicerec_dict)
1295 logger.debug("CORTEXLAB_API.PY \tGetSlices \
1296 slicerec_dict %s return_slicerec_dictlist %s \
1297 lease['reserved_nodes'] \
1298 %s" % (slicerec_dict, return_slicerec_dictlist,
1299 lease['reserved_nodes']))
1301 logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
1302 return_slicerec_dictlist %s"
1303 % (return_slicerec_dictlist))
1305 return return_slicerec_dictlist
1309 #Get all slices from the cortexlab sfa database , get the user info
1310 # as well at the same time put them in dict format
1312 query_slice_list = \
1313 dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
1315 for record in query_slice_list:
1316 tmp = record.__dict__
1317 tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
1318 return_slicerec_dictlist.append(tmp)
1321 #Get all the experiments reserved nodes
1322 leases_list = self.GetReservedNodes()
1324 for fixed_slicerec_dict in return_slicerec_dictlist:
1326 #Check if the slice belongs to a cortexlab user
1327 if fixed_slicerec_dict['peer_authority'] is None:
1328 owner = fixed_slicerec_dict['hrn'].split(
1329 ".")[1].split("_")[0]
1332 for lease in leases_list:
1333 if owner == lease['user']:
1334 slicerec_dict['experiment_id'] = lease['lease_id']
1336 #for reserved_node in lease['reserved_nodes']:
1337 logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s "
1339 slicerec_dict.update(fixed_slicerec_dict)
1340 slicerec_dict.update({'node_ids':
1341 lease['reserved_nodes']})
1342 slicerec_dict.update({'list_node_ids':
1344 lease['reserved_nodes']}})
1347 fixed_slicerec_dict.update(slicerec_dict)
1349 logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
1350 return_slicerec_dictlist %s \slice_filter %s " \
1351 %(return_slicerec_dictlist, slice_filter))
1353 return return_slicerec_dictlist
1357 #Update slice unused, therefore sfa_fields_to_iotlab_fields unused
1360 #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record):
1365 ##for field in record:
1366 ## iotlab_record[field] = record[field]
1368 #if sfa_type == "slice":
1369 ##instantion used in get_slivers ?
1370 #if not "instantiation" in iotlab_record:
1371 #iotlab_record["instantiation"] = "iotlab-instantiated"
1372 ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn)
1373 ##Unused hrn_to_pl_slicename because Iotlab's hrn already
1374 ##in the appropriate form SA 23/07/12
1375 #iotlab_record["hrn"] = hrn
1376 #logger.debug("CORTEXLAB_API.PY sfa_fields_to_iotlab_fields \
1377 #iotlab_record %s " %(iotlab_record['hrn']))
1378 #if "url" in record:
1379 #iotlab_record["url"] = record["url"]
1380 #if "description" in record:
1381 #iotlab_record["description"] = record["description"]
1382 #if "expires" in record:
1383 #iotlab_record["expires"] = int(record["expires"])
1385 ##nodes added by OAR only and then imported to SFA
1386 ##elif type == "node":
1387 ##if not "hostname" in iotlab_record:
1388 ##if not "hostname" in record:
1389 ##raise MissingSfaInfo("hostname")
1390 ##iotlab_record["hostname"] = record["hostname"]
1391 ##if not "model" in iotlab_record:
1392 ##iotlab_record["model"] = "geni"
1394 ##One authority only
1395 ##elif type == "authority":
1396 ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn)
1398 ##if not "name" in iotlab_record:
1399 ##iotlab_record["name"] = hrn
1401 ##if not "abbreviated_name" in iotlab_record:
1402 ##iotlab_record["abbreviated_name"] = hrn
1404 ##if not "enabled" in iotlab_record:
1405 ##iotlab_record["enabled"] = True
1407 ##if not "is_public" in iotlab_record:
1408 ##iotlab_record["is_public"] = True
1410 #return iotlab_record