2 File containing the CortexlabShell, used to interact with nodes, users,
3 slices, leases and keys, as well as the dedicated iotlab database and table,
4 holding information about which slice is running which job.
7 from datetime import datetime
9 from sfa.util.sfalogging import logger
11 from sfa.storage.alchemy import dbsession
12 from sqlalchemy.orm import joinedload
13 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
15 from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP
16 from sfa.cortexlab.LDAPapi import LDAPapi
18 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
20 from sfa.trust.certificate import Keypair, convert_public_key
21 from sfa.trust.gid import create_uuid
22 from sfa.trust.hierarchy import Hierarchy
24 from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
25 from sfa.cortexlab.cortexlabnodes import CortexlabQueryNodes
27 class CortexlabShell():
28 """ Class enabled to use LDAP and OAR api calls. """
30 _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins
32 def __init__(self, config):
33 """Creates an instance of OARrestapi and LDAPapi which will be used to
34 issue calls to OAR or LDAP methods.
35 Set the time format and the testbed granularity used for OAR
36 reservation and leases.
38 :param config: configuration object from sfa.util.config
39 :type config: Config object
41 self.leases_db = TestbedAdditionalSfaDB(config)
42 self.query_sites = CortexlabQueryNodes()
44 self.time_format = "%Y-%m-%d %H:%M:%S"
45 self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
46 self.grain = 60 # 10 mins lease minimum, 60 sec granularity
47 #import logging, logging.handlers
48 #from sfa.util.sfalogging import _SfaLogger
49 #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \
54 def GetMinExperimentDurationInGranularity():
55 """ Returns the minimum allowed duration for an experiment on the
59 return CortexlabShell._MINIMUM_DURATION
62 def GetPeers(self, peer_filter=None ):
63 """ Gathers registered authorities in SFA DB and looks for specific peer
64 if peer_filter is specified.
65 :param peer_filter: name of the site authority looked for.
66 :type peer_filter: string
67 :returns: list of records.
72 existing_hrns_by_types = {}
73 logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter))
74 all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
76 for record in all_records:
77 existing_records[(record.hrn, record.type)] = record
78 if record.type not in existing_hrns_by_types:
79 existing_hrns_by_types[record.type] = [record.hrn]
81 existing_hrns_by_types[record.type].append(record.hrn)
83 logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s "
84 % (existing_hrns_by_types))
89 records_list.append(existing_records[(peer_filter,
92 for hrn in existing_hrns_by_types['authority']:
93 records_list.append(existing_records[(hrn, 'authority')])
95 logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s "
101 return_records = records_list
102 logger.debug("CORTEXLAB_API \tGetPeer return_records %s "
104 return return_records
106 #TODO : Handling OR request in make_ldap_filters_from_records
107 #instead of the for loop
108 #over the records' list
109 def GetPersons(self, person_filter=None):
111 Get the enabled users and their properties from Cortexlab LDAP.
112 If a filter is specified, looks for the user whose properties match
113 the filter, otherwise returns the whole enabled users'list.
115 :param person_filter: Must be a list of dictionnaries with users
116 properties when not set to None.
117 :type person_filter: list of dict
119 :returns: Returns a list of users whose accounts are enabled
121 :rtype: list of dicts
124 logger.debug("CORTEXLAB_API \tGetPersons person_filter %s"
127 if person_filter and isinstance(person_filter, list):
128 #If we are looking for a list of users (list of dict records)
129 #Usually the list contains only one user record
130 for searched_attributes in person_filter:
132 #Get only enabled user accounts in iotlab LDAP :
133 #add a filter for make_ldap_filters_from_record
134 person = self.ldap.LdapFindUser(searched_attributes,
135 is_user_enabled=True)
136 #If a person was found, append it to the list
138 person_list.append(person)
140 #If the list is empty, return None
141 if len(person_list) is 0:
145 #Get only enabled user accounts in iotlab LDAP :
146 #add a filter for make_ldap_filters_from_record
147 person_list = self.ldap.LdapFindUser(is_user_enabled=True)
153 def DeleteOneLease(self, lease_id, username):
156 Deletes the lease with the specified lease_id and username on OAR by
157 posting a delete request to OAR.
159 :param lease_id: Reservation identifier.
160 :param username: user's iotlab login in LDAP.
161 :type lease_id: Depends on what tou are using, could be integer or
163 :type username: string
165 :returns: dictionary with the lease id and if delete has been successful
171 # Here delete the lease specified
172 answer = self.query_sites.delete_experiment(lease_id, username)
174 # If the username is not necessary to delete the lease, then you can
175 # remove it from the parameters, given that you propagate the changes
176 # Return delete status so that you know if the delete has been
180 if answer['status'] is True:
181 ret = {lease_id: True}
183 ret = {lease_id: False}
184 logger.debug("CORTEXLAB_API \DeleteOneLease lease_id %s \r\n answer %s \
185 username %s" % (lease_id, answer, username))
190 def GetNodesCurrentlyInUse(self):
191 """Returns a list of all the nodes involved in a currently running
192 experiment (and only the one not available at the moment the call to
193 this method is issued)
194 :rtype: list of nodes hostnames.
196 node_hostnames_list = []
197 return node_hostnames_list
200 def GetReservedNodes(self, username=None):
201 """ Get list of leases. Get the leases for the username if specified,
202 otherwise get all the leases. Finds the nodes hostnames for each
204 :param username: user's LDAP login
205 :type username: string
206 :returns: list of reservations dict
210 #Get the nodes in use and the reserved nodes
211 mandatory_sfa_keys = ['reserved_nodes','lease_id']
212 reservation_dict_list = \
213 self.query_sites.get_reserved_nodes(username = username)
215 if len(reservation_dict_list) == 0:
219 # Ensure mandatory keys are in the dict
220 if not self.ensure_format_is_valid(reservation_dict_list,
222 raise KeyError, "GetReservedNodes : Missing SFA mandatory keys"
225 return reservation_dict_list
228 def ensure_format_is_valid(list_dictionary_to_check, mandatory_keys_list):
229 for entry in list_dictionary_to_check:
230 if not all (key in entry for key in mandatory_keys_list):
234 def GetNodes(self, node_filter_dict=None, return_fields_list=None):
237 Make a list of cortexlab nodes and their properties from information
238 given by ?. Search for specific nodes if some filters are
239 specified. Nodes properties returned if no return_fields_list given:
240 'hrn','archi','mobile','hostname','site','boot_state','node_id',
241 'radio','posx','posy,'posz'.
243 :param node_filter_dict: dictionnary of lists with node properties. For
244 instance, if you want to look for a specific node with its hrn,
245 the node_filter_dict should be {'hrn': [hrn_of_the_node]}
246 :type node_filter_dict: dict
247 :param return_fields_list: list of specific fields the user wants to be
249 :type return_fields_list: list
250 :returns: list of dictionaries with node properties. Mandatory
251 properties hrn, site, hostname. Complete list (iotlab) ['hrn',
252 'archi', 'mobile', 'hostname', 'site', 'mobility_type',
253 'boot_state', 'node_id','radio', 'posx', 'posy', 'oar_id', 'posz']
254 Radio, archi, mobile and position are useful to help users choose
255 the appropriate nodes.
258 :TODO: FILL IN THE BLANKS
261 # Here get full dict of nodes with all their properties.
262 mandatory_sfa_keys = ['hrn', 'site', 'hostname']
263 node_list_dict = self.query_sites.get_all_nodes(node_filter_dict,
266 if len(node_list_dict) == 0:
267 return_node_list = []
270 # Ensure mandatory keys are in the dict
271 if not self.ensure_format_is_valid(node_list_dict,
273 raise KeyError, "GetNodes : Missing SFA mandatory keys"
276 return_node_list = node_list_dict
277 return return_node_list
279 def AddSlice(self, slice_record, user_record):
282 Add slice to the local cortexlab sfa tables if the slice comes
283 from a federated site and is not yet in the cortexlab sfa DB,
284 although the user has already a LDAP login.
285 Called by verify_slice during lease/sliver creation.
287 :param slice_record: record of slice, must contain hrn, gid, slice_id
288 and authority of the slice.
289 :type slice_record: dictionary
290 :param user_record: record of the user
291 :type user_record: RegUser
295 sfa_record = RegSlice(hrn=slice_record['hrn'],
296 gid=slice_record['gid'],
297 pointer=slice_record['slice_id'],
298 authority=slice_record['authority'])
299 logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s"
300 % (sfa_record, user_record))
301 sfa_record.just_created()
302 dbsession.add(sfa_record)
304 #Update the reg-researcher dependance table
305 sfa_record.reg_researchers = [user_record]
311 def GetSites(self, site_filter_name_list=None, return_fields_list=None):
312 """Returns the list of Cortexlab's sites with the associated nodes and
313 the sites' properties as dictionaries. Used in import.
316 ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids',
317 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids',
318 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created',
319 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name']
320 can be empty ( []): address_ids, slice_ids, pcu_ids, person_ids,
323 :param site_filter_name_list: used to specify specific sites
324 :param return_fields_list: field that has to be returned
325 :type site_filter_name_list: list
326 :type return_fields_list: list
327 :rtype: list of dicts
330 site_list_dict = self.query_sites.get_sites(site_filter_name_list,
333 mandatory_sfa_keys = ['name', 'node_ids', 'longitude','site' ]
335 if len(site_list_dict) == 0:
336 return_site_list = []
339 # Ensure mandatory keys are in the dict
340 if not self.ensure_format_is_valid(site_list_dict,
342 raise KeyError, "GetSites : Missing sfa mandatory keys"
344 return_site_list = site_list_dict
345 return return_site_list
348 #TODO : Check rights to delete person
349 def DeletePerson(self, person_record):
350 """Disable an existing account in cortexlab LDAP.
352 Users and techs can only delete themselves. PIs can only
353 delete themselves and other non-PIs at their sites.
354 ins can delete anyone.
356 :param person_record: user's record
357 :type person_record: dict
358 :returns: True if successful, False otherwise.
361 .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF.
363 #Disable user account in iotlab LDAP
364 ret = self.ldap.LdapMarkUserAsDeleted(person_record)
365 logger.warning("CORTEXLAB_API DeletePerson %s " % (person_record))
368 def DeleteSlice(self, slice_record):
369 """Deletes the specified slice and kills the jobs associated with
370 the slice if any, using DeleteSliceFromNodes.
372 :param slice_record: record of the slice, must contain experiment_id, user
373 :type slice_record: dict
374 :returns: True if all the jobs in the slice have been deleted,
375 or the list of jobs that could not be deleted otherwise.
376 :rtype: list or boolean
378 .. seealso:: DeleteSliceFromNodes
381 ret = self.DeleteSliceFromNodes(slice_record)
383 for experiment_id in ret:
384 if False in ret[experiment_id]:
385 if delete_failed is None:
387 delete_failed.append(experiment_id)
389 logger.info("CORTEXLAB_API DeleteSlice %s answer %s"%(slice_record, \
391 return delete_failed or True
394 def __add_person_to_db(self, user_dict):
396 Add a federated user straight to db when the user issues a lease
397 request with iotlab nodes and that he has not registered with cortexlab
398 yet (that is he does not have a LDAP entry yet).
399 Uses parts of the routines in CortexlabImport when importing user
401 Called by AddPerson, right after LdapAddUser.
402 :param user_dict: Must contain email, hrn and pkey to get a GID
403 and be added to the SFA db.
404 :type user_dict: dict
408 dbsession.query(RegUser).filter_by(email = user_dict['email']).first()
410 if not check_if_exists:
411 logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
413 hrn = user_dict['hrn']
414 person_urn = hrn_to_urn(hrn, 'user')
415 pubkey = user_dict['pkey']
417 pkey = convert_public_key(pubkey)
419 #key not good. create another pkey
420 logger.warn('__add_person_to_db: unable to convert public \
422 pkey = Keypair(create=True)
425 if pubkey is not None and pkey is not None :
426 hierarchy = Hierarchy()
427 person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
429 if user_dict['email']:
430 logger.debug("__add_person_to_db \r\n \r\n \
431 IOTLAB IMPORTER PERSON EMAIL OK email %s "\
432 %(user_dict['email']))
433 person_gid.set_email(user_dict['email'])
435 user_record = RegUser(hrn=hrn , pointer= '-1', \
436 authority=get_authority(hrn), \
437 email=user_dict['email'], gid = person_gid)
438 user_record.reg_keys = [RegKey(user_dict['pkey'])]
439 user_record.just_created()
440 dbsession.add (user_record)
445 def AddPerson(self, record):
448 Adds a new account. Any fields specified in records are used,
449 otherwise defaults are used. Creates an appropriate login by calling
452 :param record: dictionary with the sfa user's properties.
453 :returns: a dicitonary with the status. If successful, the dictionary
454 boolean is set to True and there is a 'uid' key with the new login
455 added to LDAP, otherwise the bool is set to False and a key
456 'message' is in the dictionary, with the error message.
460 ret = self.ldap.LdapAddUser(record)
462 if ret['bool'] is True:
463 record['hrn'] = self.root_auth + '.' + ret['uid']
464 logger.debug("CORTEXLAB_API AddPerson return code %s record %s "
466 self.__add_person_to_db(record)
473 #TODO AddPersonKey 04/07/2012 SA
474 def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
475 """Adds a new key to the specified account. Adds the key to the
476 iotlab ldap, provided that the person_uid is valid.
478 Non-admins can only modify their own keys.
480 :param person_uid: user's iotlab login in LDAP
481 :param old_attributes_dict: dict with the user's old sshPublicKey
482 :param new_key_dict: dict with the user's new sshPublicKey
483 :type person_uid: string
487 :returns: True if the key has been modified, False otherwise.
490 ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \
492 logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ")
497 def _process_walltime(duration):
498 """ Calculates the walltime in seconds from the duration in H:M:S
499 specified in the RSpec.
503 # Fixing the walltime by adding a few delays.
504 # First put the walltime in seconds oarAdditionalDelay = 20;
505 # additional delay for /bin/sleep command to
506 # take in account prologue and epilogue scripts execution
507 # int walltimeAdditionalDelay = 240; additional delay
508 #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
510 # Put the duration in seconds first
511 #desired_walltime = duration * 60
512 desired_walltime = duration
513 total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
514 sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
516 #Put the walltime back in str form
518 walltime.append(str(total_walltime / 3600))
519 total_walltime = total_walltime - 3600 * int(walltime[0])
520 #Get the remaining minutes
521 walltime.append(str(total_walltime / 60))
522 total_walltime = total_walltime - 60 * int(walltime[1])
524 walltime.append(str(total_walltime))
527 logger.log_exc(" __process_walltime duration null")
529 return walltime, sleep_walltime
532 def _create_job_structure_request_for_OAR(lease_dict):
533 """ Creates the structure needed for a correct POST on OAR.
534 Makes the timestamp transformation into the appropriate format.
535 Sends the POST request to create the job with the resources in
544 reqdict['workdir'] = '/tmp'
545 reqdict['resource'] = "{network_address in ("
547 for node in lease_dict['added_nodes']:
548 logger.debug("\r\n \r\n OARrestapi \t \
549 __create_job_structure_request_for_OAR node %s" %(node))
551 # Get the ID of the node
553 reqdict['resource'] += "'" + nodeid + "', "
554 nodeid_list.append(nodeid)
556 custom_length = len(reqdict['resource'])- 2
557 reqdict['resource'] = reqdict['resource'][0:custom_length] + \
558 ")}/nodes=" + str(len(nodeid_list))
561 walltime, sleep_walltime = \
562 CortexlabShell._process_walltime(\
563 int(lease_dict['lease_duration']))
566 reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
567 ":" + str(walltime[1]) + ":" + str(walltime[2])
568 reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
570 #In case of a scheduled experiment (not immediate)
571 #To run an XP immediately, don't specify date and time in RSpec
572 #They will be set to None.
573 if lease_dict['lease_start_time'] is not '0':
574 #Readable time accepted by OAR
575 start_time = datetime.fromtimestamp( \
576 int(lease_dict['lease_start_time'])).\
577 strftime(lease_dict['time_format'])
578 reqdict['reservation'] = start_time
579 #If there is not start time, Immediate XP. No need to add special
583 reqdict['type'] = "deploy"
584 reqdict['directory'] = ""
585 reqdict['name'] = "SFA_" + lease_dict['slice_user']
590 def LaunchExperimentOnTestbed(self, added_nodes, slice_name, \
591 lease_start_time, lease_duration, slice_user=None):
594 Create an experiment request structure based on the information provided
595 and schedule/run the experiment on the testbed by reserving the nodes.
596 :param added_nodes: list of nodes that belong to the described lease.
597 :param slice_name: the slice hrn associated to the lease.
598 :param lease_start_time: timestamp of the lease startting time.
599 :param lease_duration: lease duration in minutes
603 # Add in the dict whatever is necessary to create the experiment on
605 lease_dict['lease_start_time'] = lease_start_time
606 lease_dict['lease_duration'] = lease_duration
607 lease_dict['added_nodes'] = added_nodes
608 lease_dict['slice_name'] = slice_name
609 lease_dict['slice_user'] = slice_user
610 lease_dict['grain'] = self.GetLeaseGranularity()
614 answer = self.query_sites.schedule_experiment(lease_dict)
616 experiment_id = answer['id']
618 logger.log_exc("CORTEXLAB_API \tLaunchExperimentOnTestbed \
619 Impossible to create xp %s " %(answer))
623 logger.debug("CORTEXLAB_API \tLaunchExperimentOnTestbed \
624 experiment_id %s added_nodes %s slice_user %s"
625 %(experiment_id, added_nodes, slice_user))
631 def AddLeases(self, hostname_list, slice_record,
632 lease_start_time, lease_duration):
634 """Creates an experiment on the testbed corresponding to the information
635 provided as parameters. Adds the experiment id and the slice hrn in the
636 lease table on the additional sfa database so that we are able to know
637 which slice has which nodes.
639 :param hostname_list: list of nodes' OAR hostnames.
640 :param slice_record: sfa slice record, must contain login and hrn.
641 :param lease_start_time: starting time , unix timestamp format
642 :param lease_duration: duration in minutes
644 :type hostname_list: list
645 :type slice_record: dict
646 :type lease_start_time: integer
647 :type lease_duration: integer
650 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \
651 slice_record %s lease_start_time %s lease_duration %s "\
652 %( hostname_list, slice_record , lease_start_time, \
655 username = slice_record['login']
657 experiment_id = self.LaunchExperimentOnTestbed(hostname_list, \
658 slice_record['hrn'], \
659 lease_start_time, lease_duration, \
662 datetime.fromtimestamp(int(lease_start_time)).\
663 strftime(self.time_format)
664 end_time = lease_start_time + lease_duration
667 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
668 %s %s %s "%(slice_record['hrn'], experiment_id, end_time))
671 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \
672 %(type(slice_record['hrn']), type(experiment_id),
675 testbed_xp_row = LeaseTableXP(slice_hrn=slice_record['hrn'],
676 experiment_id=experiment_id, end_time=end_time)
678 logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases testbed_xp_row %s" \
680 self.leases_db.testbed_session.add(testbed_xp_row)
681 self.leases_db.testbed_session.commit()
683 logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \
688 def DeleteLeases(self, leases_id_list, slice_hrn):
691 Deletes several leases, based on their experiment ids and the slice
692 they are associated with. Uses DeleteOneLease to delete the
693 experiment on the testbed. Note that one slice can contain multiple
694 experiments, and in this
695 case all the experiments in the leases_id_list MUST belong to this
696 same slice, since there is only one slice hrn provided here.
698 :param leases_id_list: list of job ids that belong to the slice whose
699 slice hrn is provided.
700 :param slice_hrn: the slice hrn.
701 :type slice_hrn: string
703 .. warning:: Does not have a return value since there was no easy
704 way to handle failure when dealing with multiple job delete. Plus,
705 there was no easy way to report it to the user.
708 logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
709 \r\n " %(leases_id_list, slice_hrn))
710 for experiment_id in leases_id_list:
711 self.DeleteOneLease(experiment_id, slice_hrn)
715 #Delete the jobs from job_iotlab table
716 def DeleteSliceFromNodes(self, slice_record):
718 Deletes all the running or scheduled jobs of a given slice
721 :param slice_record: record of the slice, must contain experiment_id,
723 :type slice_record: dict
724 :returns: dict of the jobs'deletion status. Success= True, Failure=
725 False, for each job id.
728 .. note: used in driver delete_sliver
731 logger.debug("CORTEXLAB_API \t DeleteSliceFromNodes %s "
734 if isinstance(slice_record['experiment_id'], list):
735 experiment_bool_answer = {}
736 for experiment_id in slice_record['experiment_id']:
737 ret = self.DeleteOneLease(experiment_id, slice_record['user'])
739 experiment_bool_answer.update(ret)
742 experiment_bool_answer = [self.DeleteOneLease(
743 slice_record['experiment_id'],
744 slice_record['user'])]
746 return experiment_bool_answer
750 def GetLeaseGranularity(self):
751 """ Returns the granularity of an experiment in the Iotlab testbed.
752 OAR uses seconds for experiments duration , the granulaity is also
754 Experiments which last less than 10 min (600 sec) are invalid"""
759 # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql):
760 # """ Cleans the iotlab db by deleting expired and cancelled jobs.
761 # Compares the list of job ids given by OAR with the job ids that
762 # are already in the database, deletes the jobs that are no longer in
763 # the OAR job id list.
764 # :param job_oar_list: list of job ids coming from OAR
765 # :type job_oar_list: list
766 # :param job_psql: list of job ids cfrom the database.
767 # type job_psql: list
769 # #Turn the list into a set
770 # set_jobs_psql = set(jobs_psql)
772 # kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
773 # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \
774 # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
775 # deleted_jobs = set_jobs_psql.difference(kept_jobs)
776 # deleted_jobs = list(deleted_jobs)
777 # if len(deleted_jobs) > 0:
778 # self.leases_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
779 # self.leases_db.testbed_session.commit()
784 def filter_lease_name(reservation_list, filter_value):
785 filtered_reservation_list = list(reservation_list)
786 logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \
787 % (reservation_list))
788 for reservation in reservation_list:
789 if 'slice_hrn' in reservation and \
790 reservation['slice_hrn'] != filter_value:
791 filtered_reservation_list.remove(reservation)
793 logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \
794 % (filtered_reservation_list))
795 return filtered_reservation_list
798 def filter_lease_start_time(reservation_list, filter_value):
799 filtered_reservation_list = list(reservation_list)
801 for reservation in reservation_list:
802 if 't_from' in reservation and \
803 reservation['t_from'] > filter_value:
804 filtered_reservation_list.remove(reservation)
806 return filtered_reservation_list
808 def complete_leases_info(self, unfiltered_reservation_list, db_xp_dict):
810 """Check that the leases list of dictionaries contains the appropriate
811 fields and piece of information here
812 :param unfiltered_reservation_list: list of leases to be completed.
813 :param db_xp_dict: leases information in the lease_sfa table
814 :returns local_unfiltered_reservation_list: list of leases completed.
815 list of dictionaries describing the leases, with all the needed
816 information (sfa,ldap,nodes)to identify one particular lease.
817 :returns testbed_xp_list: list of experiments'ids running or scheduled
819 :rtype local_unfiltered_reservation_list: list of dict
820 :rtype testbed_xp_list: list
824 local_unfiltered_reservation_list = list(unfiltered_reservation_list)
825 # slice_hrn and lease_id are in the lease_table,
826 # so they are in the db_xp_dict.
827 # component_id_list : list of nodes xrns
828 # reserved_nodes : list of nodes' hostnames
829 # slice_id : slice urn, can be made from the slice hrn using hrn_to_urn
830 for resa in local_unfiltered_reservation_list:
832 #Construct list of scheduled experiments (runing, waiting..)
833 testbed_xp_list.append(resa['lease_id'])
834 #If there is information on the experiment in the lease table
835 #(slice used and experiment id), meaning the experiment was created
837 if resa['lease_id'] in db_xp_dict:
838 xp_info = db_xp_dict[resa['lease_id']]
839 logger.debug("CORTEXLAB_API \tGetLeases xp_info %s"
841 resa['slice_hrn'] = xp_info['slice_hrn']
842 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
844 #otherwise, assume it is a cortexlab slice, created via the
847 resa['slice_id'] = hrn_to_urn(self.root_auth + '.' +
848 resa['user'] + "_slice", 'slice')
849 resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
851 resa['component_id_list'] = []
852 #Transform the hostnames into urns (component ids)
853 for node in resa['reserved_nodes']:
855 iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
856 resa['component_id_list'].append(iotlab_xrn.urn)
858 return local_unfiltered_reservation_list, testbed_xp_list
860 def GetLeases(self, lease_filter_dict=None, login=None):
863 Get the list of leases from the testbed with complete information
864 about in which slice is running which experiment ans which nodes are
867 -Fetch all the experiments from the testbed (running, waiting..)
868 complete the reservation information with slice hrn
869 found in testbed_xp table. If not available in the table,
870 assume it is a cortexlab slice.
871 -Updates the cortexlab table, deleting jobs when necessary.
873 :returns: reservation_list, list of dictionaries with 'lease_id',
874 'reserved_nodes','slice_id','user', 'component_id_list',
875 'slice_hrn', 'resource_ids', 't_from', 't_until'. Other
876 keys can be returned if necessary, such as the 'state' of the lease,
877 if the information has been added in GetReservedNodes.
882 unfiltered_reservation_list = self.GetReservedNodes(login)
884 reservation_list = []
885 #Find the slice associated with this user ldap uid
886 logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\
887 unfiltered_reservation_list %s "
888 % (login, unfiltered_reservation_list))
889 #Create user dict first to avoid looking several times for
890 #the same user in LDAP SA 27/07/12
893 db_xp_query = self.leases_db.testbed_session.query(LeaseTableXP).all()
894 db_xp_dict = dict([(row.experiment_id, row.__dict__)
895 for row in db_xp_query])
897 logger.debug("CORTEXLAB_API \tGetLeases db_xp_dict %s"
899 db_xp_id_list = [row.experiment_id for row in db_xp_query]
901 required_fiels_in_leases = ['lease_id',
902 'reserved_nodes','slice_id', 'user', 'component_id_list',
903 'slice_hrn', 'resource_ids', 't_from', 't_until']
905 # Add any missing information on the leases with complete_leases_info
906 unfiltered_reservation_list, testbed_xp_list = \
907 self.complete_leases_info(unfiltered_reservation_list,
909 # Check that the list of leases is complete and have the mandatory
911 format_status = self.ensure_format_is_valid(unfiltered_reservation_list,
912 required_fiels_in_leases)
914 if not format_status:
915 logger.log_exc("\tCortexlabapi \t GetLeases : Missing fields in \
917 raise KeyError, "GetLeases : Missing fields in reservation list "
919 if lease_filter_dict:
920 logger.debug("CORTEXLAB_API \tGetLeases \
921 \r\n leasefilter %s" % ( lease_filter_dict))
923 filter_dict_functions = {
924 'slice_hrn' : CortexlabShell.filter_lease_name,
925 't_from' : CortexlabShell.filter_lease_start_time
928 reservation_list = list(unfiltered_reservation_list)
929 for filter_type in lease_filter_dict:
930 logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \
931 % (reservation_list))
932 reservation_list = filter_dict_functions[filter_type](\
933 reservation_list,lease_filter_dict[filter_type] )
936 if lease_filter_dict is None:
937 reservation_list = unfiltered_reservation_list
939 self.leases_db.update_experiments_in_additional_sfa_db(
940 testbed_xp_list, db_xp_id_list)
942 logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s"
943 % (reservation_list))
944 return reservation_list
949 #TODO FUNCTIONS SECTION 04/07/2012 SA
951 ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
954 #def UnBindObjectFromPeer( auth, object_type, object_id, shortname):
955 #""" This method is a hopefully temporary hack to let the sfa correctly
956 #detach the objects it creates from a remote peer object. This is
957 #needed so that the sfa federation link can work in parallel with
958 #RefreshPeer, as RefreshPeer depends on remote objects being correctly
961 #auth : struct, API authentication structure
962 #AuthMethod : string, Authentication method to use
963 #object_type : string, Object type, among 'site','person','slice',
965 #object_id : int, object_id
966 #shortname : string, peer shortname
970 #logger.warning("CORTEXLAB_API \tUnBindObjectFromPeer EMPTY-\
974 ##TODO Is BindObjectToPeer still necessary ? Currently does nothing
976 #|| Commented out 28/05/13 SA
977 #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
978 #remote_object_id=None):
979 #"""This method is a hopefully temporary hack to let the sfa correctly
980 #attach the objects it creates to a remote peer object. This is needed
981 #so that the sfa federation link can work in parallel with RefreshPeer,
982 #as RefreshPeer depends on remote objects being correctly marked.
984 #shortname : string, peer shortname
985 #remote_object_id : int, remote object_id, set to 0 if unknown
989 #logger.warning("CORTEXLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
992 ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA
993 ##Funciton should delete and create another job since oin iotlab slice=job
994 #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
995 #"""Updates the parameters of an existing slice with the values in
997 #Users may only update slices of which they are members.
998 #PIs may update any of the slices at their sites, or any slices of
999 #which they are members. Admins may update any slice.
1000 #Only PIs and admins may update max_nodes. Slices cannot be renewed
1001 #(by updating the expires parameter) more than 8 weeks into the future.
1002 #Returns 1 if successful, faults otherwise.
1006 #logger.warning("CORTEXLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ")
1009 #Unused SA 30/05/13, we only update the user's key or we delete it.
1010 ##TODO UpdatePerson 04/07/2012 SA
1011 #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None):
1012 #"""Updates a person. Only the fields specified in person_fields
1013 #are updated, all other fields are left untouched.
1014 #Users and techs can only update themselves. PIs can only update
1015 #themselves and other non-PIs at their sites.
1016 #Returns 1 if successful, faults otherwise.
1020 ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
1021 ##self.leases_db.testbed_session.add(new_row)
1022 ##self.leases_db.testbed_session.commit()
1024 #logger.debug("CORTEXLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
1028 def GetKeys(self, key_filter=None):
1029 """Returns a dict of dict based on the key string. Each dict entry
1030 contains the key id, the ssh key, the user's email and the
1032 If key_filter is specified and is an array of key identifiers,
1033 only keys matching the filter will be returned.
1035 Admin may query all keys. Non-admins may only query their own keys.
1038 :returns: dict with ssh key as key and dicts as value.
1041 if key_filter is None:
1042 keys = dbsession.query(RegKey).options(joinedload('reg_user')).all()
1044 keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all()
1048 key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
1049 'email': key.reg_user.email,
1050 'hrn': key.reg_user.hrn}
1052 #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
1053 #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
1054 #for user in ldap_rslt)
1056 logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
1060 def DeleteKey(self, user_record, key_string):
1061 """Deletes a key in the LDAP entry of the specified user.
1063 Removes the key_string from the user's key list and updates the LDAP
1064 user's entry with the new key attributes.
1066 :param key_string: The ssh key to remove
1067 :param user_record: User's record
1068 :type key_string: string
1069 :type user_record: dict
1070 :returns: True if sucessful, False if not.
1074 all_user_keys = user_record['keys']
1075 all_user_keys.remove(key_string)
1076 new_attributes = {'sshPublicKey':all_user_keys}
1077 ret = self.ldap.LdapModifyUser(user_record, new_attributes)
1078 logger.debug("CORTEXLAB_API DeleteKey %s- " % (ret))
1082 def _sql_get_slice_info(self, slice_filter):
1084 Get the slice record based on the slice hrn. Fetch the record of the
1085 user associated with the slice by using joinedload based on the
1086 reg_researcher relationship.
1088 :param slice_filter: the slice hrn we are looking for
1089 :type slice_filter: string
1090 :returns: the slice record enhanced with the user's information if the
1091 slice was found, None it wasn't.
1093 :rtype: dict or None.
1095 #DO NOT USE RegSlice - reg_researchers to get the hrn
1096 #of the user otherwise will mess up the RegRecord in
1097 #Resolve, don't know why - SA 08/08/2012
1099 #Only one entry for one user = one slice in testbed_xp table
1100 #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1101 raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
1102 #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1104 #load_reg_researcher
1105 #raw_slicerec.reg_researchers
1106 raw_slicerec = raw_slicerec.__dict__
1107 logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \
1108 raw_slicerec %s" % (slice_filter, raw_slicerec))
1109 slicerec = raw_slicerec
1110 #only one researcher per slice so take the first one
1111 #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
1112 #del slicerec['reg_researchers']['_sa_instance_state']
1119 def _sql_get_slice_info_from_user(self, slice_filter):
1121 Get the slice record based on the user recordid by using a joinedload
1122 on the relationship reg_slices_as_researcher. Format the sql record
1123 into a dict with the mandatory fields for user and slice.
1124 :returns: dict with slice record and user record if the record was found
1125 based on the user's id, None if not..
1126 :rtype:dict or None..
1128 #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1129 raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
1130 #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1131 #Put it in correct order
1132 user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1133 'classtype', 'authority', 'gid', 'record_id',
1134 'date_created', 'type', 'email', 'pointer']
1135 slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1136 'classtype', 'authority', 'gid', 'record_id',
1137 'date_created', 'type', 'pointer']
1139 #raw_slicerec.reg_slices_as_researcher
1140 raw_slicerec = raw_slicerec.__dict__
1143 dict([(k, raw_slicerec[
1144 'reg_slices_as_researcher'][0].__dict__[k])
1145 for k in slice_needed_fields])
1146 slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
1147 for k in user_needed_fields])
1148 #TODO Handle multiple slices for one user SA 10/12/12
1149 #for now only take the first slice record associated to the rec user
1150 ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
1151 #del raw_slicerec['reg_slices_as_researcher']
1152 #slicerec['reg_researchers'] = raw_slicerec
1153 ##del slicerec['_sa_instance_state']
1160 def _get_slice_records(self, slice_filter=None,
1161 slice_filter_type=None):
1163 Get the slice record depending on the slice filter and its type.
1164 :param slice_filter: Can be either the slice hrn or the user's record
1166 :type slice_filter: string
1167 :param slice_filter_type: describes the slice filter type used, can be
1168 slice_hrn or record_id_user
1170 :returns: the slice record
1172 .. seealso::_sql_get_slice_info_from_user
1173 .. seealso:: _sql_get_slice_info
1176 #Get list of slices based on the slice hrn
1177 if slice_filter_type == 'slice_hrn':
1179 #if get_authority(slice_filter) == self.root_auth:
1180 #login = slice_filter.split(".")[1].split("_")[0]
1182 slicerec = self._sql_get_slice_info(slice_filter)
1184 if slicerec is None:
1188 #Get slice based on user id
1189 if slice_filter_type == 'record_id_user':
1191 slicerec = self._sql_get_slice_info_from_user(slice_filter)
1194 fixed_slicerec_dict = slicerec
1195 #At this point if there is no login it means
1196 #record_id_user filter has been used for filtering
1198 ##If theslice record is from iotlab
1199 #if fixed_slicerec_dict['peer_authority'] is None:
1200 #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
1201 #return login, fixed_slicerec_dict
1202 return fixed_slicerec_dict
1207 def GetSlices(self, slice_filter=None, slice_filter_type=None,
1209 """Get the slice records from the sfa db and add lease information
1212 :param slice_filter: can be the slice hrn or slice record id in the db
1213 depending on the slice_filter_type.
1214 :param slice_filter_type: defines the type of the filtering used, Can be
1215 either 'slice_hrn' or 'record_id'.
1216 :type slice_filter: string
1217 :type slice_filter_type: string
1218 :returns: a slice dict if slice_filter and slice_filter_type
1219 are specified and a matching entry is found in the db. The result
1220 is put into a list.Or a list of slice dictionnaries if no filters
1227 authorized_filter_types_list = ['slice_hrn', 'record_id_user']
1228 return_slicerec_dictlist = []
1230 #First try to get information on the slice based on the filter provided
1231 if slice_filter_type in authorized_filter_types_list:
1232 fixed_slicerec_dict = self._get_slice_records(slice_filter,
1234 # if the slice was not found in the sfa db
1235 if fixed_slicerec_dict is None:
1236 return return_slicerec_dictlist
1238 slice_hrn = fixed_slicerec_dict['hrn']
1240 logger.debug(" CORTEXLAB_API \tGetSlices login %s \
1241 slice record %s slice_filter %s \
1242 slice_filter_type %s " % (login,
1243 fixed_slicerec_dict, slice_filter,
1247 #Now we have the slice record fixed_slicerec_dict, get the
1248 #jobs associated to this slice
1251 leases_list = self.GetLeases(login=login)
1252 #If no job is running or no job scheduled
1253 #return only the slice record
1254 if leases_list == [] and fixed_slicerec_dict:
1255 return_slicerec_dictlist.append(fixed_slicerec_dict)
1257 # if the jobs running don't belong to the user/slice we are looking
1259 leases_hrn = [lease['slice_hrn'] for lease in leases_list]
1260 if slice_hrn not in leases_hrn:
1261 return_slicerec_dictlist.append(fixed_slicerec_dict)
1262 #If several experiments for one slice , put the slice record into
1263 # each lease information dict
1264 for lease in leases_list:
1266 logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \
1267 \t lease['slice_hrn'] %s"
1268 % (slice_filter, lease['slice_hrn']))
1269 if lease['slice_hrn'] == slice_hrn:
1270 slicerec_dict['experiment_id'] = lease['lease_id']
1271 #Update lease dict with the slice record
1272 if fixed_slicerec_dict:
1273 fixed_slicerec_dict['experiment_id'] = []
1274 fixed_slicerec_dict['experiment_id'].append(
1275 slicerec_dict['experiment_id'])
1276 slicerec_dict.update(fixed_slicerec_dict)
1278 slicerec_dict['slice_hrn'] = lease['slice_hrn']
1279 slicerec_dict['hrn'] = lease['slice_hrn']
1280 slicerec_dict['user'] = lease['user']
1281 slicerec_dict.update(
1283 {'hostname': lease['reserved_nodes']}})
1284 slicerec_dict.update({'node_ids': lease['reserved_nodes']})
1287 return_slicerec_dictlist.append(slicerec_dict)
1290 logger.debug("CORTEXLAB_API.PY \tGetSlices \
1291 slicerec_dict %s return_slicerec_dictlist %s \
1292 lease['reserved_nodes'] \
1293 %s" % (slicerec_dict, return_slicerec_dictlist,
1294 lease['reserved_nodes']))
1296 logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
1297 return_slicerec_dictlist %s"
1298 % (return_slicerec_dictlist))
1300 return return_slicerec_dictlist
1304 #Get all slices from the cortexlab sfa database , get the user info
1305 # as well at the same time put them in dict format
1307 query_slice_list = \
1308 dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
1310 for record in query_slice_list:
1311 tmp = record.__dict__
1312 tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
1313 return_slicerec_dictlist.append(tmp)
1316 #Get all the experiments reserved nodes
1317 leases_list = self.GetReservedNodes()
1319 for fixed_slicerec_dict in return_slicerec_dictlist:
1321 #Check if the slice belongs to a cortexlab user
1322 if fixed_slicerec_dict['peer_authority'] is None:
1323 owner = fixed_slicerec_dict['hrn'].split(
1324 ".")[1].split("_")[0]
1327 for lease in leases_list:
1328 if owner == lease['user']:
1329 slicerec_dict['experiment_id'] = lease['lease_id']
1331 #for reserved_node in lease['reserved_nodes']:
1332 logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s "
1334 slicerec_dict.update(fixed_slicerec_dict)
1335 slicerec_dict.update({'node_ids':
1336 lease['reserved_nodes']})
1337 slicerec_dict.update({'list_node_ids':
1339 lease['reserved_nodes']}})
1342 fixed_slicerec_dict.update(slicerec_dict)
1344 logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
1345 return_slicerec_dictlist %s \slice_filter %s " \
1346 %(return_slicerec_dictlist, slice_filter))
1348 return return_slicerec_dictlist
1352 #Update slice unused, therefore sfa_fields_to_iotlab_fields unused
1355 #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record):
1360 ##for field in record:
1361 ## iotlab_record[field] = record[field]
1363 #if sfa_type == "slice":
1364 ##instantion used in get_slivers ?
1365 #if not "instantiation" in iotlab_record:
1366 #iotlab_record["instantiation"] = "iotlab-instantiated"
1367 ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn)
1368 ##Unused hrn_to_pl_slicename because Iotlab's hrn already
1369 ##in the appropriate form SA 23/07/12
1370 #iotlab_record["hrn"] = hrn
1371 #logger.debug("CORTEXLAB_API.PY sfa_fields_to_iotlab_fields \
1372 #iotlab_record %s " %(iotlab_record['hrn']))
1373 #if "url" in record:
1374 #iotlab_record["url"] = record["url"]
1375 #if "description" in record:
1376 #iotlab_record["description"] = record["description"]
1377 #if "expires" in record:
1378 #iotlab_record["expires"] = int(record["expires"])
1380 ##nodes added by OAR only and then imported to SFA
1381 ##elif type == "node":
1382 ##if not "hostname" in iotlab_record:
1383 ##if not "hostname" in record:
1384 ##raise MissingSfaInfo("hostname")
1385 ##iotlab_record["hostname"] = record["hostname"]
1386 ##if not "model" in iotlab_record:
1387 ##iotlab_record["model"] = "geni"
1389 ##One authority only
1390 ##elif type == "authority":
1391 ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn)
1393 ##if not "name" in iotlab_record:
1394 ##iotlab_record["name"] = hrn
1396 ##if not "abbreviated_name" in iotlab_record:
1397 ##iotlab_record["abbreviated_name"] = hrn
1399 ##if not "enabled" in iotlab_record:
1400 ##iotlab_record["enabled"] = True
1402 ##if not "is_public" in iotlab_record:
1403 ##iotlab_record["is_public"] = True
1405 #return iotlab_record