2 File containing the IotlabTestbedAPI, used to interact with nodes, users,
3 slices, leases and keys, as well as the dedicated iotlab database and table,
4 holding information about which slice is running which job.
7 from datetime import datetime
9 from sfa.util.sfalogging import logger
11 from sfa.storage.alchemy import dbsession
12 from sqlalchemy.orm import joinedload
13 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
14 from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP
15 from sfa.iotlab.OARrestapi import OARrestapi
16 from sfa.iotlab.LDAPapi import LDAPapi
18 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
20 from sfa.trust.certificate import Keypair, convert_public_key
21 from sfa.trust.gid import create_uuid
22 from sfa.trust.hierarchy import Hierarchy
24 from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
26 class IotlabTestbedAPI():
27 """ Class enabled to use LDAP and OAR api calls. """
29 _MINIMUM_DURATION = 600
31 def __init__(self, config):
32 """Creates an instance of OARrestapi and LDAPapi which will be used to
33 issue calls to OAR or LDAP methods.
34 Set the time format and the testbed granularity used for OAR
35 reservation and leases.
37 :param config: configuration object from sfa.util.config
38 :type config: Config object
40 self.iotlab_db = IotlabDB(config)
41 self.oar = OARrestapi()
43 self.time_format = "%Y-%m-%d %H:%M:%S"
44 self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
45 self.grain = 1 # 10 mins lease minimum, 1 sec granularity
46 #import logging, logging.handlers
47 #from sfa.util.sfalogging import _SfaLogger
48 #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \
53 def GetMinExperimentDurationInSec():
54 """ Returns the minimum allowed duration for an experiment on the
58 return IotlabTestbedAPI._MINIMUM_DURATION
61 def GetPeers(peer_filter=None ):
62 """ Gathers registered authorities in SFA DB and looks for specific peer
63 if peer_filter is specified.
64 :param peer_filter: name of the site authority looked for.
65 :type peer_filter: string
66 :returns: list of records.
71 existing_hrns_by_types = {}
72 logger.debug("IOTLAB_API \tGetPeers peer_filter %s, \
74 all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
76 for record in all_records:
77 existing_records[(record.hrn, record.type)] = record
78 if record.type not in existing_hrns_by_types:
79 existing_hrns_by_types[record.type] = [record.hrn]
81 existing_hrns_by_types[record.type].append(record.hrn)
84 logger.debug("IOTLAB_API \tGetPeer\texisting_hrns_by_types %s "\
85 %( existing_hrns_by_types))
90 records_list.append(existing_records[(peer_filter,'authority')])
92 for hrn in existing_hrns_by_types['authority']:
93 records_list.append(existing_records[(hrn,'authority')])
95 logger.debug("IOTLAB_API \tGetPeer \trecords_list %s " \
101 return_records = records_list
102 logger.debug("IOTLAB_API \tGetPeer return_records %s "
104 return return_records
106 #TODO : Handling OR request in make_ldap_filters_from_records
107 #instead of the for loop
108 #over the records' list
109 def GetPersons(self, person_filter=None):
111 Get the enabled users and their properties from Iotlab LDAP.
112 If a filter is specified, looks for the user whose properties match
113 the filter, otherwise returns the whole enabled users'list.
115 :param person_filter: Must be a list of dictionnaries
116 with users properties when not set to None.
117 :param person_filter: list of dict
118 :returns:Returns a list of users whose accounts are enabled
120 :rtype: list of dicts
123 logger.debug("IOTLAB_API \tGetPersons person_filter %s"
126 if person_filter and isinstance(person_filter, list):
127 #If we are looking for a list of users (list of dict records)
128 #Usually the list contains only one user record
129 for searched_attributes in person_filter:
131 #Get only enabled user accounts in iotlab LDAP :
132 #add a filter for make_ldap_filters_from_record
133 person = self.ldap.LdapFindUser(searched_attributes,
134 is_user_enabled=True)
135 #If a person was found, append it to the list
137 person_list.append(person)
139 #If the list is empty, return None
140 if len(person_list) is 0:
144 #Get only enabled user accounts in iotlab LDAP :
145 #add a filter for make_ldap_filters_from_record
146 person_list = self.ldap.LdapFindUser(is_user_enabled=True)
151 #def GetTimezone(self):
152 #""" Returns the OAR server time and timezone.
153 #Unused SA 30/05/13"""
154 #server_timestamp, server_tz = self.oar.parser.\
155 #SendRequest("GET_timezone")
156 #return server_timestamp, server_tz
158 def DeleteJobs(self, job_id, username):
161 Deletes the job with the specified job_id and username on OAR by
162 posting a delete request to OAR.
164 :param job_id: job id in OAR.
165 :param username: user's iotlab login in LDAP.
166 :type job_id: integer
167 :type username: string
169 :returns: dictionary with the job id and if delete has been successful
174 logger.debug("IOTLAB_API \tDeleteJobs jobid %s username %s "
175 % (job_id, username))
176 if not job_id or job_id is -1:
180 reqdict['method'] = "delete"
181 reqdict['strval'] = str(job_id)
183 answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id',
185 if answer['status'] == 'Delete request registered':
188 ret = {job_id: False}
189 logger.debug("IOTLAB_API \tDeleteJobs jobid %s \r\n answer %s \
190 username %s" % (job_id, answer, username))
195 ##TODO : Unused GetJobsId ? SA 05/07/12
196 #def GetJobsId(self, job_id, username = None ):
198 #Details about a specific job.
199 #Includes details about submission time, jot type, state, events,
200 #owner, assigned ressources, walltime etc...
204 #node_list_k = 'assigned_network_address'
205 ##Get job info from OAR
206 #job_info = self.oar.parser.SendRequest(req, job_id, username)
208 #logger.debug("IOTLAB_API \t GetJobsId %s " %(job_info))
210 #if job_info['state'] == 'Terminated':
211 #logger.debug("IOTLAB_API \t GetJobsId job %s TERMINATED"\
214 #if job_info['state'] == 'Error':
215 #logger.debug("IOTLAB_API \t GetJobsId ERROR message %s "\
220 #logger.error("IOTLAB_API \tGetJobsId KeyError")
223 #parsed_job_info = self.get_info_on_reserved_nodes(job_info, \
225 ##Replaces the previous entry
226 ##"assigned_network_address" / "reserved_resources"
228 #job_info.update({'node_ids':parsed_job_info[node_list_k]})
229 #del job_info[node_list_k]
230 #logger.debug(" \r\nIOTLAB_API \t GetJobsId job_info %s " %(job_info))
234 def GetJobsResources(self, job_id, username = None):
235 """ Gets the list of nodes associated with the job_id and username
237 Transforms the iotlab hostnames to the corresponding
239 Rertuns dict key :'node_ids' , value : hostnames list
240 :param username: user's LDAP login
241 :paran job_id: job's OAR identifier.
242 :type username: string
243 :type job_id: integer
245 :returns: dicionary with nodes' hostnames belonging to the job.
249 req = "GET_jobs_id_resources"
252 #Get job resources list from OAR
253 node_id_list = self.oar.parser.SendRequest(req, job_id, username)
254 logger.debug("IOTLAB_API \t GetJobsResources %s " %(node_id_list))
257 self.__get_hostnames_from_oar_node_ids(node_id_list)
260 #Replaces the previous entry "assigned_network_address" /
261 #"reserved_resources" with "node_ids"
262 job_info = {'node_ids': hostname_list}
267 #def get_info_on_reserved_nodes(self, job_info, node_list_name):
269 #..warning:unused SA 23/05/13
271 ##Get the list of the testbed nodes records and make a
272 ##dictionnary keyed on the hostname out of it
273 #node_list_dict = self.GetNodes()
274 ##node_hostname_list = []
275 #node_hostname_list = [node['hostname'] for node in node_list_dict]
276 ##for node in node_list_dict:
277 ##node_hostname_list.append(node['hostname'])
278 #node_dict = dict(zip(node_hostname_list, node_list_dict))
280 #reserved_node_hostname_list = []
281 #for index in range(len(job_info[node_list_name])):
282 ##job_info[node_list_name][k] =
283 #reserved_node_hostname_list[index] = \
284 #node_dict[job_info[node_list_name][index]]['hostname']
286 #logger.debug("IOTLAB_API \t get_info_on_reserved_nodes \
287 #reserved_node_hostname_list %s" \
288 #%(reserved_node_hostname_list))
290 #logger.error("IOTLAB_API \t get_info_on_reserved_nodes KEYERROR " )
292 #return reserved_node_hostname_list
294 def GetNodesCurrentlyInUse(self):
295 """Returns a list of all the nodes already involved in an oar running
297 :rtype: list of nodes hostnames.
299 return self.oar.parser.SendRequest("GET_running_jobs")
301 def __get_hostnames_from_oar_node_ids(self, resource_id_list ):
302 """Get the hostnames of the nodes from their OAR identifiers.
303 Get the list of nodes dict using GetNodes and find the hostname
304 associated with the identifier.
305 :param resource_id_list: list of nodes identifiers
306 :returns: list of node hostnames.
308 full_nodes_dict_list = self.GetNodes()
309 #Put the full node list into a dictionary keyed by oar node id
310 oar_id_node_dict = {}
311 for node in full_nodes_dict_list:
312 oar_id_node_dict[node['oar_id']] = node
315 for resource_id in resource_id_list:
316 #Because jobs requested "asap" do not have defined resources
317 if resource_id is not "Undefined":
318 hostname_list.append(\
319 oar_id_node_dict[resource_id]['hostname'])
321 #hostname_list.append(oar_id_node_dict[resource_id]['hostname'])
324 def GetReservedNodes(self, username=None):
325 """ Get list of leases. Get the leases for the username if specified,
326 otherwise get all the leases. Finds the nodes hostnames for each
328 :param username: user's LDAP login
329 :type username: string
330 :returns: list of reservations dict
334 #Get the nodes in use and the reserved nodes
335 reservation_dict_list = \
336 self.oar.parser.SendRequest("GET_reserved_nodes", \
340 for resa in reservation_dict_list:
341 logger.debug ("GetReservedNodes resa %s"%(resa))
342 #dict list of hostnames and their site
343 resa['reserved_nodes'] = \
344 self.__get_hostnames_from_oar_node_ids(resa['resource_ids'])
346 #del resa['resource_ids']
347 return reservation_dict_list
349 def GetNodes(self, node_filter_dict=None, return_fields_list=None):
352 Make a list of iotlab nodes and their properties from information
353 given by OAR. Search for specific nodes if some filters are
354 specified. Nodes properties returned if no return_fields_list given:
355 'hrn','archi','mobile','hostname','site','boot_state','node_id',
356 'radio','posx','posy','oar_id','posz'.
358 :param node_filter_dict: dictionnary of lists with node properties. For
359 instance, if you want to look for a specific node with its hrn,
360 the node_filter_dict should be {'hrn': [hrn_of_the_node]}
361 :type node_filter_dict: dict
362 :param return_fields_list: list of specific fields the user wants to be
364 :type return_fields_list: list
365 :returns: list of dictionaries with node properties
369 node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
370 node_dict_list = node_dict_by_id.values()
371 logger.debug (" IOTLAB_API GetNodes node_filter_dict %s \
372 return_fields_list %s " % (node_filter_dict, return_fields_list))
373 #No filtering needed return the list directly
374 if not (node_filter_dict or return_fields_list):
375 return node_dict_list
377 return_node_list = []
379 for filter_key in node_filter_dict:
381 #Filter the node_dict_list by each value contained in the
382 #list node_filter_dict[filter_key]
383 for value in node_filter_dict[filter_key]:
384 for node in node_dict_list:
385 if node[filter_key] == value:
386 if return_fields_list:
388 for k in return_fields_list:
390 return_node_list.append(tmp)
392 return_node_list.append(node)
394 logger.log_exc("GetNodes KeyError")
398 return return_node_list
403 def AddSlice(slice_record, user_record):
406 Add slice to the local iotlab sfa tables if the slice comes
407 from a federated site and is not yet in the iotlab sfa DB,
408 although the user has already a LDAP login.
409 Called by verify_slice during lease/sliver creation.
411 :param slice_record: record of slice, must contain hrn, gid, slice_id
412 and authority of the slice.
413 :type slice_record: dictionary
414 :param user_record: record of the user
415 :type user_record: RegUser
419 sfa_record = RegSlice(hrn=slice_record['hrn'],
420 gid=slice_record['gid'],
421 pointer=slice_record['slice_id'],
422 authority=slice_record['authority'])
423 logger.debug("IOTLAB_API.PY AddSlice sfa_record %s user_record %s"
424 % (sfa_record, user_record))
425 sfa_record.just_created()
426 dbsession.add(sfa_record)
428 #Update the reg-researcher dependance table
429 sfa_record.reg_researchers = [user_record]
435 def GetSites(self, site_filter_name_list=None, return_fields_list=None):
436 """Returns the list of Iotlab's sites with the associated nodes and
437 their properties as dictionaries.
439 Uses the OAR request GET_sites to find the Iotlab's sites.
441 :param site_filter_name_list: used to specify specific sites
442 :param return_fields_list: field that has to be returned
443 :type site_filter_name_list: list
444 :type return_fields_list: list
448 site_dict = self.oar.parser.SendRequest("GET_sites")
449 #site_dict : dict where the key is the sit ename
450 return_site_list = []
451 if not (site_filter_name_list or return_fields_list):
452 return_site_list = site_dict.values()
453 return return_site_list
455 for site_filter_name in site_filter_name_list:
456 if site_filter_name in site_dict:
457 if return_fields_list:
458 for field in return_fields_list:
461 tmp[field] = site_dict[site_filter_name][field]
463 logger.error("GetSites KeyError %s " % (field))
465 return_site_list.append(tmp)
467 return_site_list.append(site_dict[site_filter_name])
469 return return_site_list
472 #TODO : Check rights to delete person
473 def DeletePerson(self, person_record):
474 """Disable an existing account in iotlab LDAP.
476 Users and techs can only delete themselves. PIs can only
477 delete themselves and other non-PIs at their sites.
478 ins can delete anyone.
480 :param person_record: user's record
481 :type person_record: dict
482 :returns: True if successful, False otherwise.
485 .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF.
487 #Disable user account in iotlab LDAP
488 ret = self.ldap.LdapMarkUserAsDeleted(person_record)
489 logger.warning("IOTLAB_API DeletePerson %s " % (person_record))
492 def DeleteSlice(self, slice_record):
493 """Deletes the specified slice and kills the jobs associated with
494 the slice if any, using DeleteSliceFromNodes.
496 :param slice_record: record of the slice, must contain oar_job_id, user
497 :type slice_record: dict
498 :returns: True if all the jobs in the slice have been deleted,
499 or the list of jobs that could not be deleted otherwise.
500 :rtype: list or boolean
502 .. seealso:: DeleteSliceFromNodes
505 ret = self.DeleteSliceFromNodes(slice_record)
508 if False in ret[job_id]:
509 if delete_failed is None:
511 delete_failed.append(job_id)
513 logger.info("IOTLAB_API DeleteSlice %s answer %s"%(slice_record, \
515 return delete_failed or True
518 def __add_person_to_db(user_dict):
520 Add a federated user straight to db when the user issues a lease
521 request with iotlab nodes and that he has not registered with iotlab
522 yet (that is he does not have a LDAP entry yet).
523 Uses parts of the routines in SlabImport when importing user from LDAP.
524 Called by AddPerson, right after LdapAddUser.
525 :param user_dict: Must contain email, hrn and pkey to get a GID
526 and be added to the SFA db.
527 :type user_dict: dict
531 dbsession.query(RegUser).filter_by(email = user_dict['email']).first()
533 if not check_if_exists:
534 logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
536 hrn = user_dict['hrn']
537 person_urn = hrn_to_urn(hrn, 'user')
538 pubkey = user_dict['pkey']
540 pkey = convert_public_key(pubkey)
542 #key not good. create another pkey
543 logger.warn('__add_person_to_db: unable to convert public \
545 pkey = Keypair(create=True)
548 if pubkey is not None and pkey is not None :
549 hierarchy = Hierarchy()
550 person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
552 if user_dict['email']:
553 logger.debug("__add_person_to_db \r\n \r\n \
554 IOTLAB IMPORTER PERSON EMAIL OK email %s "\
555 %(user_dict['email']))
556 person_gid.set_email(user_dict['email'])
558 user_record = RegUser(hrn=hrn , pointer= '-1', \
559 authority=get_authority(hrn), \
560 email=user_dict['email'], gid = person_gid)
561 user_record.reg_keys = [RegKey(user_dict['pkey'])]
562 user_record.just_created()
563 dbsession.add (user_record)
568 def AddPerson(self, record):
571 Adds a new account. Any fields specified in records are used,
572 otherwise defaults are used. Creates an appropriate login by calling
575 :param record: dictionary with the sfa user's properties.
576 :returns: a dicitonary with the status. If successful, the dictionary
577 boolean is set to True and there is a 'uid' key with the new login
578 added to LDAP, otherwise the bool is set to False and a key
579 'message' is in the dictionary, with the error message.
583 ret = self.ldap.LdapAddUser(record)
585 if ret['bool'] is True:
586 record['hrn'] = self.root_auth + '.' + ret['uid']
587 logger.debug("IOTLAB_API AddPerson return code %s record %s "
589 self.__add_person_to_db(record)
596 #TODO AddPersonKey 04/07/2012 SA
597 def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
598 """Adds a new key to the specified account. Adds the key to the
599 iotlab ldap, provided that the person_uid is valid.
601 Non-admins can only modify their own keys.
603 :param person_uid: user's iotlab login in LDAP
604 :param old_attributes_dict: dict with the user's old sshPublicKey
605 :param new_key_dict: dict with the user's new sshPublicKey
606 :type person_uid: string
610 :returns: True if the key has been modified, False otherwise.
613 ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \
615 logger.warning("IOTLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ")
618 def DeleteLeases(self, leases_id_list, slice_hrn):
621 Deletes several leases, based on their job ids and the slice
622 they are associated with. Uses DeleteJobs to delete the jobs
623 on OAR. Note that one slice can contain multiple jobs, and in this
624 case all the jobs in the leases_id_list MUST belong to ONE slice,
625 since there is only one slice hrn provided here.
627 :param leases_id_list: list of job ids that belong to the slice whose
628 slice hrn is provided.
629 :param slice_hrn: the slice hrn.
630 :type slice_hrn: string
632 .. warning:: Does not have a return value since there was no easy
633 way to handle failure when dealing with multiple job delete. Plus,
634 there was no easy way to report it to the user.
637 logger.debug("IOTLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
638 \r\n " %(leases_id_list, slice_hrn))
639 for job_id in leases_id_list:
640 self.DeleteJobs(job_id, slice_hrn)
645 def _process_walltime(duration):
646 """ Calculates the walltime in seconds from the duration in H:M:S
647 specified in the RSpec.
651 # Fixing the walltime by adding a few delays.
652 # First put the walltime in seconds oarAdditionalDelay = 20;
653 # additional delay for /bin/sleep command to
654 # take in account prologue and epilogue scripts execution
655 # int walltimeAdditionalDelay = 240; additional delay
656 #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
658 # Put the duration in seconds first
659 #desired_walltime = duration * 60
660 desired_walltime = duration
661 total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
662 sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
664 #Put the walltime back in str form
666 walltime.append(str(total_walltime / 3600))
667 total_walltime = total_walltime - 3600 * int(walltime[0])
668 #Get the remaining minutes
669 walltime.append(str(total_walltime / 60))
670 total_walltime = total_walltime - 60 * int(walltime[1])
672 walltime.append(str(total_walltime))
675 logger.log_exc(" __process_walltime duration null")
677 return walltime, sleep_walltime
680 def _create_job_structure_request_for_OAR(lease_dict):
681 """ Creates the structure needed for a correct POST on OAR.
682 Makes the timestamp transformation into the appropriate format.
683 Sends the POST request to create the job with the resources in
692 reqdict['workdir'] = '/tmp'
693 reqdict['resource'] = "{network_address in ("
695 for node in lease_dict['added_nodes']:
696 logger.debug("\r\n \r\n OARrestapi \t \
697 __create_job_structure_request_for_OAR node %s" %(node))
699 # Get the ID of the node
701 reqdict['resource'] += "'" + nodeid + "', "
702 nodeid_list.append(nodeid)
704 custom_length = len(reqdict['resource'])- 2
705 reqdict['resource'] = reqdict['resource'][0:custom_length] + \
706 ")}/nodes=" + str(len(nodeid_list))
709 walltime, sleep_walltime = \
710 IotlabTestbedAPI._process_walltime(\
711 int(lease_dict['lease_duration']))
714 reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
715 ":" + str(walltime[1]) + ":" + str(walltime[2])
716 reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
718 #In case of a scheduled experiment (not immediate)
719 #To run an XP immediately, don't specify date and time in RSpec
720 #They will be set to None.
721 if lease_dict['lease_start_time'] is not '0':
722 #Readable time accepted by OAR
723 start_time = datetime.fromtimestamp( \
724 int(lease_dict['lease_start_time'])).\
725 strftime(lease_dict['time_format'])
726 reqdict['reservation'] = start_time
727 #If there is not start time, Immediate XP. No need to add special
731 reqdict['type'] = "deploy"
732 reqdict['directory'] = ""
733 reqdict['name'] = "SFA_" + lease_dict['slice_user']
738 def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
739 lease_start_time, lease_duration, slice_user=None):
742 Create a job request structure based on the information provided
743 and post the job on OAR.
744 :param added_nodes: list of nodes that belong to the described lease.
745 :param slice_name: the slice hrn associated to the lease.
746 :param lease_start_time: timestamp of the lease startting time.
747 :param lease_duration: lease durationin minutes
751 lease_dict['lease_start_time'] = lease_start_time
752 lease_dict['lease_duration'] = lease_duration
753 lease_dict['added_nodes'] = added_nodes
754 lease_dict['slice_name'] = slice_name
755 lease_dict['slice_user'] = slice_user
756 lease_dict['grain'] = self.GetLeaseGranularity()
757 lease_dict['time_format'] = self.time_format
760 logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\
761 \r\n " %(slice_user))
762 #Create the request for OAR
763 reqdict = self._create_job_structure_request_for_OAR(lease_dict)
764 # first step : start the OAR job and update the job
765 logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\
768 answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
770 logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s " %(answer))
774 logger.log_exc("IOTLAB_API \tLaunchExperimentOnOAR \
775 Impossible to create job %s " %(answer))
782 logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s \
783 added_nodes %s slice_user %s" %(jobid, added_nodes, \
790 def AddLeases(self, hostname_list, slice_record,
791 lease_start_time, lease_duration):
793 """Creates a job in OAR corresponding to the information provided
794 as parameters. Adds the job id and the slice hrn in the iotlab
795 database so that we are able to know which slice has which nodes.
797 :param hostname_list: list of nodes' OAR hostnames.
798 :param slice_record: sfa slice record, must contain login and hrn.
799 :param lease_start_time: starting time , unix timestamp format
800 :param lease_duration: duration in minutes
802 :type hostname_list: list
803 :type slice_record: dict
804 :type lease_start_time: integer
805 :type lease_duration: integer
808 logger.debug("IOTLAB_API \r\n \r\n \t AddLeases hostname_list %s \
809 slice_record %s lease_start_time %s lease_duration %s "\
810 %( hostname_list, slice_record , lease_start_time, \
813 #tmp = slice_record['reg-researchers'][0].split(".")
814 username = slice_record['login']
815 #username = tmp[(len(tmp)-1)]
816 job_id = self.LaunchExperimentOnOAR(hostname_list, \
817 slice_record['hrn'], \
818 lease_start_time, lease_duration, \
821 datetime.fromtimestamp(int(lease_start_time)).\
822 strftime(self.time_format)
823 end_time = lease_start_time + lease_duration
826 logger.debug("IOTLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
827 %s %s %s "%(slice_record['hrn'], job_id, end_time))
830 logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \
831 %(type(slice_record['hrn']), type(job_id), type(end_time)))
833 iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id,
836 logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \
838 self.iotlab_db.iotlab_session.add(iotlab_ex_row)
839 self.iotlab_db.iotlab_session.commit()
841 logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s " \
847 #Delete the jobs from job_iotlab table
848 def DeleteSliceFromNodes(self, slice_record):
851 Deletes all the running or scheduled jobs of a given slice
854 :param slice_record: record of the slice, must contain oar_job_id, user
855 :type slice_record: dict
857 :returns: dict of the jobs'deletion status. Success= True, Failure=
858 False, for each job id.
862 logger.debug("IOTLAB_API \t DeleteSliceFromNodes %s "
865 if isinstance(slice_record['oar_job_id'], list):
867 for job_id in slice_record['oar_job_id']:
868 ret = self.DeleteJobs(job_id, slice_record['user'])
870 oar_bool_answer.update(ret)
873 oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'],
874 slice_record['user'])]
876 return oar_bool_answer
880 def GetLeaseGranularity(self):
881 """ Returns the granularity of an experiment in the Iotlab testbed.
882 OAR uses seconds for experiments duration , the granulaity is also
884 Experiments which last less than 10 min (600 sec) are invalid"""
889 # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql):
890 # """ Cleans the iotlab db by deleting expired and cancelled jobs.
891 # Compares the list of job ids given by OAR with the job ids that
892 # are already in the database, deletes the jobs that are no longer in
893 # the OAR job id list.
894 # :param job_oar_list: list of job ids coming from OAR
895 # :type job_oar_list: list
896 # :param job_psql: list of job ids cfrom the database.
897 # type job_psql: list
899 # #Turn the list into a set
900 # set_jobs_psql = set(jobs_psql)
902 # kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
903 # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \
904 # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
905 # deleted_jobs = set_jobs_psql.difference(kept_jobs)
906 # deleted_jobs = list(deleted_jobs)
907 # if len(deleted_jobs) > 0:
908 # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
909 # self.iotlab_db.iotlab_session.commit()
914 def GetLeases(self, lease_filter_dict=None, login=None):
917 Get the list of leases from OAR with complete information
918 about which slice owns which jobs and nodes.
920 -Fetch all the jobs from OAR (running, waiting..)
921 complete the reservation information with slice hrn
922 found in iotlab_xp table. If not available in the table,
923 assume it is a iotlab slice.
924 -Updates the iotlab table, deleting jobs when necessary.
926 :returns: reservation_list, list of dictionaries with 'lease_id',
927 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list',
928 'slice_hrn', 'resource_ids', 't_from', 't_until'
933 unfiltered_reservation_list = self.GetReservedNodes()
935 reservation_list = []
936 #Find the slice associated with this user iotlab ldap uid
937 logger.debug(" IOTLAB_API.PY \tGetLeases login %s\
938 unfiltered_reservation_list %s "
939 % (login, unfiltered_reservation_list))
940 #Create user dict first to avoid looking several times for
941 #the same user in LDAP SA 27/07/12
944 jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all()
945 jobs_psql_dict = dict([(row.job_id, row.__dict__)
946 for row in jobs_psql_query])
947 #jobs_psql_dict = jobs_psql_dict)
948 logger.debug("IOTLAB_API \tGetLeases jobs_psql_dict %s"
950 jobs_psql_id_list = [row.job_id for row in jobs_psql_query]
952 for resa in unfiltered_reservation_list:
953 logger.debug("IOTLAB_API \tGetLeases USER %s"
955 #Construct list of jobs (runing, waiting..) in oar
956 job_oar_list.append(resa['lease_id'])
957 #If there is information on the job in IOTLAB DB ]
958 #(slice used and job id)
959 if resa['lease_id'] in jobs_psql_dict:
960 job_info = jobs_psql_dict[resa['lease_id']]
961 logger.debug("IOTLAB_API \tGetLeases job_info %s"
963 resa['slice_hrn'] = job_info['slice_hrn']
964 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
966 #otherwise, assume it is a iotlab slice:
968 resa['slice_id'] = hrn_to_urn(self.root_auth + '.' +
969 resa['user'] + "_slice", 'slice')
970 resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
972 resa['component_id_list'] = []
973 #Transform the hostnames into urns (component ids)
974 for node in resa['reserved_nodes']:
976 iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
977 resa['component_id_list'].append(iotlab_xrn.urn)
979 if lease_filter_dict:
980 logger.debug("IOTLAB_API \tGetLeases resa_ %s \
981 \r\n leasefilter %s" % (resa, lease_filter_dict))
983 if lease_filter_dict['name'] == resa['slice_hrn']:
984 reservation_list.append(resa)
986 if lease_filter_dict is None:
987 reservation_list = unfiltered_reservation_list
989 self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list)
991 logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s"
992 % (reservation_list))
993 return reservation_list
998 #TODO FUNCTIONS SECTION 04/07/2012 SA
1000 ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
1003 #def UnBindObjectFromPeer( auth, object_type, object_id, shortname):
1004 #""" This method is a hopefully temporary hack to let the sfa correctly
1005 #detach the objects it creates from a remote peer object. This is
1006 #needed so that the sfa federation link can work in parallel with
1007 #RefreshPeer, as RefreshPeer depends on remote objects being correctly
1010 #auth : struct, API authentication structure
1011 #AuthMethod : string, Authentication method to use
1012 #object_type : string, Object type, among 'site','person','slice',
1014 #object_id : int, object_id
1015 #shortname : string, peer shortname
1019 #logger.warning("IOTLAB_API \tUnBindObjectFromPeer EMPTY-\
1023 ##TODO Is BindObjectToPeer still necessary ? Currently does nothing
1025 #|| Commented out 28/05/13 SA
1026 #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
1027 #remote_object_id=None):
1028 #"""This method is a hopefully temporary hack to let the sfa correctly
1029 #attach the objects it creates to a remote peer object. This is needed
1030 #so that the sfa federation link can work in parallel with RefreshPeer,
1031 #as RefreshPeer depends on remote objects being correctly marked.
1033 #shortname : string, peer shortname
1034 #remote_object_id : int, remote object_id, set to 0 if unknown
1038 #logger.warning("IOTLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
1041 ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA
1042 ##Funciton should delete and create another job since oin iotlab slice=job
1043 #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
1044 #"""Updates the parameters of an existing slice with the values in
1046 #Users may only update slices of which they are members.
1047 #PIs may update any of the slices at their sites, or any slices of
1048 #which they are members. Admins may update any slice.
1049 #Only PIs and admins may update max_nodes. Slices cannot be renewed
1050 #(by updating the expires parameter) more than 8 weeks into the future.
1051 #Returns 1 if successful, faults otherwise.
1055 #logger.warning("IOTLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ")
1058 #Unused SA 30/05/13, we only update the user's key or we delete it.
1059 ##TODO UpdatePerson 04/07/2012 SA
1060 #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None):
1061 #"""Updates a person. Only the fields specified in person_fields
1062 #are updated, all other fields are left untouched.
1063 #Users and techs can only update themselves. PIs can only update
1064 #themselves and other non-PIs at their sites.
1065 #Returns 1 if successful, faults otherwise.
1069 ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
1070 ##self.iotlab_db.iotlab_session.add(new_row)
1071 ##self.iotlab_db.iotlab_session.commit()
1073 #logger.debug("IOTLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
1077 def GetKeys(key_filter=None):
1078 """Returns a dict of dict based on the key string. Each dict entry
1079 contains the key id, the ssh key, the user's email and the
1081 If key_filter is specified and is an array of key identifiers,
1082 only keys matching the filter will be returned.
1084 Admin may query all keys. Non-admins may only query their own keys.
1087 :returns: dict with ssh key as key and dicts as value.
1090 if key_filter is None:
1091 keys = dbsession.query(RegKey).options(joinedload('reg_user')).all()
1093 keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all()
1097 key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
1098 'email': key.reg_user.email,
1099 'hrn': key.reg_user.hrn}
1101 #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
1102 #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
1103 #for user in ldap_rslt)
1105 logger.debug("IOTLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
1109 def DeleteKey(self, user_record, key_string):
1110 """Deletes a key in the LDAP entry of the specified user.
1112 Removes the key_string from the user's key list and updates the LDAP
1113 user's entry with the new key attributes.
1115 :param key_string: The ssh key to remove
1116 :param user_record: User's record
1117 :type key_string: string
1118 :type user_record: dict
1119 :returns: True if sucessful, False if not.
1123 all_user_keys = user_record['keys']
1124 all_user_keys.remove(key_string)
1125 new_attributes = {'sshPublicKey':all_user_keys}
1126 ret = self.ldap.LdapModifyUser(user_record, new_attributes)
1127 logger.debug("IOTLAB_API DeleteKey %s- " % (ret))
1134 def _sql_get_slice_info(slice_filter):
1136 Get the slice record based on the slice hrn. Fetch the record of the
1137 user associated with the slice by using joinedload based on the
1138 reg_researcher relationship.
1140 :param slice_filter: the slice hrn we are looking for
1141 :type slice_filter: string
1142 :returns: the slice record enhanced with the user's information if the
1143 slice was found, None it wasn't.
1145 :rtype: dict or None.
1147 #DO NOT USE RegSlice - reg_researchers to get the hrn
1148 #of the user otherwise will mess up the RegRecord in
1149 #Resolve, don't know why - SA 08/08/2012
1151 #Only one entry for one user = one slice in iotlab_xp table
1152 #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1153 raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
1154 #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1156 #load_reg_researcher
1157 #raw_slicerec.reg_researchers
1158 raw_slicerec = raw_slicerec.__dict__
1159 logger.debug(" IOTLAB_API \t _sql_get_slice_info slice_filter %s \
1160 raw_slicerec %s" % (slice_filter, raw_slicerec))
1161 slicerec = raw_slicerec
1162 #only one researcher per slice so take the first one
1163 #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
1164 #del slicerec['reg_researchers']['_sa_instance_state']
1171 def _sql_get_slice_info_from_user(slice_filter):
1173 Get the slice record based on the user recordid by using a joinedload
1174 on the relationship reg_slices_as_researcher. Format the sql record
1175 into a dict with the mandatory fields for user and slice.
1176 :returns: dict with slice record and user record if the record was found
1177 based on the user's id, None if not..
1178 :rtype:dict or None..
1180 #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1181 raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
1182 #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1183 #Put it in correct order
1184 user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1185 'classtype', 'authority', 'gid', 'record_id',
1186 'date_created', 'type', 'email', 'pointer']
1187 slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
1188 'classtype', 'authority', 'gid', 'record_id',
1189 'date_created', 'type', 'pointer']
1191 #raw_slicerec.reg_slices_as_researcher
1192 raw_slicerec = raw_slicerec.__dict__
1195 dict([(k, raw_slicerec[
1196 'reg_slices_as_researcher'][0].__dict__[k])
1197 for k in slice_needed_fields])
1198 slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
1199 for k in user_needed_fields])
1200 #TODO Handle multiple slices for one user SA 10/12/12
1201 #for now only take the first slice record associated to the rec user
1202 ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
1203 #del raw_slicerec['reg_slices_as_researcher']
1204 #slicerec['reg_researchers'] = raw_slicerec
1205 ##del slicerec['_sa_instance_state']
1212 def _get_slice_records(self, slice_filter=None,
1213 slice_filter_type=None):
1215 Get the slice record depending on the slice filter and its type.
1216 :param slice_filter: Can be either the slice hrn or the user's record
1218 :type slice_filter: string
1219 :param slice_filter_type: describes the slice filter type used, can be
1220 slice_hrn or record_id_user
1222 :returns: the slice record
1224 .. seealso::_sql_get_slice_info_from_user
1225 .. seealso:: _sql_get_slice_info
1228 #Get list of slices based on the slice hrn
1229 if slice_filter_type == 'slice_hrn':
1231 #if get_authority(slice_filter) == self.root_auth:
1232 #login = slice_filter.split(".")[1].split("_")[0]
1234 slicerec = self._sql_get_slice_info(slice_filter)
1236 if slicerec is None:
1240 #Get slice based on user id
1241 if slice_filter_type == 'record_id_user':
1243 slicerec = self._sql_get_slice_info_from_user(slice_filter)
1246 fixed_slicerec_dict = slicerec
1247 #At this point if there is no login it means
1248 #record_id_user filter has been used for filtering
1250 ##If theslice record is from iotlab
1251 #if fixed_slicerec_dict['peer_authority'] is None:
1252 #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
1253 #return login, fixed_slicerec_dict
1254 return fixed_slicerec_dict
1259 def GetSlices(self, slice_filter=None, slice_filter_type=None,
1261 """Get the slice records from the iotlab db and add lease information
1264 :param slice_filter: can be the slice hrn or slice record id in the db
1265 depending on the slice_filter_type.
1266 :param slice_filter_type: defines the type of the filtering used, Can be
1267 either 'slice_hrn' or "record_id'.
1268 :type slice_filter: string
1269 :type slice_filter_type: string
1270 :returns: a slice dict if slice_filter and slice_filter_type
1271 are specified and a matching entry is found in the db. The result
1272 is put into a list.Or a list of slice dictionnaries if no filters
1279 authorized_filter_types_list = ['slice_hrn', 'record_id_user']
1280 return_slicerec_dictlist = []
1282 #First try to get information on the slice based on the filter provided
1283 if slice_filter_type in authorized_filter_types_list:
1284 fixed_slicerec_dict = self._get_slice_records(slice_filter,
1286 # if the slice was not found in the sfa db
1287 if fixed_slicerec_dict is None:
1288 return return_slicerec_dictlist
1290 slice_hrn = fixed_slicerec_dict['hrn']
1292 logger.debug(" IOTLAB_API \tGetSlices login %s \
1293 slice record %s slice_filter %s \
1294 slice_filter_type %s " % (login,
1295 fixed_slicerec_dict, slice_filter,
1299 #Now we have the slice record fixed_slicerec_dict, get the
1300 #jobs associated to this slice
1303 leases_list = self.GetLeases(login=login)
1304 #If no job is running or no job scheduled
1305 #return only the slice record
1306 if leases_list == [] and fixed_slicerec_dict:
1307 return_slicerec_dictlist.append(fixed_slicerec_dict)
1309 # if the jobs running don't belong to the user/slice we are looking
1311 leases_hrn = [lease['slice_hrn'] for lease in leases_list]
1312 if slice_hrn not in leases_hrn:
1313 return_slicerec_dictlist.append(fixed_slicerec_dict)
1314 #If several jobs for one slice , put the slice record into
1315 # each lease information dict
1316 for lease in leases_list:
1318 logger.debug("IOTLAB_API.PY \tGetSlices slice_filter %s \
1319 \t lease['slice_hrn'] %s"
1320 % (slice_filter, lease['slice_hrn']))
1321 if lease['slice_hrn'] == slice_hrn:
1322 slicerec_dict['oar_job_id'] = lease['lease_id']
1323 #Update lease dict with the slice record
1324 if fixed_slicerec_dict:
1325 fixed_slicerec_dict['oar_job_id'] = []
1326 fixed_slicerec_dict['oar_job_id'].append(
1327 slicerec_dict['oar_job_id'])
1328 slicerec_dict.update(fixed_slicerec_dict)
1329 #slicerec_dict.update({'hrn':\
1330 #str(fixed_slicerec_dict['slice_hrn'])})
1331 slicerec_dict['slice_hrn'] = lease['slice_hrn']
1332 slicerec_dict['hrn'] = lease['slice_hrn']
1333 slicerec_dict['user'] = lease['user']
1334 slicerec_dict.update(
1336 {'hostname': lease['reserved_nodes']}})
1337 slicerec_dict.update({'node_ids': lease['reserved_nodes']})
1341 return_slicerec_dictlist.append(slicerec_dict)
1342 logger.debug("IOTLAB_API.PY \tGetSlices \
1343 OHOHOHOH %s" %(return_slicerec_dictlist))
1345 logger.debug("IOTLAB_API.PY \tGetSlices \
1346 slicerec_dict %s return_slicerec_dictlist %s \
1347 lease['reserved_nodes'] \
1348 %s" % (slicerec_dict, return_slicerec_dictlist,
1349 lease['reserved_nodes']))
1351 logger.debug("IOTLAB_API.PY \tGetSlices RETURN \
1352 return_slicerec_dictlist %s"
1353 % (return_slicerec_dictlist))
1355 return return_slicerec_dictlist
1359 #Get all slices from the iotlab sfa database ,
1360 #put them in dict format
1361 #query_slice_list = dbsession.query(RegRecord).all()
1362 query_slice_list = \
1363 dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
1365 for record in query_slice_list:
1366 tmp = record.__dict__
1367 tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
1368 #del tmp['reg_researchers']['_sa_instance_state']
1369 return_slicerec_dictlist.append(tmp)
1370 #return_slicerec_dictlist.append(record.__dict__)
1372 #Get all the jobs reserved nodes
1373 leases_list = self.GetReservedNodes()
1375 for fixed_slicerec_dict in return_slicerec_dictlist:
1377 #Check if the slice belongs to a iotlab user
1378 if fixed_slicerec_dict['peer_authority'] is None:
1379 owner = fixed_slicerec_dict['hrn'].split(
1380 ".")[1].split("_")[0]
1383 for lease in leases_list:
1384 if owner == lease['user']:
1385 slicerec_dict['oar_job_id'] = lease['lease_id']
1387 #for reserved_node in lease['reserved_nodes']:
1388 logger.debug("IOTLAB_API.PY \tGetSlices lease %s "
1390 slicerec_dict.update(fixed_slicerec_dict)
1391 slicerec_dict.update({'node_ids':
1392 lease['reserved_nodes']})
1393 slicerec_dict.update({'list_node_ids':
1395 lease['reserved_nodes']}})
1397 #slicerec_dict.update({'hrn':\
1398 #str(fixed_slicerec_dict['slice_hrn'])})
1399 #return_slicerec_dictlist.append(slicerec_dict)
1400 fixed_slicerec_dict.update(slicerec_dict)
1402 logger.debug("IOTLAB_API.PY \tGetSlices RETURN \
1403 return_slicerec_dictlist %s \slice_filter %s " \
1404 %(return_slicerec_dictlist, slice_filter))
1406 return return_slicerec_dictlist
1410 #Update slice unused, therefore sfa_fields_to_iotlab_fields unused
1413 #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record):
1418 ##for field in record:
1419 ## iotlab_record[field] = record[field]
1421 #if sfa_type == "slice":
1422 ##instantion used in get_slivers ?
1423 #if not "instantiation" in iotlab_record:
1424 #iotlab_record["instantiation"] = "iotlab-instantiated"
1425 ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn)
1426 ##Unused hrn_to_pl_slicename because Iotlab's hrn already
1427 ##in the appropriate form SA 23/07/12
1428 #iotlab_record["hrn"] = hrn
1429 #logger.debug("IOTLAB_API.PY sfa_fields_to_iotlab_fields \
1430 #iotlab_record %s " %(iotlab_record['hrn']))
1431 #if "url" in record:
1432 #iotlab_record["url"] = record["url"]
1433 #if "description" in record:
1434 #iotlab_record["description"] = record["description"]
1435 #if "expires" in record:
1436 #iotlab_record["expires"] = int(record["expires"])
1438 ##nodes added by OAR only and then imported to SFA
1439 ##elif type == "node":
1440 ##if not "hostname" in iotlab_record:
1441 ##if not "hostname" in record:
1442 ##raise MissingSfaInfo("hostname")
1443 ##iotlab_record["hostname"] = record["hostname"]
1444 ##if not "model" in iotlab_record:
1445 ##iotlab_record["model"] = "geni"
1447 ##One authority only
1448 ##elif type == "authority":
1449 ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn)
1451 ##if not "name" in iotlab_record:
1452 ##iotlab_record["name"] = hrn
1454 ##if not "abbreviated_name" in iotlab_record:
1455 ##iotlab_record["abbreviated_name"] = hrn
1457 ##if not "enabled" in iotlab_record:
1458 ##iotlab_record["enabled"] = True
1460 ##if not "is_public" in iotlab_record:
1461 ##iotlab_record["is_public"] = True
1463 #return iotlab_record