1 from datetime import datetime
3 from sfa.util.sfalogging import logger
5 from sfa.storage.alchemy import dbsession
6 from sqlalchemy.orm import joinedload
7 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
8 from sfa.iotlab.iotlabpostgres import iotlab_dbsession, IotlabXP
10 from sfa.iotlab.OARrestapi import OARrestapi
11 from sfa.iotlab.LDAPapi import LDAPapi
13 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
15 from sfa.trust.certificate import Keypair, convert_public_key
16 from sfa.trust.gid import create_uuid
17 from sfa.trust.hierarchy import Hierarchy
18 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
21 from sfa.senslab.slabaggregate import slab_xrn_object
22 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
24 from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
26 class IotlabTestbedAPI():
27 """ Class enabled to use LDAP and OAR api calls. """
29 def __init__(self, config):
30 """Creates an instance of OARrestapi and LDAPapi which will be used to
31 issue calls to OAR or LDAP methods.
32 Set the time format and the testbed granularity used for OAR
33 reservation and leases.
35 :param config: configuration object from sfa.util.config
36 :type config: Config object
38 self.oar = OARrestapi()
40 self.time_format = "%Y-%m-%d %H:%M:%S"
41 self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
42 self.grain = 1 # 10 mins lease minimum
43 #import logging, logging.handlers
44 #from sfa.util.sfalogging import _SfaLogger
45 #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \
51 def GetMinExperimentDurationInSec():
55 def GetPeers (peer_filter=None ):
56 """ Gathers registered authorities in SFA DB and looks for specific peer
57 if peer_filter is specified.
58 :param peer_filter: name of the site authority looked for.
59 :type peer_filter: string
60 :return: list of records.
65 existing_hrns_by_types = {}
66 logger.debug("IOTLABDRIVER \tGetPeers peer_filter %s, \
68 all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
70 for record in all_records:
71 existing_records[(record.hrn, record.type)] = record
72 if record.type not in existing_hrns_by_types:
73 existing_hrns_by_types[record.type] = [record.hrn]
75 existing_hrns_by_types[record.type].append(record.hrn)
78 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
79 logger.debug("IOTLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\
81 logger.debug("SLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\
82 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
83 %( existing_hrns_by_types))
88 records_list.append(existing_records[(peer_filter,'authority')])
90 for hrn in existing_hrns_by_types['authority']:
91 records_list.append(existing_records[(hrn,'authority')])
93 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
94 logger.debug("IOTLABDRIVER \tGetPeer \trecords_list %s " \
96 logger.debug("SLABDRIVER \tGetPeer \trecords_list %s " \
97 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
103 return_records = records_list
104 logger.debug("IOTLABDRIVER \tGetPeer return_records %s " \
106 return return_records
110 #TODO : Handling OR request in make_ldap_filters_from_records
111 #instead of the for loop
112 #over the records' list
113 def GetPersons(self, person_filter=None):
115 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
116 Get the enabled users and their properties from Iotlab LDAP.
118 Get the enabled users and their properties from Senslab LDAP.
119 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
120 If a filter is specified, looks for the user whose properties match
121 the filter, otherwise returns the whole enabled users'list.
122 :param person_filter: Must be a list of dictionnaries
123 with users properties when not set to None.
124 :param person_filter: list of dict
125 :return:Returns a list of users whose accounts are enabled
127 :rtype: list of dicts
130 logger.debug("IOTLABDRIVER \tGetPersons person_filter %s" \
133 if person_filter and isinstance(person_filter, list):
134 #If we are looking for a list of users (list of dict records)
135 #Usually the list contains only one user record
136 for searched_attributes in person_filter:
138 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
139 #Get only enabled user accounts in iotlab LDAP :
141 #Get only enabled user accounts in senslab LDAP :
142 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
143 #add a filter for make_ldap_filters_from_record
144 person = self.ldap.LdapFindUser(searched_attributes, \
145 is_user_enabled=True)
146 #If a person was found, append it to the list
148 person_list.append(person)
150 #If the list is empty, return None
151 if len(person_list) is 0:
155 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
156 #Get only enabled user accounts in iotlab LDAP :
158 #Get only enabled user accounts in senslab LDAP :
159 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
160 #add a filter for make_ldap_filters_from_record
161 person_list = self.ldap.LdapFindUser(is_user_enabled=True)
166 #def GetTimezone(self):
167 #""" Returns the OAR server time and timezone.
168 #Unused SA 30/05/13"""
169 #server_timestamp, server_tz = self.oar.parser.\
170 #SendRequest("GET_timezone")
171 #return server_timestamp, server_tz
173 def DeleteJobs(self, job_id, username):
175 """ Deletes the job with the specified job_id and username on OAR by
176 posting a delete request to OAR.
178 :param job_id: job id in OAR.
179 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
180 :param username: user's iotlab login in LDAP.
182 :param username: user's senslab login in LDAP.
183 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
185 :type username: string
187 :return: dictionary with the job id and if delete has been successful
191 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
192 logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s username %s "\
194 logger.debug("SLABDRIVER \tDeleteJobs jobid %s username %s "\
195 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
197 if not job_id or job_id is -1:
201 reqdict['method'] = "delete"
202 reqdict['strval'] = str(job_id)
205 answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', \
207 if answer['status'] == 'Delete request registered':
208 ret = {job_id : True }
210 ret = {job_id :False }
211 logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s \r\n answer %s \
212 username %s" %(job_id, answer, username))
217 ##TODO : Unused GetJobsId ? SA 05/07/12
218 #def GetJobsId(self, job_id, username = None ):
220 #Details about a specific job.
221 #Includes details about submission time, jot type, state, events,
222 #owner, assigned ressources, walltime etc...
226 #node_list_k = 'assigned_network_address'
227 ##Get job info from OAR
228 #job_info = self.oar.parser.SendRequest(req, job_id, username)
230 #logger.debug("IOTLABDRIVER \t GetJobsId %s " %(job_info))
232 #if job_info['state'] == 'Terminated':
233 #logger.debug("IOTLABDRIVER \t GetJobsId job %s TERMINATED"\
236 #if job_info['state'] == 'Error':
237 #logger.debug("IOTLABDRIVER \t GetJobsId ERROR message %s "\
242 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
243 #logger.error("IOTLABDRIVER \tGetJobsId KeyError")
245 #logger.error("SLABDRIVER \tGetJobsId KeyError")
246 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
249 #parsed_job_info = self.get_info_on_reserved_nodes(job_info, \
251 ##Replaces the previous entry
252 ##"assigned_network_address" / "reserved_resources"
254 #job_info.update({'node_ids':parsed_job_info[node_list_k]})
255 #del job_info[node_list_k]
256 #logger.debug(" \r\nIOTLABDRIVER \t GetJobsId job_info %s " %(job_info))
260 def GetJobsResources(self, job_id, username = None):
261 """ Gets the list of nodes associated with the job_id and username
263 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
264 Transforms the iotlab hostnames to the corresponding
266 Transforms the senslab hostnames to the corresponding
267 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
269 Rertuns dict key :'node_ids' , value : hostnames list
270 :param username: user's LDAP login
271 :paran job_id: job's OAR identifier.
272 :type username: string
273 :type job_id: integer
275 :return: dicionary with nodes' hostnames belonging to the job.
279 req = "GET_jobs_id_resources"
282 #Get job resources list from OAR
283 node_id_list = self.oar.parser.SendRequest(req, job_id, username)
284 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
285 logger.debug("IOTLABDRIVER \t GetJobsResources %s " %(node_id_list))
287 logger.debug("SLABDRIVER \t GetJobsResources %s " %(node_id_list))
288 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
291 self.__get_hostnames_from_oar_node_ids(node_id_list)
294 #Replaces the previous entry "assigned_network_address" /
295 #"reserved_resources" with "node_ids"
296 job_info = {'node_ids': hostname_list}
301 #def get_info_on_reserved_nodes(self, job_info, node_list_name):
303 #..warning:unused SA 23/05/13
305 ##Get the list of the testbed nodes records and make a
306 ##dictionnary keyed on the hostname out of it
307 #node_list_dict = self.GetNodes()
308 ##node_hostname_list = []
309 #node_hostname_list = [node['hostname'] for node in node_list_dict]
310 ##for node in node_list_dict:
311 ##node_hostname_list.append(node['hostname'])
312 #node_dict = dict(zip(node_hostname_list, node_list_dict))
314 #reserved_node_hostname_list = []
315 #for index in range(len(job_info[node_list_name])):
316 ##job_info[node_list_name][k] =
317 #reserved_node_hostname_list[index] = \
318 #node_dict[job_info[node_list_name][index]]['hostname']
320 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
321 #logger.debug("IOTLABDRIVER \t get_info_on_reserved_nodes \
322 #reserved_node_hostname_list %s" \
323 #%(reserved_node_hostname_list))
325 #logger.error("IOTLABDRIVER \t get_info_on_reserved_nodes KEYERROR " )
327 #logger.debug("SLABDRIVER \t get_info_on_reserved_nodes \
328 #reserved_node_hostname_list %s" \
329 #%(reserved_node_hostname_list))
331 #logger.error("SLABDRIVER \t get_info_on_reserved_nodes KEYERROR " )
332 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
334 #return reserved_node_hostname_list
336 def GetNodesCurrentlyInUse(self):
337 """Returns a list of all the nodes already involved in an oar running
339 :rtype: list of nodes hostnames.
341 return self.oar.parser.SendRequest("GET_running_jobs")
343 def __get_hostnames_from_oar_node_ids(self, resource_id_list ):
344 """Get the hostnames of the nodes from their OAR identifiers.
345 Get the list of nodes dict using GetNodes and find the hostname
346 associated with the identifier.
347 :param resource_id_list: list of nodes identifiers
348 :returns: list of node hostnames.
350 full_nodes_dict_list = self.GetNodes()
351 #Put the full node list into a dictionary keyed by oar node id
352 oar_id_node_dict = {}
353 for node in full_nodes_dict_list:
354 oar_id_node_dict[node['oar_id']] = node
357 for resource_id in resource_id_list:
358 #Because jobs requested "asap" do not have defined resources
359 if resource_id is not "Undefined":
360 hostname_list.append(\
361 oar_id_node_dict[resource_id]['hostname'])
363 #hostname_list.append(oar_id_node_dict[resource_id]['hostname'])
366 def GetReservedNodes(self, username = None):
367 """ Get list of leases. Get the leases for the username if specified,
368 otherwise get all the leases. Finds the nodes hostnames for each
370 :param username: user's LDAP login
371 :type username: string
372 :return: list of reservations dict
376 #Get the nodes in use and the reserved nodes
377 reservation_dict_list = \
378 self.oar.parser.SendRequest("GET_reserved_nodes", \
382 for resa in reservation_dict_list:
383 logger.debug ("GetReservedNodes resa %s"%(resa))
384 #dict list of hostnames and their site
385 resa['reserved_nodes'] = \
386 self.__get_hostnames_from_oar_node_ids(resa['resource_ids'])
388 #del resa['resource_ids']
389 return reservation_dict_list
391 def GetNodes(self, node_filter_dict = None, return_fields_list = None):
394 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
395 Make a list of iotlab nodes and their properties from information
397 Make a list of senslab nodes and their properties from information
398 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
399 given by OAR. Search for specific nodes if some filters are specified.
400 Nodes properties returned if no return_fields_list given:
401 'hrn','archi','mobile','hostname','site','boot_state','node_id',
402 'radio','posx','posy','oar_id','posz'.
404 :param node_filter_dict: dictionnary of lists with node properties
405 :type node_filter_dict: dict
406 :param return_fields_list: list of specific fields the user wants to be
408 :type return_fields_list: list
409 :return: list of dictionaries with node properties
413 node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
414 node_dict_list = node_dict_by_id.values()
415 logger.debug (" IOTLABDRIVER GetNodes node_filter_dict %s \
416 return_fields_list %s "%(node_filter_dict, return_fields_list))
417 #No filtering needed return the list directly
418 if not (node_filter_dict or return_fields_list):
419 return node_dict_list
421 return_node_list = []
423 for filter_key in node_filter_dict:
425 #Filter the node_dict_list by each value contained in the
426 #list node_filter_dict[filter_key]
427 for value in node_filter_dict[filter_key]:
428 for node in node_dict_list:
429 if node[filter_key] == value:
430 if return_fields_list :
432 for k in return_fields_list:
434 return_node_list.append(tmp)
436 return_node_list.append(node)
438 logger.log_exc("GetNodes KeyError")
442 return return_node_list
447 def AddSlice(slice_record, user_record):
448 """Add slice to the local iotlab sfa tables if the slice comes
449 from a federated site and is not yet in the iotlab sfa DB,
450 although the user has already a LDAP login.
451 Called by verify_slice during lease/sliver creation.
452 :param slice_record: record of slice, must contain hrn, gid, slice_id
453 and authority of the slice.
454 :type slice_record: dictionary
455 :param user_record: record of the user
456 :type user_record: RegUser
459 sfa_record = RegSlice(hrn=slice_record['hrn'],
460 gid=slice_record['gid'],
461 pointer=slice_record['slice_id'],
462 authority=slice_record['authority'])
463 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
464 logger.debug("IOTLABDRIVER.PY AddSlice sfa_record %s user_record %s"
466 logger.debug("SLABDRIVER.PY AddSlice sfa_record %s user_record %s"
467 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
468 % (sfa_record, user_record))
469 sfa_record.just_created()
470 dbsession.add(sfa_record)
472 #Update the reg-researcher dependance table
473 sfa_record.reg_researchers = [user_record]
479 def GetSites(self, site_filter_name_list = None, return_fields_list = None):
480 site_dict = self.oar.parser.SendRequest("GET_sites")
481 #site_dict : dict where the key is the sit ename
482 return_site_list = []
483 if not ( site_filter_name_list or return_fields_list):
484 return_site_list = site_dict.values()
485 return return_site_list
487 for site_filter_name in site_filter_name_list:
488 if site_filter_name in site_dict:
489 if return_fields_list:
490 for field in return_fields_list:
493 tmp[field] = site_dict[site_filter_name][field]
495 logger.error("GetSites KeyError %s "%(field))
497 return_site_list.append(tmp)
499 return_site_list.append( site_dict[site_filter_name])
502 return return_site_list
508 #TODO : Check rights to delete person
509 def DeletePerson(self, person_record):
510 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
511 """ Disable an existing account in iotlab LDAP.
513 """ Disable an existing account in senslab LDAP.
514 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
515 Users and techs can only delete themselves. PIs can only
516 delete themselves and other non-PIs at their sites.
517 ins can delete anyone.
518 :param person_record: user's record
519 :type person_record: dict
520 :return: True if successful, False otherwise.
524 #Disable user account in iotlab LDAP
525 ret = self.ldap.LdapMarkUserAsDeleted(person_record)
526 logger.warning("IOTLABDRIVER DeletePerson %s " %(person_record))
530 def DeleteSlice(self, slice_record):
531 """ Deletes the specified slice and kills the jobs associated with
532 the slice if any, using DeleteSliceFromNodes.
534 :return: True if all the jobs in the slice have been deleted,
535 or the list of jobs that could not be deleted otherwise.
536 :rtype: list or boolean
539 ret = self.DeleteSliceFromNodes(slice_record)
542 if False in ret[job_id]:
543 if delete_failed is None:
545 delete_failed.append(job_id)
547 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
548 logger.info("IOTLABDRIVER DeleteSlice %s answer %s"%(slice_record, \
550 logger.info("SLABDRIVER DeleteSlice %s answer %s"%(slice_record, \
551 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
553 return delete_failed or True
556 def __add_person_to_db(user_dict):
558 Add a federated user straight to db when the user issues a lease
559 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
560 request with iotlab nodes and that he has not registered with iotlab
562 request with senslab nodes and that he has not registered with senslab
563 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
564 yet (that is he does not have a LDAP entry yet).
565 Uses parts of the routines in SlabImport when importing user from LDAP.
566 Called by AddPerson, right after LdapAddUser.
567 :param user_dict: Must contain email, hrn and pkey to get a GID
568 and be added to the SFA db.
569 :type user_dict: dict
573 dbsession.query(RegUser).filter_by(email = user_dict['email']).first()
575 if not check_if_exists:
576 logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
578 hrn = user_dict['hrn']
579 person_urn = hrn_to_urn(hrn, 'user')
580 pubkey = user_dict['pkey']
582 pkey = convert_public_key(pubkey)
584 #key not good. create another pkey
585 logger.warn('__add_person_to_db: unable to convert public \
587 pkey = Keypair(create=True)
590 if pubkey is not None and pkey is not None :
591 hierarchy = Hierarchy()
592 person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
594 if user_dict['email']:
595 logger.debug("__add_person_to_db \r\n \r\n \
596 SLAB IMPORTER PERSON EMAIL OK email %s "\
597 %(user_dict['email']))
598 person_gid.set_email(user_dict['email'])
600 user_record = RegUser(hrn=hrn , pointer= '-1', \
601 authority=get_authority(hrn), \
602 email=user_dict['email'], gid = person_gid)
603 user_record.reg_keys = [RegKey(user_dict['pkey'])]
604 user_record.just_created()
605 dbsession.add (user_record)
610 def AddPerson(self, record):
611 """Adds a new account. Any fields specified in records are used,
612 otherwise defaults are used. Creates an appropriate login by calling
614 :param record: dictionary with the sfa user's properties.
615 :return: The uid of the added person if sucessful, otherwise returns
616 the error message from LDAP.
617 :rtype: interger or string
619 ret = self.ldap.LdapAddUser(record)
621 if ret['bool'] is True:
622 record['hrn'] = self.root_auth + '.' + ret['uid']
623 logger.debug("IOTLABDRIVER AddPerson return code %s record %s \r\n "\
625 self.__add_person_to_db(record)
628 return ret['message']
632 #TODO AddPersonKey 04/07/2012 SA
633 def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
634 """Adds a new key to the specified account. Adds the key to the
635 iotlab ldap, provided that the person_uid is valid.
636 Non-admins can only modify their own keys.
638 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
639 :param person_uid: user's iotlab login in LDAP
641 :param person_uid: user's senslab login in LDAP
642 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
643 :param old_attributes_dict: dict with the user's old sshPublicKey
644 :param new_key_dict:dict with the user's new sshPublicKey
645 :type person_uid: string
649 :return: True if the key has been modified, False otherwise.
652 ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \
654 logger.warning("IOTLABDRIVER AddPersonKey EMPTY - DO NOTHING \r\n ")
657 def DeleteLeases(self, leases_id_list, slice_hrn ):
659 Deletes several leases, based on their job ids and the slice
660 they are associated with. Uses DeleteJobs to delete the jobs
661 on OAR. Note that one slice can contain multiple jobs, and in this case
662 all the jobs in the leases_id_list MUST belong to ONE slice,
663 since there is only one slice hrn provided here.
664 :param leases_id_list: list of job ids that belong to the slice whose
665 slice hrn is provided.
666 :param slice_hrn: the slice hrn .
667 ..warning: Does not have a return value since there was no easy
668 way to handle failure when dealing with multiple job delete. Plus,
669 there was no easy way to report it to the user.
671 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
672 logger.debug("IOTLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \
674 logger.debug("SLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \
675 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
676 \r\n " %(leases_id_list, slice_hrn))
677 for job_id in leases_id_list:
678 self.DeleteJobs(job_id, slice_hrn)
683 def _process_walltime(duration):
684 """ Calculates the walltime in seconds from the duration in H:M:S
685 specified in the RSpec.
689 # Fixing the walltime by adding a few delays.
690 # First put the walltime in seconds oarAdditionalDelay = 20;
691 # additional delay for /bin/sleep command to
692 # take in account prologue and epilogue scripts execution
693 # int walltimeAdditionalDelay = 240; additional delay
694 #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
696 # Put the duration in seconds first
697 #desired_walltime = duration * 60
698 desired_walltime = duration
699 total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
700 sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
702 #Put the walltime back in str form
704 walltime.append(str(total_walltime / 3600))
705 total_walltime = total_walltime - 3600 * int(walltime[0])
706 #Get the remaining minutes
707 walltime.append(str(total_walltime / 60))
708 total_walltime = total_walltime - 60 * int(walltime[1])
710 walltime.append(str(total_walltime))
713 logger.log_exc(" __process_walltime duration null")
715 return walltime, sleep_walltime
718 def _create_job_structure_request_for_OAR(lease_dict):
719 """ Creates the structure needed for a correct POST on OAR.
720 Makes the timestamp transformation into the appropriate format.
721 Sends the POST request to create the job with the resources in
730 reqdict['workdir'] = '/tmp'
731 reqdict['resource'] = "{network_address in ("
733 for node in lease_dict['added_nodes']:
734 logger.debug("\r\n \r\n OARrestapi \t \
735 __create_job_structure_request_for_OAR node %s" %(node))
737 # Get the ID of the node
739 reqdict['resource'] += "'" + nodeid + "', "
740 nodeid_list.append(nodeid)
742 custom_length = len(reqdict['resource'])- 2
743 reqdict['resource'] = reqdict['resource'][0:custom_length] + \
744 ")}/nodes=" + str(len(nodeid_list))
747 walltime, sleep_walltime = \
748 IotlabTestbedAPI._process_walltime(\
749 int(lease_dict['lease_duration']))
752 reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
753 ":" + str(walltime[1]) + ":" + str(walltime[2])
754 reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
756 #In case of a scheduled experiment (not immediate)
757 #To run an XP immediately, don't specify date and time in RSpec
758 #They will be set to None.
759 if lease_dict['lease_start_time'] is not '0':
760 #Readable time accepted by OAR
761 start_time = datetime.fromtimestamp( \
762 int(lease_dict['lease_start_time'])).\
763 strftime(lease_dict['time_format'])
764 reqdict['reservation'] = start_time
765 #If there is not start time, Immediate XP. No need to add special
769 reqdict['type'] = "deploy"
770 reqdict['directory'] = ""
771 reqdict['name'] = "SFA_" + lease_dict['slice_user']
776 def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
777 lease_start_time, lease_duration, slice_user=None):
780 Create a job request structure based on the information provided
781 and post the job on OAR.
782 :param added_nodes: list of nodes that belong to the described lease.
783 :param slice_name: the slice hrn associated to the lease.
784 :param lease_start_time: timestamp of the lease startting time.
785 :param lease_duration: lease durationin minutes
789 lease_dict['lease_start_time'] = lease_start_time
790 lease_dict['lease_duration'] = lease_duration
791 lease_dict['added_nodes'] = added_nodes
792 lease_dict['slice_name'] = slice_name
793 lease_dict['slice_user'] = slice_user
794 lease_dict['grain'] = self.GetLeaseGranularity()
795 lease_dict['time_format'] = self.time_format
798 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
799 logger.debug("IOTLABDRIVER.PY \tLaunchExperimentOnOAR slice_user %s\
801 logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR slice_user %s\
802 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
803 \r\n " %(slice_user))
804 #Create the request for OAR
805 reqdict = self._create_job_structure_request_for_OAR(lease_dict)
806 # first step : start the OAR job and update the job
807 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
808 logger.debug("IOTLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\
810 logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\
811 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
814 answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
816 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
817 logger.debug("IOTLABDRIVER \tLaunchExperimentOnOAR jobid %s " %(answer))
819 logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s " %(answer))
820 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
824 logger.log_exc("IOTLABDRIVER \tLaunchExperimentOnOAR \
825 Impossible to create job %s " %(answer))
832 logger.debug("IOTLABDRIVER \tLaunchExperimentOnOAR jobid %s \
833 added_nodes %s slice_user %s" %(jobid, added_nodes, \
840 def AddLeases(self, hostname_list, slice_record, \
841 lease_start_time, lease_duration):
843 """Creates a job in OAR corresponding to the information provided
844 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
845 as parameters. Adds the job id and the slice hrn in the iotlab
847 as parameters. Adds the job id and the slice hrn in the senslab
848 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
849 database so that we are able to know which slice has which nodes.
851 :param hostname_list: list of nodes' OAR hostnames.
852 :param slice_record: sfa slice record, must contain login and hrn.
853 :param lease_start_time: starting time , unix timestamp format
854 :param lease_duration: duration in minutes
856 :type hostname_list: list
857 :type slice_record: dict
858 :type lease_start_time: integer
859 :type lease_duration: integer
862 logger.debug("IOTLABDRIVER \r\n \r\n \t AddLeases hostname_list %s \
863 slice_record %s lease_start_time %s lease_duration %s "\
864 %( hostname_list, slice_record , lease_start_time, \
867 #tmp = slice_record['reg-researchers'][0].split(".")
868 username = slice_record['login']
869 #username = tmp[(len(tmp)-1)]
870 job_id = self.LaunchExperimentOnOAR(hostname_list, \
871 slice_record['hrn'], \
872 lease_start_time, lease_duration, \
875 datetime.fromtimestamp(int(lease_start_time)).\
876 strftime(self.time_format)
877 end_time = lease_start_time + lease_duration
880 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
881 logger.debug("IOTLABDRIVER \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
882 %s %s %s "%(slice_record['hrn'], job_id, end_time))
885 logger.debug("IOTLABDRIVER \r\n \r\n \t AddLeases %s %s %s " \
886 %(type(slice_record['hrn']), type(job_id), type(end_time)))
888 iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], \
889 job_id = job_id, end_time= end_time)
891 logger.debug("IOTLABDRIVER \r\n \r\n \t AddLeases iotlab_ex_row %s" \
893 iotlab_dbsession.add(iotlab_ex_row)
894 iotlab_dbsession.commit()
896 logger.debug("IOTLABDRIVER \t AddLeases hostname_list start_time %s " \
898 logger.debug("SLABDRIVER \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
899 %s %s %s "%(slice_record['hrn'], job_id, end_time))
902 logger.debug("SLABDRIVER \r\n \r\n \t AddLeases %s %s %s " \
903 %(type(slice_record['hrn']), type(job_id), type(end_time)))
905 slab_ex_row = SenslabXP(slice_hrn = slice_record['hrn'], \
906 job_id = job_id, end_time= end_time)
908 logger.debug("SLABDRIVER \r\n \r\n \t AddLeases slab_ex_row %s" \
910 slab_dbsession.add(slab_ex_row)
911 slab_dbsession.commit()
913 logger.debug("SLABDRIVER \t AddLeases hostname_list start_time %s " \
914 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
920 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
921 #Delete the jobs from job_iotlab table
923 #Delete the jobs from job_senslab table
924 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
925 def DeleteSliceFromNodes(self, slice_record):
926 """ Deletes all the running or scheduled jobs of a given slice
928 :param slice_record: record of the slice
929 :type slice_record: dict
931 :return: dict of the jobs'deletion status. Success= True, Failure=
932 False, for each job id.
935 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
936 logger.debug("IOTLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record))
938 logger.debug("SLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record))
939 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
941 if isinstance(slice_record['oar_job_id'], list):
943 for job_id in slice_record['oar_job_id']:
944 ret = self.DeleteJobs(job_id, slice_record['user'])
946 oar_bool_answer.update(ret)
949 oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], \
950 slice_record['user'])]
952 return oar_bool_answer
956 def GetLeaseGranularity(self):
957 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
958 """ Returns the granularity of an experiment in the Iotlab testbed.
960 """ Returns the granularity of an experiment in the Senslab testbed.
961 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
962 OAR uses seconds for experiments duration , the granulaity is also
964 Experiments which last less than 10 min (600 sec) are invalid"""
969 def update_jobs_in_iotlabdb( job_oar_list, jobs_psql):
970 """ Cleans the iotlab db by deleting expired and cancelled jobs.
971 Compares the list of job ids given by OAR with the job ids that
972 are already in the database, deletes the jobs that are no longer in
974 :param job_oar_list: list of job ids coming from OAR
975 :type job_oar_list: list
976 :param job_psql: list of job ids cfrom the database.
979 #Turn the list into a set
980 set_jobs_psql = set(jobs_psql)
982 kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
983 logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \
984 job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
985 deleted_jobs = set_jobs_psql.difference(kept_jobs)
986 deleted_jobs = list(deleted_jobs)
987 if len(deleted_jobs) > 0:
988 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
989 iotlab_dbsession.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
990 iotlab_dbsession.commit()
992 slab_dbsession.query(SenslabXP).filter(SenslabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
993 slab_dbsession.commit()
994 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1000 def GetLeases(self, lease_filter_dict=None, login=None):
1001 """ Get the list of leases from OAR with complete information
1002 about which slice owns which jobs and nodes.
1004 -Fetch all the jobs from OAR (running, waiting..)
1005 complete the reservation information with slice hrn
1006 found in iotlab_xp table. If not available in the table,
1007 assume it is a iotlab slice.
1008 -Updates the iotlab table, deleting jobs when necessary.
1009 :return: reservation_list, list of dictionaries with 'lease_id',
1010 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list',
1011 'slice_hrn', 'resource_ids', 't_from', 't_until'
1015 unfiltered_reservation_list = self.GetReservedNodes(login)
1017 reservation_list = []
1018 #Find the slice associated with this user iotlab ldap uid
1019 logger.debug(" IOTLABDRIVER.PY \tGetLeases login %s\
1020 unfiltered_reservation_list %s " %(login, unfiltered_reservation_list))
1021 #Create user dict first to avoid looking several times for
1022 #the same user in LDAP SA 27/07/12
1025 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1026 jobs_psql_query = iotlab_dbsession.query(IotlabXP).all()
1028 jobs_psql_query = slab_dbsession.query(SenslabXP).all()
1029 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1030 jobs_psql_dict = dict([(row.job_id, row.__dict__ ) for row in jobs_psql_query ])
1031 #jobs_psql_dict = jobs_psql_dict)
1032 logger.debug("IOTLABDRIVER \tGetLeases jobs_psql_dict %s"\
1034 jobs_psql_id_list = [ row.job_id for row in jobs_psql_query ]
1038 for resa in unfiltered_reservation_list:
1039 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1040 logger.debug("IOTLABDRIVER \tGetLeases USER %s"\
1042 logger.debug("SLABDRIVER \tGetLeases USER %s"\
1043 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1045 #Construct list of jobs (runing, waiting..) in oar
1046 job_oar_list.append(resa['lease_id'])
1047 #If there is information on the job in SLAB DB ]
1048 #(slice used and job id)
1049 if resa['lease_id'] in jobs_psql_dict:
1050 job_info = jobs_psql_dict[resa['lease_id']]
1051 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1052 logger.debug("IOTLABDRIVER \tGetLeases job_info %s"\
1054 logger.debug("SLABDRIVER \tGetLeases job_info %s"\
1055 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1057 resa['slice_hrn'] = job_info['slice_hrn']
1058 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
1060 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1061 #otherwise, assume it is a iotlab slice:
1063 #otherwise, assume it is a senslab slice:
1064 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1066 resa['slice_id'] = hrn_to_urn(self.root_auth+'.'+ \
1067 resa['user'] +"_slice" , 'slice')
1068 resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
1070 resa['component_id_list'] = []
1071 #Transform the hostnames into urns (component ids)
1072 for node in resa['reserved_nodes']:
1074 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1075 iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
1076 resa['component_id_list'].append(iotlab_xrn.urn)
1078 if lease_filter_dict:
1079 logger.debug("IOTLABDRIVER \tGetLeases resa_ %s \
1081 slab_xrn = slab_xrn_object(self.root_auth, node)
1082 resa['component_id_list'].append(slab_xrn.urn)
1084 if lease_filter_dict:
1085 logger.debug("SLABDRIVER \tGetLeases resa_ %s \
1086 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1087 \r\n leasefilter %s" %(resa, lease_filter_dict))
1089 if lease_filter_dict['name'] == resa['slice_hrn']:
1090 reservation_list.append(resa)
1092 if lease_filter_dict is None:
1093 reservation_list = unfiltered_reservation_list
1096 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1097 self.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list)
1099 logger.debug(" IOTLABDRIVER.PY \tGetLeases reservation_list %s"\
1101 self.update_jobs_in_slabdb(job_oar_list, jobs_psql_id_list)
1103 logger.debug(" SLABDRIVER.PY \tGetLeases reservation_list %s"\
1104 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1105 %(reservation_list))
1106 return reservation_list
1111 #TODO FUNCTIONS SECTION 04/07/2012 SA
1113 ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
1116 #def UnBindObjectFromPeer( auth, object_type, object_id, shortname):
1117 #""" This method is a hopefully temporary hack to let the sfa correctly
1118 #detach the objects it creates from a remote peer object. This is
1119 #needed so that the sfa federation link can work in parallel with
1120 #RefreshPeer, as RefreshPeer depends on remote objects being correctly
1123 #auth : struct, API authentication structure
1124 #AuthMethod : string, Authentication method to use
1125 #object_type : string, Object type, among 'site','person','slice',
1127 #object_id : int, object_id
1128 #shortname : string, peer shortname
1132 #logger.warning("IOTLABDRIVER \tUnBindObjectFromPeer EMPTY-\
1136 ##TODO Is BindObjectToPeer still necessary ? Currently does nothing
1138 #|| Commented out 28/05/13 SA
1139 #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
1140 #remote_object_id=None):
1141 #"""This method is a hopefully temporary hack to let the sfa correctly
1142 #attach the objects it creates to a remote peer object. This is needed
1143 #so that the sfa federation link can work in parallel with RefreshPeer,
1144 #as RefreshPeer depends on remote objects being correctly marked.
1146 #shortname : string, peer shortname
1147 #remote_object_id : int, remote object_id, set to 0 if unknown
1151 #logger.warning("IOTLABDRIVER \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
1154 ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA
1155 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1156 ##Funciton should delete and create another job since oin iotlab slice=job
1158 ##Funciton should delete and create another job since oin senslab slice=job
1159 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1160 #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
1161 #"""Updates the parameters of an existing slice with the values in
1163 #Users may only update slices of which they are members.
1164 #PIs may update any of the slices at their sites, or any slices of
1165 #which they are members. Admins may update any slice.
1166 #Only PIs and admins may update max_nodes. Slices cannot be renewed
1167 #(by updating the expires parameter) more than 8 weeks into the future.
1168 #Returns 1 if successful, faults otherwise.
1172 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1173 #logger.warning("IOTLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ")
1175 #logger.warning("SLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ")
1176 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1179 #Unused SA 30/05/13, we only update the user's key or we delete it.
1180 ##TODO UpdatePerson 04/07/2012 SA
1181 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1182 #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None):
1184 #def UpdatePerson(self, slab_hrn, federated_hrn, person_fields=None):
1185 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1186 #"""Updates a person. Only the fields specified in person_fields
1187 #are updated, all other fields are left untouched.
1188 #Users and techs can only update themselves. PIs can only update
1189 #themselves and other non-PIs at their sites.
1190 #Returns 1 if successful, faults otherwise.
1194 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1195 ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
1196 ##iotlab_dbsession.add(new_row)
1197 ##iotlab_dbsession.commit()
1199 #logger.debug("IOTLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ")
1201 ##new_row = FederatedToSenslab(slab_hrn, federated_hrn)
1202 ##slab_dbsession.add(new_row)
1203 ##slab_dbsession.commit()
1205 #logger.debug("SLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ")
1206 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1210 def GetKeys(key_filter=None):
1211 """Returns a dict of dict based on the key string. Each dict entry
1212 contains the key id, the ssh key, the user's email and the
1214 If key_filter is specified and is an array of key identifiers,
1215 only keys matching the filter will be returned.
1217 Admin may query all keys. Non-admins may only query their own keys.
1220 :return: dict with ssh key as key and dicts as value.
1223 if key_filter is None:
1224 keys = dbsession.query(RegKey).options(joinedload('reg_user')).all()
1226 keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all()
1230 key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, \
1231 'email': key.reg_user.email, 'hrn':key.reg_user.hrn}
1233 #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
1234 #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
1235 #for user in ldap_rslt)
1237 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1238 logger.debug("IOTLABDRIVER GetKeys -key_dict %s \r\n " %(key_dict))
1240 logger.debug("SLABDRIVER GetKeys -key_dict %s \r\n " %(key_dict))
1241 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1245 def DeleteKey(self, user_record, key_string):
1246 """ Deletes a key in the LDAP entry of the specified user.
1247 Removes the key_string from the user's key list and updates the LDAP
1248 user's entry with the new key attributes.
1249 :param key_string: The ssh key to remove
1250 :param user_record: User's record
1251 :type key_string: string
1252 :type user_record: dict
1253 :return: True if sucessful, False if not.
1257 all_user_keys = user_record['keys']
1258 all_user_keys.remove(key_string)
1259 new_attributes = {'sshPublicKey':all_user_keys}
1260 ret = self.ldap.LdapModifyUser(user_record, new_attributes)
1261 logger.debug("IOTLABDRIVER DeleteKey %s- "%(ret))
1268 def _sql_get_slice_info( slice_filter ):
1270 Get the slice record based on the slice hrn. Fetch the record of the
1271 user associated with the slice by usingjoinedload based on t
1272 he reg_researcher relationship.
1273 :param slice_filter: the slice hrn we are looking for
1274 :type slice_filter: string
1275 :return: the slice record enhanced with the user's information if the
1276 slice was found, None it wasn't.
1277 :rtype: dict or None.
1279 #DO NOT USE RegSlice - reg_researchers to get the hrn
1280 #of the user otherwise will mess up the RegRecord in
1281 #Resolve, don't know why - SA 08/08/2012
1283 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1284 #Only one entry for one user = one slice in iotlab_xp table
1286 #Only one entry for one user = one slice in slab_xp table
1287 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1288 #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1289 raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn = slice_filter).first()
1290 #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
1292 #load_reg_researcher
1293 #raw_slicerec.reg_researchers
1294 raw_slicerec = raw_slicerec.__dict__
1295 logger.debug(" IOTLABDRIVER \t get_slice_info slice_filter %s \
1296 raw_slicerec %s"%(slice_filter, raw_slicerec))
1297 slicerec = raw_slicerec
1298 #only one researcher per slice so take the first one
1299 #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
1300 #del slicerec['reg_researchers']['_sa_instance_state']
1307 def _sql_get_slice_info_from_user(slice_filter ):
1309 Get the slice record based on the user recordid by using a joinedload
1310 on the relationship reg_slices_as_researcher. Format the sql record
1311 into a dict with the mandatory fields for user and slice.
1312 :return: dict with slice record and user record if the record was found
1313 based on the user's id, None if not..
1314 :rtype:dict or None..
1316 #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1317 raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id = slice_filter).first()
1318 #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
1319 #Put it in correct order
1320 user_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'email', 'pointer']
1321 slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'pointer']
1323 #raw_slicerec.reg_slices_as_researcher
1324 raw_slicerec = raw_slicerec.__dict__
1327 dict([(k, raw_slicerec['reg_slices_as_researcher'][0].__dict__[k]) \
1328 for k in slice_needed_fields])
1329 slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) \
1330 for k in user_needed_fields])
1331 #TODO Handle multiple slices for one user SA 10/12/12
1332 #for now only take the first slice record associated to the rec user
1333 ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
1334 #del raw_slicerec['reg_slices_as_researcher']
1335 #slicerec['reg_researchers'] = raw_slicerec
1336 ##del slicerec['_sa_instance_state']
1343 def _get_slice_records(self, slice_filter = None, \
1344 slice_filter_type = None):
1346 Get the slice record depending on the slice filter and its type.
1347 :param slice_filter: Can be either the slice hrn or the user's record
1349 :type slice_filter: string
1350 :param slice_filter_type: describes the slice filter type used, can be
1351 slice_hrn or record_id_user
1353 :return: the slice record
1355 ..seealso:_sql_get_slice_info_from_user
1356 ..seealso: _sql_get_slice_info
1359 #Get list of slices based on the slice hrn
1360 if slice_filter_type == 'slice_hrn':
1362 #if get_authority(slice_filter) == self.root_auth:
1363 #login = slice_filter.split(".")[1].split("_")[0]
1365 slicerec = self._sql_get_slice_info(slice_filter)
1367 if slicerec is None:
1371 #Get slice based on user id
1372 if slice_filter_type == 'record_id_user':
1374 slicerec = self._sql_get_slice_info_from_user(slice_filter)
1377 fixed_slicerec_dict = slicerec
1378 #At this point if there is no login it means
1379 #record_id_user filter has been used for filtering
1381 ##If theslice record is from iotlab
1382 #if fixed_slicerec_dict['peer_authority'] is None:
1383 #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
1384 #return login, fixed_slicerec_dict
1385 return fixed_slicerec_dict
1389 def GetSlices(self, slice_filter = None, slice_filter_type = None, \
1391 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1392 """ Get the slice records from the iotlab db and add lease information
1394 """ Get the slice records from the slab db and add lease information
1395 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1398 :param slice_filter: can be the slice hrn or slice record id in the db
1399 depending on the slice_filter_type.
1400 :param slice_filter_type: defines the type of the filtering used, Can be
1401 either 'slice_hrn' or "record_id'.
1402 :type slice_filter: string
1403 :type slice_filter_type: string
1404 :return: a slice dict if slice_filter and slice_filter_type
1405 are specified and a matching entry is found in the db. The result
1406 is put into a list.Or a list of slice dictionnaries if no filters are
1412 authorized_filter_types_list = ['slice_hrn', 'record_id_user']
1413 return_slicerec_dictlist = []
1415 #First try to get information on the slice based on the filter provided
1416 if slice_filter_type in authorized_filter_types_list:
1417 fixed_slicerec_dict = \
1418 self._get_slice_records(slice_filter, slice_filter_type)
1419 slice_hrn = fixed_slicerec_dict['hrn']
1421 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1422 logger.debug(" IOTLABDRIVER \tGetSlices login %s \
1424 logger.debug(" SLABDRIVER \tGetSlices login %s \
1425 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1426 slice record %s slice_filter %s \
1427 slice_filter_type %s " %(login, \
1428 fixed_slicerec_dict, slice_filter, \
1432 #Now we have the slice record fixed_slicerec_dict, get the
1433 #jobs associated to this slice
1436 leases_list = self.GetLeases(login = login)
1437 #If no job is running or no job scheduled
1438 #return only the slice record
1439 if leases_list == [] and fixed_slicerec_dict:
1440 return_slicerec_dictlist.append(fixed_slicerec_dict)
1442 #If several jobs for one slice , put the slice record into
1443 # each lease information dict
1446 for lease in leases_list :
1448 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1449 logger.debug("IOTLABDRIVER.PY \tGetSlices slice_filter %s \
1451 logger.debug("SLABDRIVER.PY \tGetSlices slice_filter %s \
1452 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1453 \ lease['slice_hrn'] %s" \
1454 %(slice_filter, lease['slice_hrn']))
1455 if lease['slice_hrn'] == slice_hrn:
1456 slicerec_dict['slice_hrn'] = lease['slice_hrn']
1457 slicerec_dict['hrn'] = lease['slice_hrn']
1458 slicerec_dict['user'] = lease['user']
1459 slicerec_dict['oar_job_id'] = lease['lease_id']
1460 slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}})
1461 slicerec_dict.update({'node_ids':lease['reserved_nodes']})
1463 #Update lease dict with the slice record
1464 if fixed_slicerec_dict:
1465 fixed_slicerec_dict['oar_job_id'] = []
1466 fixed_slicerec_dict['oar_job_id'].append(slicerec_dict['oar_job_id'])
1467 slicerec_dict.update(fixed_slicerec_dict)
1468 #slicerec_dict.update({'hrn':\
1469 #str(fixed_slicerec_dict['slice_hrn'])})
1471 return_slicerec_dictlist.append(slicerec_dict)
1472 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1473 logger.debug("IOTLABDRIVER.PY \tGetSlices \
1474 OHOHOHOH %s" %(return_slicerec_dictlist ))
1476 logger.debug("IOTLABDRIVER.PY \tGetSlices \
1478 logger.debug("SLABDRIVER.PY \tGetSlices \
1479 OHOHOHOH %s" %(return_slicerec_dictlist ))
1481 logger.debug("SLABDRIVER.PY \tGetSlices \
1482 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1483 slicerec_dict %s return_slicerec_dictlist %s \
1484 lease['reserved_nodes'] \
1485 %s" %(slicerec_dict, return_slicerec_dictlist, \
1486 lease['reserved_nodes'] ))
1488 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1489 logger.debug("IOTLABDRIVER.PY \tGetSlices RETURN \
1491 logger.debug("SLABDRIVER.PY \tGetSlices RETURN \
1492 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1493 return_slicerec_dictlist %s" \
1494 %(return_slicerec_dictlist))
1496 return return_slicerec_dictlist
1500 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1501 #Get all slices from the iotlab sfa database ,
1503 #Get all slices from the senslab sfa database ,
1504 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1505 #put them in dict format
1506 #query_slice_list = dbsession.query(RegRecord).all()
1507 query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
1509 for record in query_slice_list:
1510 tmp = record.__dict__
1511 tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
1512 #del tmp['reg_researchers']['_sa_instance_state']
1513 return_slicerec_dictlist.append(tmp)
1514 #return_slicerec_dictlist.append(record.__dict__)
1516 #Get all the jobs reserved nodes
1517 leases_list = self.GetReservedNodes()
1520 for fixed_slicerec_dict in return_slicerec_dictlist:
1522 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1523 #Check if the slice belongs to a iotlab user
1525 #Check if the slice belongs to a senslab user
1526 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1527 if fixed_slicerec_dict['peer_authority'] is None:
1528 owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
1531 for lease in leases_list:
1532 if owner == lease['user']:
1533 slicerec_dict['oar_job_id'] = lease['lease_id']
1535 #for reserved_node in lease['reserved_nodes']:
1536 logger.debug("IOTLABDRIVER.PY \tGetSlices lease %s "\
1539 slicerec_dict.update({'node_ids':lease['reserved_nodes']})
1540 slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}})
1541 slicerec_dict.update(fixed_slicerec_dict)
1542 #slicerec_dict.update({'hrn':\
1543 #str(fixed_slicerec_dict['slice_hrn'])})
1544 #return_slicerec_dictlist.append(slicerec_dict)
1545 fixed_slicerec_dict.update(slicerec_dict)
1547 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1548 logger.debug("IOTLABDRIVER.PY \tGetSlices RETURN \
1550 logger.debug("SLABDRIVER.PY \tGetSlices RETURN \
1551 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1552 return_slicerec_dictlist %s \slice_filter %s " \
1553 %(return_slicerec_dictlist, slice_filter))
1555 return return_slicerec_dictlist
1556 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1559 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1563 #Update slice unused, therefore sfa_fields_to_iotlab_fields unused
1566 #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record):
1571 ##for field in record:
1572 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1573 ## iotlab_record[field] = record[field]
1575 #if sfa_type == "slice":
1576 ##instantion used in get_slivers ?
1577 #if not "instantiation" in iotlab_record:
1578 #iotlab_record["instantiation"] = "iotlab-instantiated"
1579 ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn)
1580 ##Unused hrn_to_pl_slicename because Slab's hrn already
1581 ##in the appropriate form SA 23/07/12
1582 #iotlab_record["hrn"] = hrn
1583 #logger.debug("IOTLABDRIVER.PY sfa_fields_to_iotlab_fields \
1584 #iotlab_record %s " %(iotlab_record['hrn']))
1586 ## slab_record[field] = record[field]
1588 #if sfa_type == "slice":
1589 ##instantion used in get_slivers ?
1590 #if not "instantiation" in slab_record:
1591 #slab_record["instantiation"] = "senslab-instantiated"
1592 ##slab_record["hrn"] = hrn_to_pl_slicename(hrn)
1593 ##Unused hrn_to_pl_slicename because Slab's hrn already
1594 ##in the appropriate form SA 23/07/12
1595 #slab_record["hrn"] = hrn
1596 #logger.debug("SLABDRIVER.PY sfa_fields_to_slab_fields \
1597 #slab_record %s " %(slab_record['hrn']))
1598 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1599 #if "url" in record:
1600 #iotlab_record["url"] = record["url"]
1601 #if "description" in record:
1602 #iotlab_record["description"] = record["description"]
1603 #if "expires" in record:
1604 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1605 #iotlab_record["expires"] = int(record["expires"])
1607 #slab_record["expires"] = int(record["expires"])
1608 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1610 ##nodes added by OAR only and then imported to SFA
1611 ##elif type == "node":
1612 ##if not "hostname" in iotlab_record:
1613 ##if not "hostname" in record:
1614 ##raise MissingSfaInfo("hostname")
1615 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1616 ##iotlab_record["hostname"] = record["hostname"]
1617 ##if not "model" in iotlab_record:
1618 ##iotlab_record["model"] = "geni"
1620 ##slab_record["hostname"] = record["hostname"]
1621 ##if not "model" in slab_record:
1622 ##slab_record["model"] = "geni"
1623 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py
1625 ##One authority only
1626 ##elif type == "authority":
1627 ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn)
1629 ##if not "name" in iotlab_record:
1630 ##iotlab_record["name"] = hrn
1632 ##if not "abbreviated_name" in iotlab_record:
1633 ##iotlab_record["abbreviated_name"] = hrn
1635 ##if not "enabled" in iotlab_record:
1636 ##iotlab_record["enabled"] = True
1638 ##if not "is_public" in iotlab_record:
1639 ##iotlab_record["is_public"] = True
1641 #return iotlab_record
1652 <<<<<<< HEAD:sfa/iotlab/iotlabapi.py
1661 >>>>>>> 3fe7429... SA:sfa/senslab/slabapi.py