1 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
2 from sfa.util.sfalogging import logger
3 from sfa.storage.alchemy import dbsession
4 from sfa.storage.model import RegRecord
8 from sfa.managers.driver import Driver
9 from sfa.rspecs.version_manager import VersionManager
10 from sfa.rspecs.rspec import RSpec
12 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
15 from sfa.iotlab.iotlabpostgres import IotlabDB
18 from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname
20 from sfa.iotlab.iotlabslices import IotlabSlices
23 from sfa.iotlab.iotlabapi import IotlabTestbedAPI
26 class IotlabDriver(Driver):
27 """ Iotlab Driver class inherited from Driver generic class.
29 Contains methods compliant with the SFA standard and the testbed
30 infrastructure (calls to LDAP and OAR).
32 ..seealso:: Driver class
35 def __init__(self, config):
38 Sets the iotlab SFA config parameters ,
39 instanciates the testbed api and the iotlab database.
41 :param config: iotlab SFA configuration object
42 :type config: Config object
44 Driver.__init__ (self, config)
47 self.db = IotlabDB(config, debug = False)
48 self.iotlab_api = IotlabTestbedAPI(config)
51 def augment_records_with_testbed_info (self, record_list ):
54 Adds specific testbed info to the records.
56 :param record_list: list of sfa dictionaries records
57 :type record_list: list
58 :return: list of records with extended information in each record
61 return self.fill_record_info (record_list)
63 def fill_record_info(self, record_list):
65 For each SFA record, fill in the iotlab specific and SFA specific
68 :param record_list: list of sfa dictionaries records
69 :type record_list: list
70 :return: list of records with extended information in each record
73 .. warnings:: Should not be modifying record_list directly because modi
74 fication are kept outside the method's scope. Howerver, there is no
75 other way to do it given the way it's called in registry manager.
78 logger.debug("IOTLABDRIVER \tfill_record_info records %s " %(record_list))
79 if not isinstance(record_list, list):
80 record_list = [record_list]
84 for record in record_list:
85 #If the record is a SFA slice record, then add information
86 #about the user of this slice. This kind of
87 #information is in the Iotlab's DB.
88 if str(record['type']) == 'slice':
89 if 'reg_researchers' in record and \
90 isinstance(record['reg_researchers'], list) :
91 record['reg_researchers'] = \
92 record['reg_researchers'][0].__dict__
93 record.update({'PI':[record['reg_researchers']['hrn']],
94 'researcher': [record['reg_researchers']['hrn']],
98 'person_ids':[record['reg_researchers']['record_id']],
99 'geni_urn':'', #For client_helper.py compatibility
100 'keys':'', #For client_helper.py compatibility
101 'key_ids':''}) #For client_helper.py compatibility
104 #Get iotlab slice record and oar job id if any.
105 recslice_list = self.iotlab_api.GetSlices(slice_filter = \
107 slice_filter_type = 'slice_hrn')
110 logger.debug("IOTLABDRIVER \tfill_record_info \
111 TYPE SLICE RECUSER record['hrn'] %s ecord['oar_job_id']\
112 %s " %(record['hrn'], record['oar_job_id']))
113 del record['reg_researchers']
115 for rec in recslice_list:
116 logger.debug("IOTLABDRIVER\r\n \t \
117 fill_record_info oar_job_id %s " \
118 %(rec['oar_job_id']))
120 record['node_ids'] = [ self.iotlab_api.root_auth + \
121 hostname for hostname in rec['node_ids']]
126 logger.debug( "IOTLABDRIVER.PY \t fill_record_info SLICE \
127 recslice_list %s \r\n \t RECORD %s \r\n \
128 \r\n" %(recslice_list, record))
130 if str(record['type']) == 'user':
131 #The record is a SFA user record.
132 #Get the information about his slice from Iotlab's DB
133 #and add it to the user record.
134 recslice_list = self.iotlab_api.GetSlices(\
135 slice_filter = record['record_id'],\
136 slice_filter_type = 'record_id_user')
138 logger.debug( "IOTLABDRIVER.PY \t fill_record_info TYPE USER \
139 recslice_list %s \r\n \t RECORD %s \r\n" \
140 %(recslice_list , record))
141 #Append slice record in records list,
142 #therefore fetches user and slice info again(one more loop)
143 #Will update PIs and researcher for the slice
145 recuser = recslice_list[0]['reg_researchers']
146 logger.debug( "IOTLABDRIVER.PY \t fill_record_info USER \
147 recuser %s \r\n \r\n" %(recuser))
149 recslice = recslice_list[0]
150 recslice.update({'PI':[recuser['hrn']],
151 'researcher': [recuser['hrn']],
152 'name':record['hrn'],
155 'person_ids':[recuser['record_id']]})
157 for rec in recslice_list:
158 recslice['oar_job_id'].append(rec['oar_job_id'])
162 recslice.update({'type':'slice', \
163 'hrn':recslice_list[0]['hrn']})
166 #GetPersons takes [] as filters
167 user_iotlab = self.iotlab_api.GetPersons([record])
170 record.update(user_iotlab[0])
171 #For client_helper.py compatibility
172 record.update( { 'geni_urn':'',
175 record_list.append(recslice)
177 logger.debug("IOTLABDRIVER.PY \tfill_record_info ADDING SLICE\
178 INFO TO USER records %s" %(record_list))
181 except TypeError, error:
182 logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"\
188 def sliver_status(self, slice_urn, slice_hrn):
190 Receive a status request for slice named urn/hrn
191 urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
192 shall return a structure as described in
193 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
194 NT : not sure if we should implement this or not, but used by sface.
196 :param slice_urn: slice urn
197 :type slice_urn: string
198 :param slice_hrn: slice hrn
199 :type slice_hrn: string
204 #First get the slice with the slice hrn
205 slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \
206 slice_filter_type = 'slice_hrn')
208 if len(slice_list) is 0:
209 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
211 #Used for fetching the user info witch comes along the slice info
212 one_slice = slice_list[0]
215 #Make a list of all the nodes hostnames in use for this slice
216 slice_nodes_list = []
217 #for single_slice in slice_list:
218 #for node in single_slice['node_ids']:
219 #slice_nodes_list.append(node['hostname'])
220 #for node in one_slice:
221 #slice_nodes_list.append(node['hostname'])
222 slice_nodes_list = one_slice['node_ids']
223 #Get all the corresponding nodes details
224 nodes_all = self.iotlab_api.GetNodes({'hostname':slice_nodes_list},
225 ['node_id', 'hostname','site','boot_state'])
226 nodeall_byhostname = dict([(one_node['hostname'], one_node) \
227 for one_node in nodes_all])
231 for single_slice in slice_list:
234 top_level_status = 'empty'
237 ['geni_urn','geni_error', 'pl_login','geni_status','geni_resources'], None)
238 result['pl_login'] = one_slice['reg_researchers'][0].hrn
239 logger.debug("Slabdriver - sliver_status Sliver status \
240 urn %s hrn %s single_slice %s \r\n " \
241 %(slice_urn, slice_hrn, single_slice))
243 if 'node_ids' not in single_slice:
245 result['geni_status'] = top_level_status
246 result['geni_resources'] = []
249 top_level_status = 'ready'
251 #A job is running on Iotlab for this slice
252 # report about the local nodes that are in the slice only
254 result['geni_urn'] = slice_urn
257 for node_hostname in single_slice['node_ids']:
259 res['iotlab_hostname'] = node_hostname
260 res['iotlab_boot_state'] = nodeall_byhostname[node_hostname]['boot_state']
262 #res['pl_hostname'] = node['hostname']
263 #res['pl_boot_state'] = \
264 #nodeall_byhostname[node['hostname']]['boot_state']
265 #res['pl_last_contact'] = strftime(self.time_format, \
266 #gmtime(float(timestamp)))
267 sliver_id = Xrn(slice_urn, type='slice', \
268 id=nodeall_byhostname[node_hostname]['node_id'], \
269 authority=self.hrn).urn
271 res['geni_urn'] = sliver_id
272 #node_name = node['hostname']
273 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
275 res['geni_status'] = 'ready'
277 res['geni_status'] = 'failed'
278 top_level_status = 'failed'
280 res['geni_error'] = ''
282 resources.append(res)
284 result['geni_status'] = top_level_status
285 result['geni_resources'] = resources
286 logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "\
291 def get_user_record(hrn):
293 Returns the user record based on the hrn from the SFA DB .
295 :param hrn: user's hrn
297 :return : user record from SFA database
301 return dbsession.query(RegRecord).filter_by(hrn = hrn).first()
304 def testbed_name (self):
306 Returns testbed's name.
312 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
313 def aggregate_version (self):
316 Returns the testbed's supported rspec advertisement and
321 version_manager = VersionManager()
322 ad_rspec_versions = []
323 request_rspec_versions = []
324 for rspec_version in version_manager.versions:
325 if rspec_version.content_type in ['*', 'ad']:
326 ad_rspec_versions.append(rspec_version.to_dict())
327 if rspec_version.content_type in ['*', 'request']:
328 request_rspec_versions.append(rspec_version.to_dict())
330 'testbed':self.testbed_name(),
331 'geni_request_rspec_versions': request_rspec_versions,
332 'geni_ad_rspec_versions': ad_rspec_versions,
337 def _get_requested_leases_list(self, rspec):
339 Process leases in rspec depending on the rspec version (format)
340 type. Find the lease requests in the rspec and creates
341 a lease request list with the mandatory information ( nodes,
342 start time and duration) of the valid leases (duration above or equal
343 to the iotlab experiment minimum duration).
345 :param rspec: rspec request received.
347 :return: list of lease requests found in the rspec
350 requested_lease_list = []
351 for lease in rspec.version.get_leases():
352 single_requested_lease = {}
353 logger.debug("IOTLABDRIVER.PY \t_get_requested_leases_list lease %s " %(lease))
355 if not lease.get('lease_id'):
356 if get_authority(lease['component_id']) == \
357 self.iotlab_api.root_auth:
358 single_requested_lease['hostname'] = \
359 iotlab_xrn_to_hostname(\
360 lease.get('component_id').strip())
361 single_requested_lease['start_time'] = \
362 lease.get('start_time')
363 single_requested_lease['duration'] = lease.get('duration')
364 #Check the experiment's duration is valid before adding
365 #the lease to the requested leases list
366 duration_in_seconds = \
367 int(single_requested_lease['duration'])
368 if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec() :
369 requested_lease_list.append(single_requested_lease)
371 return requested_lease_list
374 def _group_leases_by_start_time(requested_lease_list):
376 Create dict of leases by start_time, regrouping nodes reserved
377 at the same time, for the same amount of time so as to
378 define one job on OAR.
380 :param requested_lease_list: list of leases
381 :type requested_lease_list: list
382 :return: Dictionary with key = start time, value = list of leases
383 with the same start time.
387 requested_job_dict = {}
388 for lease in requested_lease_list:
390 #In case it is an asap experiment start_time is empty
391 if lease['start_time'] == '':
392 lease['start_time'] = '0'
394 if lease['start_time'] not in requested_job_dict:
395 if isinstance(lease['hostname'], str):
396 lease['hostname'] = [lease['hostname']]
399 requested_job_dict[lease['start_time']] = lease
402 job_lease = requested_job_dict[lease['start_time']]
403 if lease['duration'] == job_lease['duration'] :
404 job_lease['hostname'].append(lease['hostname'])
406 return requested_job_dict
408 def _process_requested_jobs(self, rspec):
410 Turns the requested leases and information into a dictionary
411 of requested jobs, grouped by starting time.
413 :param rspec: RSpec received
417 requested_lease_list = self._get_requested_leases_list(rspec)
418 logger.debug("IOTLABDRIVER _process_requested_jobs requested_lease_list \
419 %s"%(requested_lease_list))
420 job_dict = self._group_leases_by_start_time(requested_lease_list)
421 logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\
426 def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \
429 Answer to CreateSliver.
430 Creates the leases and slivers for the users from the information
431 found in the rspec string.
432 Launch experiment on OAR if the requested leases is valid. Delete
433 no longer requested leases.
436 :param creds: user's credentials
438 :param users: user record list
443 :return: a valid Rspec for the slice which has just been
449 aggregate = IotlabAggregate(self)
451 slices = IotlabSlices(self)
452 peer = slices.get_peer(slice_hrn)
453 sfa_peer = slices.get_sfa_peer(slice_hrn)
456 if not isinstance(creds, list):
460 slice_record = users[0].get('slice_record', {})
461 logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\
462 creds %s \r\n \r\n users %s" \
464 slice_record['user'] = {'keys':users[0]['keys'], \
465 'email':users[0]['email'], \
466 'hrn':slice_record['reg-researchers'][0]}
468 rspec = RSpec(rspec_string)
469 logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \
470 %s slice_record %s users %s" \
471 %(rspec.version,slice_record, users))
474 # ensure site record exists?
475 # ensure slice record exists
476 #Removed options to verify_slice SA 14/08/12
477 sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \
480 # ensure person records exists
481 #verify_persons returns added persons but since the return value
483 slices.verify_persons(slice_hrn, sfa_slice, users, peer, \
484 sfa_peer, options=options)
485 #requested_attributes returned by rspec.version.get_slice_attributes()
486 #unused, removed SA 13/08/12
487 #rspec.version.get_slice_attributes()
489 logger.debug("IOTLABDRIVER.PY create_sliver slice %s " %(sfa_slice))
491 # add/remove slice from nodes
493 #requested_slivers = [node.get('component_id') \
494 #for node in rspec.version.get_nodes_with_slivers()\
495 #if node.get('authority_id') is self.iotlab_api.root_auth]
496 #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
497 #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
498 #requested_slivers %s listnodes %s" \
499 #%(requested_slivers,l))
500 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
501 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
504 requested_job_dict = self._process_requested_jobs(rspec)
507 logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\
508 %(requested_job_dict))
509 #verify_slice_leases returns the leases , but the return value is unused
510 #here. Removed SA 13/08/12
511 slices.verify_slice_leases(sfa_slice, \
512 requested_job_dict, peer)
514 return aggregate.get_rspec(slice_xrn=slice_urn, \
515 login=sfa_slice['login'], version=rspec.version)
518 def delete_sliver (self, slice_urn, slice_hrn, creds, options):
520 Deletes the lease associated with the slice hrn and the credentials
521 if the slice belongs to iotlab. Answer to DeleteSliver.
523 :return: 1 if the slice to delete was not found on iotlab,
524 True if the deletion was successful, False otherwise otherwise.
526 .. note:: Should really be named delete_leases because iotlab does
527 not have any slivers, but only deals with leases. However, SFA api only
528 have delete_sliver define so far. SA 13.05/2013
531 sfa_slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \
532 slice_filter_type = 'slice_hrn')
534 if not sfa_slice_list:
537 #Delete all leases in the slice
538 for sfa_slice in sfa_slice_list:
541 logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" %(sfa_slice))
542 slices = IotlabSlices(self)
543 # determine if this is a peer slice
545 peer = slices.get_peer(slice_hrn)
547 logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
548 \r\n \t sfa_slice %s " %(peer, sfa_slice))
551 self.iotlab_api.DeleteSliceFromNodes(sfa_slice)
557 def list_resources (self, slice_urn, slice_hrn, creds, options):
559 List resources from the iotlab aggregate and returns a Rspec
560 advertisement with resources found when slice_urn and slice_hrn are None
561 (in case of resource discovery).
562 If a slice hrn and urn are provided, list experiment's slice
563 nodes in a rspec format. Answer to ListResources.
565 :param options: options used when listing resources (list_leases, info,
567 :return: rspec string in xml
571 #cached_requested = options.get('cached', True)
573 version_manager = VersionManager()
574 # get the rspec's return format from options
576 version_manager.get_version(options.get('geni_rspec_version'))
577 version_string = "rspec_%s" % (rspec_version)
579 #panos adding the info option to the caching key (can be improved)
580 if options.get('info'):
581 version_string = version_string + "_" + \
582 options.get('info', 'default')
584 # Adding the list_leases option to the caching key
585 if options.get('list_leases'):
586 version_string = version_string + "_" + \
587 options.get('list_leases', 'default')
589 # Adding geni_available to caching key
590 if options.get('geni_available'):
591 version_string = version_string + "_" + \
592 str(options.get('geni_available'))
594 # look in cache first
595 #if cached_requested and self.cache and not slice_hrn:
596 #rspec = self.cache.get(version_string)
598 #logger.debug("IotlabDriver.ListResources: \
599 #returning cached advertisement")
602 #panos: passing user-defined options
603 aggregate = IotlabAggregate(self)
605 rspec = aggregate.get_rspec(slice_xrn=slice_urn, \
606 version=rspec_version, options=options)
609 #if self.cache and not slice_hrn:
610 #logger.debug("Slab.ListResources: stores advertisement in cache")
611 #self.cache.add(version_string, rspec)
616 def list_slices (self, creds, options):
618 Answer to ListSlices.
619 List slices belonging to iotlab, returns slice urns list.
620 No caching used. Options unused but are defined in the SFA method
623 :return: slice urns list
627 # look in cache first
629 #slices = self.cache.get('slices')
631 #logger.debug("PlDriver.list_slices returns from cache")
636 slices = self.iotlab_api.GetSlices()
637 logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices))
638 slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
640 slice_urns = [hrn_to_urn(slice_hrn, 'slice') \
641 for slice_hrn in slice_hrns]
645 #logger.debug ("IotlabDriver.list_slices stores value in cache")
646 #self.cache.add('slices', slice_urns)
651 def register (self, sfa_record, hrn, pub_key):
653 Adding new user, slice, node or site should not be handled
656 ..warnings:: should not be used. Different components are in charge of
657 doing this task. Adding nodes = OAR
658 Adding users = LDAP Iotlab
659 Adding slice = Import from LDAP users
662 :param sfa_record: record provided by the client of the
664 :type sfa_record: dict
669 def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
670 """No site or node record update allowed in Iotlab.
671 The only modifications authorized here are key deletion/addition
672 on an existing user and password change.
673 On an existing user, CAN NOT BE MODIFIED:
674 'first_name', 'last_name', 'email'
675 DOES NOT EXIST IN SENSLAB:
676 'phone', 'url', 'bio','title', 'accepted_aup',
677 A slice is bound to its user, so modifying the user's ssh key should
678 modify the slice's GID after an import procedure.
680 :param old_sfa_record: what is in the db for this hrn
681 :param new_sfa_record: what was passed to the Update call
683 ..seealso:: update in driver.py.
686 pointer = old_sfa_record['pointer']
687 old_sfa_record_type = old_sfa_record['type']
689 # new_key implemented for users only
690 if new_key and old_sfa_record_type not in [ 'user' ]:
691 raise UnknownSfaType(old_sfa_record_type)
694 if old_sfa_record_type == "user":
696 all_fields = new_sfa_record
697 for key in all_fields.keys():
698 if key in ['key', 'password']:
699 update_fields[key] = all_fields[key]
703 # must check this key against the previous one if it exists
704 persons = self.iotlab_api.GetPersons([old_sfa_record])
706 keys = [person['pkey']]
707 #Get all the person's keys
708 keys_dict = self.iotlab_api.GetKeys(keys)
710 # Delete all stale keys, meaning the user has only one key
712 #TODO: do we really want to delete all the other keys?
713 #Is this a problem with the GID generation to have multiple
719 #remove all the other keys
720 for key in keys_dict:
721 self.iotlab_api.DeleteKey(person, key)
722 self.iotlab_api.AddPersonKey(person, \
723 {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} )
724 #self.iotlab_api.AddPersonKey(person, {'key_type': 'ssh', \
729 def remove (self, sfa_record):
731 Removes users only. Mark the user as disabled in
732 LDAP. The user and his slice are then deleted from the db by running an
733 import on the registry.
737 :param sfa_record: record is the existing sfa record in the db
738 :type sfa_record: dict
740 ..warning::As fas as the slice is concerned, here only the leases are
741 removed from the slice. The slice is record itself is not removed from
743 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
745 TODO: return boolean for the slice part
747 sfa_record_type = sfa_record['type']
748 hrn = sfa_record['hrn']
749 if sfa_record_type == 'user':
751 #get user from iotlab ldap
752 person = self.iotlab_api.GetPersons(sfa_record)
753 #No registering at a given site in Iotlab.
754 #Once registered to the LDAP, all iotlab sites are
757 #Mark account as disabled in ldap
758 return self.iotlab_api.DeletePerson(sfa_record)
760 elif sfa_record_type == 'slice':
761 if self.iotlab_api.GetSlices(slice_filter = hrn, \
762 slice_filter_type = 'slice_hrn'):
763 ret = self.iotlab_api.DeleteSlice(sfa_record)