1 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
2 from sfa.util.sfalogging import logger
3 from sfa.storage.alchemy import dbsession
4 from sfa.storage.model import RegRecord
6 from sfa.managers.driver import Driver
7 from sfa.rspecs.version_manager import VersionManager
8 from sfa.rspecs.rspec import RSpec
10 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
13 from sfa.iotlab.iotlabpostgres import IotlabDB
15 from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname
16 from sfa.iotlab.iotlabslices import IotlabSlices
19 from sfa.iotlab.iotlabapi import IotlabTestbedAPI
22 class IotlabDriver(Driver):
23 """ Iotlab Driver class inherited from Driver generic class.
25 Contains methods compliant with the SFA standard and the testbed
26 infrastructure (calls to LDAP and OAR).
28 .. seealso::: Driver class
31 def __init__(self, config):
34 Sets the iotlab SFA config parameters,
35 instanciates the testbed api and the iotlab database.
37 :param config: iotlab SFA configuration object
38 :type config: Config object
41 Driver.__init__ (self, config)
44 # self.db = IotlabDB(config, debug = False)
45 self.iotlab_api = IotlabTestbedAPI(config)
48 def augment_records_with_testbed_info (self, record_list ):
51 Adds specific testbed info to the records.
53 :param record_list: list of sfa dictionaries records
54 :type record_list: list
55 :returns: list of records with extended information in each record
59 return self.fill_record_info (record_list)
61 def fill_record_info(self, record_list):
64 For each SFA record, fill in the iotlab specific and SFA specific
67 :param record_list: list of sfa dictionaries records
68 :type record_list: list
69 :returns: list of records with extended information in each record
72 .. warning:: Should not be modifying record_list directly because modi
73 fication are kept outside the method's scope. Howerver, there is no
74 other way to do it given the way it's called in registry manager.
78 logger.debug("IOTLABDRIVER \tfill_record_info records %s "
80 if not isinstance(record_list, list):
81 record_list = [record_list]
84 for record in record_list:
85 #If the record is a SFA slice record, then add information
86 #about the user of this slice. This kind of
87 #information is in the Iotlab's DB.
88 if str(record['type']) == 'slice':
89 if 'reg_researchers' in record and \
90 isinstance(record['reg_researchers'], list) :
91 record['reg_researchers'] = \
92 record['reg_researchers'][0].__dict__
93 record.update({'PI':[record['reg_researchers']['hrn']],
94 'researcher': [record['reg_researchers']['hrn']],
98 'person_ids': [record['reg_researchers']
100 # For client_helper.py compatibility
102 # For client_helper.py compatibility
104 # For client_helper.py compatibility
107 #Get iotlab slice record and oar job id if any.
108 recslice_list = self.iotlab_api.GetSlices(
109 slice_filter=str(record['hrn']),
110 slice_filter_type='slice_hrn')
112 logger.debug("IOTLABDRIVER \tfill_record_info \
113 TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\
114 %s " % (record['hrn'], record['oar_job_id']))
115 del record['reg_researchers']
117 for rec in recslice_list:
118 logger.debug("IOTLABDRIVER\r\n \t \
119 fill_record_info oar_job_id %s "
120 % (rec['oar_job_id']))
122 record['node_ids'] = [self.iotlab_api.root_auth +
123 hostname for hostname in
128 logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \
129 recslice_list %s \r\n \t RECORD %s \r\n \
130 \r\n" % (recslice_list, record))
132 if str(record['type']) == 'user':
133 #The record is a SFA user record.
134 #Get the information about his slice from Iotlab's DB
135 #and add it to the user record.
136 recslice_list = self.iotlab_api.GetSlices(
137 slice_filter=record['record_id'],
138 slice_filter_type='record_id_user')
140 logger.debug("IOTLABDRIVER.PY \t fill_record_info \
141 TYPE USER recslice_list %s \r\n \t RECORD %s \r\n"
142 % (recslice_list, record))
143 #Append slice record in records list,
144 #therefore fetches user and slice info again(one more loop)
145 #Will update PIs and researcher for the slice
147 recuser = recslice_list[0]['reg_researchers']
148 logger.debug( "IOTLABDRIVER.PY \t fill_record_info USER \
149 recuser %s \r\n \r\n" % (recuser))
151 recslice = recslice_list[0]
152 recslice.update({'PI':[recuser['hrn']],
153 'researcher': [recuser['hrn']],
154 'name':record['hrn'],
157 'person_ids':[recuser['record_id']]})
159 for rec in recslice_list:
160 recslice['oar_job_id'].append(rec['oar_job_id'])
164 recslice.update({'type':'slice', \
165 'hrn':recslice_list[0]['hrn']})
168 #GetPersons takes [] as filters
169 user_iotlab = self.iotlab_api.GetPersons([record])
172 record.update(user_iotlab[0])
173 #For client_helper.py compatibility
174 record.update( { 'geni_urn':'',
177 record_list.append(recslice)
179 logger.debug("IOTLABDRIVER.PY \t \
180 fill_record_info ADDING SLICE\
181 INFO TO USER records %s" % (record_list))
184 except TypeError, error:
185 logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"\
191 def sliver_status(self, slice_urn, slice_hrn):
194 Receive a status request for slice named urn/hrn
195 urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
196 shall return a structure as described in
197 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
198 NT : not sure if we should implement this or not, but used by sface.
200 :param slice_urn: slice urn
201 :type slice_urn: string
202 :param slice_hrn: slice hrn
203 :type slice_hrn: string
208 #First get the slice with the slice hrn
209 slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn,
210 slice_filter_type='slice_hrn')
212 if len(slice_list) == 0:
213 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
215 #Used for fetching the user info witch comes along the slice info
216 one_slice = slice_list[0]
219 #Make a list of all the nodes hostnames in use for this slice
220 slice_nodes_list = []
221 #for single_slice in slice_list:
222 #for node in single_slice['node_ids']:
223 #slice_nodes_list.append(node['hostname'])
224 #for node in one_slice:
225 #slice_nodes_list.append(node['hostname'])
226 slice_nodes_list = one_slice['node_ids']
227 #Get all the corresponding nodes details
228 nodes_all = self.iotlab_api.GetNodes({'hostname':slice_nodes_list},
229 ['node_id', 'hostname','site','boot_state'])
230 nodeall_byhostname = dict([(one_node['hostname'], one_node) \
231 for one_node in nodes_all])
235 for single_slice in slice_list:
238 top_level_status = 'empty'
241 ['geni_urn','geni_error', 'iotlab_login','geni_status',
242 'geni_resources'], None)
244 # ['geni_urn','geni_error', 'pl_login','geni_status',
245 # 'geni_resources'], None)
246 # result['pl_login'] = one_slice['reg_researchers'][0].hrn
247 result['iotlab_login'] = one_slice['user']
248 logger.debug("Slabdriver - sliver_status Sliver status \
249 urn %s hrn %s single_slice %s \r\n " \
250 %(slice_urn, slice_hrn, single_slice))
252 if 'node_ids' not in single_slice:
254 result['geni_status'] = top_level_status
255 result['geni_resources'] = []
258 top_level_status = 'ready'
260 #A job is running on Iotlab for this slice
261 # report about the local nodes that are in the slice only
263 result['geni_urn'] = slice_urn
266 for node_hostname in single_slice['node_ids']:
268 res['iotlab_hostname'] = node_hostname
269 res['iotlab_boot_state'] = \
270 nodeall_byhostname[node_hostname]['boot_state']
272 #res['pl_hostname'] = node['hostname']
273 #res['pl_boot_state'] = \
274 #nodeall_byhostname[node['hostname']]['boot_state']
275 #res['pl_last_contact'] = strftime(self.time_format, \
276 #gmtime(float(timestamp)))
277 sliver_id = Xrn(slice_urn, type='slice', \
278 id=nodeall_byhostname[node_hostname]['node_id']).urn
280 res['geni_urn'] = sliver_id
281 #node_name = node['hostname']
282 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
284 res['geni_status'] = 'ready'
286 res['geni_status'] = 'failed'
287 top_level_status = 'failed'
289 res['geni_error'] = ''
291 resources.append(res)
293 result['geni_status'] = top_level_status
294 result['geni_resources'] = resources
295 logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "\
300 def get_user_record(hrn):
303 Returns the user record based on the hrn from the SFA DB .
305 :param hrn: user's hrn
307 :returns: user record from SFA database
311 return dbsession.query(RegRecord).filter_by(hrn = hrn).first()
314 def testbed_name (self):
317 Returns testbed's name.
324 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
325 def aggregate_version (self):
328 Returns the testbed's supported rspec advertisement and
334 version_manager = VersionManager()
335 ad_rspec_versions = []
336 request_rspec_versions = []
337 for rspec_version in version_manager.versions:
338 if rspec_version.content_type in ['*', 'ad']:
339 ad_rspec_versions.append(rspec_version.to_dict())
340 if rspec_version.content_type in ['*', 'request']:
341 request_rspec_versions.append(rspec_version.to_dict())
343 'testbed':self.testbed_name(),
344 'geni_request_rspec_versions': request_rspec_versions,
345 'geni_ad_rspec_versions': ad_rspec_versions,
350 def _get_requested_leases_list(self, rspec):
352 Process leases in rspec depending on the rspec version (format)
353 type. Find the lease requests in the rspec and creates
354 a lease request list with the mandatory information ( nodes,
355 start time and duration) of the valid leases (duration above or
356 equal to the iotlab experiment minimum duration).
358 :param rspec: rspec request received.
360 :returns: list of lease requests found in the rspec
363 requested_lease_list = []
364 for lease in rspec.version.get_leases():
365 single_requested_lease = {}
366 logger.debug("IOTLABDRIVER.PY \t \
367 _get_requested_leases_list lease %s " % (lease))
369 if not lease.get('lease_id'):
370 if get_authority(lease['component_id']) == \
371 self.iotlab_api.root_auth:
372 single_requested_lease['hostname'] = \
373 iotlab_xrn_to_hostname(\
374 lease.get('component_id').strip())
375 single_requested_lease['start_time'] = \
376 lease.get('start_time')
377 single_requested_lease['duration'] = lease.get('duration')
378 #Check the experiment's duration is valid before adding
379 #the lease to the requested leases list
380 duration_in_seconds = \
381 int(single_requested_lease['duration'])
382 if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec() :
383 requested_lease_list.append(single_requested_lease)
385 return requested_lease_list
388 def _group_leases_by_start_time(requested_lease_list):
390 Create dict of leases by start_time, regrouping nodes reserved
391 at the same time, for the same amount of time so as to
392 define one job on OAR.
394 :param requested_lease_list: list of leases
395 :type requested_lease_list: list
396 :returns: Dictionary with key = start time, value = list of leases
397 with the same start time.
402 requested_job_dict = {}
403 for lease in requested_lease_list:
405 #In case it is an asap experiment start_time is empty
406 if lease['start_time'] == '':
407 lease['start_time'] = '0'
409 if lease['start_time'] not in requested_job_dict:
410 if isinstance(lease['hostname'], str):
411 lease['hostname'] = [lease['hostname']]
414 requested_job_dict[lease['start_time']] = lease
417 job_lease = requested_job_dict[lease['start_time']]
418 if lease['duration'] == job_lease['duration'] :
419 job_lease['hostname'].append(lease['hostname'])
421 return requested_job_dict
423 def _process_requested_jobs(self, rspec):
425 Turns the requested leases and information into a dictionary
426 of requested jobs, grouped by starting time.
428 :param rspec: RSpec received
433 requested_lease_list = self._get_requested_leases_list(rspec)
434 logger.debug("IOTLABDRIVER _process_requested_jobs requested_lease_list \
435 %s"%(requested_lease_list))
436 job_dict = self._group_leases_by_start_time(requested_lease_list)
437 logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\
442 def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \
444 """Answer to CreateSliver.
446 Creates the leases and slivers for the users from the information
447 found in the rspec string.
448 Launch experiment on OAR if the requested leases is valid. Delete
449 no longer requested leases.
452 :param creds: user's credentials
454 :param users: user record list
459 :returns: a valid Rspec for the slice which has just been
465 aggregate = IotlabAggregate(self)
467 slices = IotlabSlices(self)
468 peer = slices.get_peer(slice_hrn)
469 sfa_peer = slices.get_sfa_peer(slice_hrn)
472 if not isinstance(creds, list):
476 slice_record = users[0].get('slice_record', {})
477 logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\
478 creds %s \r\n \r\n users %s" \
480 slice_record['user'] = {'keys':users[0]['keys'], \
481 'email':users[0]['email'], \
482 'hrn':slice_record['reg-researchers'][0]}
484 rspec = RSpec(rspec_string)
485 logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \
486 %s slice_record %s users %s" \
487 %(rspec.version,slice_record, users))
490 # ensure site record exists?
491 # ensure slice record exists
492 #Removed options to verify_slice SA 14/08/12
493 sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \
496 # ensure person records exists
497 #verify_persons returns added persons but since the return value
499 slices.verify_persons(slice_hrn, sfa_slice, users, peer, \
500 sfa_peer, options=options)
501 #requested_attributes returned by rspec.version.get_slice_attributes()
502 #unused, removed SA 13/08/12
503 #rspec.version.get_slice_attributes()
505 logger.debug("IOTLABDRIVER.PY create_sliver slice %s " %(sfa_slice))
507 # add/remove slice from nodes
509 #requested_slivers = [node.get('component_id') \
510 #for node in rspec.version.get_nodes_with_slivers()\
511 #if node.get('authority_id') is self.iotlab_api.root_auth]
512 #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
513 #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
514 #requested_slivers %s listnodes %s" \
515 #%(requested_slivers,l))
516 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
517 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
520 requested_job_dict = self._process_requested_jobs(rspec)
523 logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\
524 %(requested_job_dict))
525 #verify_slice_leases returns the leases , but the return value is unused
526 #here. Removed SA 13/08/12
527 slices.verify_slice_leases(sfa_slice, \
528 requested_job_dict, peer)
530 return aggregate.get_rspec(slice_xrn=slice_urn, \
531 login=sfa_slice['login'], version=rspec.version)
534 def delete_sliver (self, slice_urn, slice_hrn, creds, options):
536 Deletes the lease associated with the slice hrn and the credentials
537 if the slice belongs to iotlab. Answer to DeleteSliver.
539 :returns: 1 if the slice to delete was not found on iotlab,
540 True if the deletion was successful, False otherwise otherwise.
542 .. note:: Should really be named delete_leases because iotlab does
543 not have any slivers, but only deals with leases. However,
544 SFA api only have delete_sliver define so far. SA 13.05/2013
547 sfa_slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \
548 slice_filter_type = 'slice_hrn')
550 if not sfa_slice_list:
553 #Delete all leases in the slice
554 for sfa_slice in sfa_slice_list:
557 logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" %(sfa_slice))
558 slices = IotlabSlices(self)
559 # determine if this is a peer slice
561 peer = slices.get_peer(slice_hrn)
563 logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
564 \r\n \t sfa_slice %s " %(peer, sfa_slice))
567 self.iotlab_api.DeleteSliceFromNodes(sfa_slice)
573 def list_resources (self, slice_urn, slice_hrn, creds, options):
576 List resources from the iotlab aggregate and returns a Rspec
577 advertisement with resources found when slice_urn and slice_hrn are
578 None (in case of resource discovery).
579 If a slice hrn and urn are provided, list experiment's slice
580 nodes in a rspec format. Answer to ListResources.
583 :param options: options used when listing resources (list_leases, info,
585 :returns: rspec string in xml
590 #cached_requested = options.get('cached', True)
592 version_manager = VersionManager()
593 # get the rspec's return format from options
595 version_manager.get_version(options.get('geni_rspec_version'))
596 version_string = "rspec_%s" % (rspec_version)
598 #panos adding the info option to the caching key (can be improved)
599 if options.get('info'):
600 version_string = version_string + "_" + \
601 options.get('info', 'default')
603 # Adding the list_leases option to the caching key
604 if options.get('list_leases'):
605 version_string = version_string + "_" + \
606 options.get('list_leases', 'default')
608 # Adding geni_available to caching key
609 if options.get('geni_available'):
610 version_string = version_string + "_" + \
611 str(options.get('geni_available'))
613 # look in cache first
614 #if cached_requested and self.cache and not slice_hrn:
615 #rspec = self.cache.get(version_string)
617 #logger.debug("IotlabDriver.ListResources: \
618 #returning cached advertisement")
621 #panos: passing user-defined options
622 aggregate = IotlabAggregate(self)
624 rspec = aggregate.get_rspec(slice_xrn=slice_urn, \
625 version=rspec_version, options=options)
628 #if self.cache and not slice_hrn:
629 #logger.debug("Iotlab.ListResources: stores advertisement in cache")
630 #self.cache.add(version_string, rspec)
635 def list_slices (self, creds, options):
636 """Answer to ListSlices.
638 List slices belonging to iotlab, returns slice urns list.
639 No caching used. Options unused but are defined in the SFA method
642 :returns: slice urns list
646 # look in cache first
648 #slices = self.cache.get('slices')
650 #logger.debug("PlDriver.list_slices returns from cache")
655 slices = self.iotlab_api.GetSlices()
656 logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices))
657 slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
659 slice_urns = [hrn_to_urn(slice_hrn, 'slice') \
660 for slice_hrn in slice_hrns]
664 #logger.debug ("IotlabDriver.list_slices stores value in cache")
665 #self.cache.add('slices', slice_urns)
670 def register (self, sfa_record, hrn, pub_key):
672 Adding new user, slice, node or site should not be handled
675 ..warnings:: should not be used. Different components are in charge of
676 doing this task. Adding nodes = OAR
677 Adding users = LDAP Iotlab
678 Adding slice = Import from LDAP users
681 :param sfa_record: record provided by the client of the
683 :type sfa_record: dict
689 def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
691 No site or node record update allowed in Iotlab.
692 The only modifications authorized here are key deletion/addition
693 on an existing user and password change.
694 On an existing user, CAN NOT BE MODIFIED:
695 'first_name', 'last_name', 'email'
696 DOES NOT EXIST IN SENSLAB:
697 'phone', 'url', 'bio','title', 'accepted_aup',
698 A slice is bound to its user, so modifying the user's ssh key should
699 modify the slice's GID after an import procedure.
701 :param old_sfa_record: what is in the db for this hrn
702 :param new_sfa_record: what was passed to the Update call
704 .. seealso::: update in driver.py.
708 pointer = old_sfa_record['pointer']
709 old_sfa_record_type = old_sfa_record['type']
711 # new_key implemented for users only
712 if new_key and old_sfa_record_type not in [ 'user' ]:
713 raise UnknownSfaType(old_sfa_record_type)
716 if old_sfa_record_type == "user":
718 all_fields = new_sfa_record
719 for key in all_fields.keys():
720 if key in ['key', 'password']:
721 update_fields[key] = all_fields[key]
725 # must check this key against the previous one if it exists
726 persons = self.iotlab_api.GetPersons([old_sfa_record])
728 keys = [person['pkey']]
729 #Get all the person's keys
730 keys_dict = self.iotlab_api.GetKeys(keys)
732 # Delete all stale keys, meaning the user has only one key
734 #TODO: do we really want to delete all the other keys?
735 #Is this a problem with the GID generation to have multiple
741 #remove all the other keys
742 for key in keys_dict:
743 self.iotlab_api.DeleteKey(person, key)
744 self.iotlab_api.AddPersonKey(person, \
745 {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} )
746 #self.iotlab_api.AddPersonKey(person, {'key_type': 'ssh', \
751 def remove (self, sfa_record):
754 Removes users only. Mark the user as disabled in
755 LDAP. The user and his slice are then deleted from the
756 db by running an import on the registry.
760 :param sfa_record: record is the existing sfa record in the db
761 :type sfa_record: dict
763 ..warning::As fas as the slice is concerned, here only the leases are
764 removed from the slice. The slice is record itself is not removed
767 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
769 TODO: return boolean for the slice part
771 sfa_record_type = sfa_record['type']
772 hrn = sfa_record['hrn']
773 if sfa_record_type == 'user':
775 #get user from iotlab ldap
776 person = self.iotlab_api.GetPersons(sfa_record)
777 #No registering at a given site in Iotlab.
778 #Once registered to the LDAP, all iotlab sites are
781 #Mark account as disabled in ldap
782 return self.iotlab_api.DeletePerson(sfa_record)
784 elif sfa_record_type == 'slice':
785 if self.iotlab_api.GetSlices(slice_filter = hrn, \
786 slice_filter_type = 'slice_hrn'):
787 ret = self.iotlab_api.DeleteSlice(sfa_record)