2 Implements what a driver should provide for SFA to work.
4 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
5 from sfa.util.sfalogging import logger
6 from sfa.storage.alchemy import dbsession
7 from sfa.storage.model import RegRecord
9 from sfa.managers.driver import Driver
10 from sfa.rspecs.version_manager import VersionManager
11 from sfa.rspecs.rspec import RSpec
13 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
15 from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname
16 from sfa.iotlab.iotlabslices import IotlabSlices
19 from sfa.iotlab.iotlabshell import IotlabShell
22 class IotlabDriver(Driver):
23 """ Iotlab Driver class inherited from Driver generic class.
25 Contains methods compliant with the SFA standard and the testbed
26 infrastructure (calls to LDAP and OAR).
28 .. seealso::: Driver class
31 def __init__(self, api):
34 Sets the iotlab SFA config parameters,
35 instanciates the testbed api and the iotlab database.
37 :param config: iotlab SFA configuration object
38 :type config: Config object
41 Driver.__init__(self, api)
44 self.testbed_shell = IotlabShell(api)
47 def augment_records_with_testbed_info(self, record_list):
50 Adds specific testbed info to the records.
52 :param record_list: list of sfa dictionaries records
53 :type record_list: list
54 :returns: list of records with extended information in each record
58 return self.fill_record_info(record_list)
60 def fill_record_info(self, record_list):
63 For each SFA record, fill in the iotlab specific and SFA specific
66 :param record_list: list of sfa dictionaries records
67 :type record_list: list
68 :returns: list of records with extended information in each record
71 .. warning:: Should not be modifying record_list directly because modi
72 fication are kept outside the method's scope. Howerver, there is no
73 other way to do it given the way it's called in registry manager.
77 logger.debug("IOTLABDRIVER \tfill_record_info records %s "
79 if not isinstance(record_list, list):
80 record_list = [record_list]
83 for record in record_list:
85 if str(record['type']) == 'node':
86 # look for node info using GetNodes
87 # the record is about one node only
88 filter_dict = {'hrn': [record['hrn']]}
89 node_info = self.testbed_shell.GetNodes(filter_dict)
90 # the node_info is about one node only, but it is formatted
92 record.update(node_info[0])
93 logger.debug("IOTLABDRIVER.PY \t \
94 fill_record_info NODE" % (record))
96 #If the record is a SFA slice record, then add information
97 #about the user of this slice. This kind of
98 #information is in the Iotlab's DB.
99 if str(record['type']) == 'slice':
100 if 'reg_researchers' in record and isinstance(record
103 record['reg_researchers'] = \
104 record['reg_researchers'][0].__dict__
106 {'PI': [record['reg_researchers']['hrn']],
107 'researcher': [record['reg_researchers']['hrn']],
108 'name': record['hrn'],
111 'person_ids': [record['reg_researchers']
113 # For client_helper.py compatibility
115 # For client_helper.py compatibility
117 # For client_helper.py compatibility
120 #Get iotlab slice record and oar job id if any.
121 recslice_list = self.testbed_shell.GetSlices(
122 slice_filter=str(record['hrn']),
123 slice_filter_type='slice_hrn')
125 logger.debug("IOTLABDRIVER \tfill_record_info \
126 TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\
127 %s " % (record['hrn'], record['oar_job_id']))
128 del record['reg_researchers']
130 for rec in recslice_list:
131 logger.debug("IOTLABDRIVER\r\n \t \
132 fill_record_info oar_job_id %s "
133 % (rec['oar_job_id']))
135 record['node_ids'] = [self.testbed_shell.root_auth +
136 '.' + hostname for hostname
141 logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \
142 recslice_list %s \r\n \t RECORD %s \r\n \
143 \r\n" % (recslice_list, record))
145 if str(record['type']) == 'user':
146 #The record is a SFA user record.
147 #Get the information about his slice from Iotlab's DB
148 #and add it to the user record.
149 recslice_list = self.testbed_shell.GetSlices(
150 slice_filter=record['record_id'],
151 slice_filter_type='record_id_user')
153 logger.debug("IOTLABDRIVER.PY \t fill_record_info \
154 TYPE USER recslice_list %s \r\n \t RECORD %s \r\n"
155 % (recslice_list, record))
156 #Append slice record in records list,
157 #therefore fetches user and slice info again(one more loop)
158 #Will update PIs and researcher for the slice
160 recuser = recslice_list[0]['reg_researchers']
161 logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \
162 recuser %s \r\n \r\n" % (recuser))
164 recslice = recslice_list[0]
166 {'PI': [recuser['hrn']],
167 'researcher': [recuser['hrn']],
168 'name': record['hrn'],
171 'person_ids': [recuser['record_id']]})
173 for rec in recslice_list:
174 recslice['oar_job_id'].append(rec['oar_job_id'])
178 recslice.update({'type': 'slice',
179 'hrn': recslice_list[0]['hrn']})
181 #GetPersons takes [] as filters
182 user_iotlab = self.testbed_shell.GetPersons([record])
184 record.update(user_iotlab[0])
185 #For client_helper.py compatibility
190 record_list.append(recslice)
192 logger.debug("IOTLABDRIVER.PY \t \
193 fill_record_info ADDING SLICE\
194 INFO TO USER records %s" % (record_list))
196 except TypeError, error:
197 logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"
202 def sliver_status(self, slice_urn, slice_hrn):
204 Receive a status request for slice named urn/hrn
205 urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
206 shall return a structure as described in
207 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
208 NT : not sure if we should implement this or not, but used by sface.
210 :param slice_urn: slice urn
211 :type slice_urn: string
212 :param slice_hrn: slice hrn
213 :type slice_hrn: string
217 #First get the slice with the slice hrn
218 slice_list = self.testbed_shell.GetSlices(slice_filter=slice_hrn,
219 slice_filter_type='slice_hrn')
221 if len(slice_list) == 0:
222 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
224 #Used for fetching the user info witch comes along the slice info
225 one_slice = slice_list[0]
227 #Make a list of all the nodes hostnames in use for this slice
228 slice_nodes_list = []
229 slice_nodes_list = one_slice['node_ids']
230 #Get all the corresponding nodes details
231 nodes_all = self.testbed_shell.GetNodes(
232 {'hostname': slice_nodes_list},
233 ['node_id', 'hostname', 'site', 'boot_state'])
234 nodeall_byhostname = dict([(one_node['hostname'], one_node)
235 for one_node in nodes_all])
237 for single_slice in slice_list:
239 top_level_status = 'empty'
242 ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status',
243 'geni_resources'], None)
245 # ['geni_urn','geni_error', 'pl_login','geni_status',
246 # 'geni_resources'], None)
247 # result['pl_login'] = one_slice['reg_researchers'][0].hrn
248 result['iotlab_login'] = one_slice['user']
249 logger.debug("Slabdriver - sliver_status Sliver status \
250 urn %s hrn %s single_slice %s \r\n "
251 % (slice_urn, slice_hrn, single_slice))
253 if 'node_ids' not in single_slice:
255 result['geni_status'] = top_level_status
256 result['geni_resources'] = []
259 top_level_status = 'ready'
261 #A job is running on Iotlab for this slice
262 # report about the local nodes that are in the slice only
264 result['geni_urn'] = slice_urn
267 for node_hostname in single_slice['node_ids']:
269 res['iotlab_hostname'] = node_hostname
270 res['iotlab_boot_state'] = \
271 nodeall_byhostname[node_hostname]['boot_state']
273 #res['pl_hostname'] = node['hostname']
274 #res['pl_boot_state'] = \
275 #nodeall_byhostname[node['hostname']]['boot_state']
276 #res['pl_last_contact'] = strftime(self.time_format, \
277 #gmtime(float(timestamp)))
279 slice_urn, type='slice',
280 id=nodeall_byhostname[node_hostname]['node_id']).urn
282 res['geni_urn'] = sliver_id
283 #node_name = node['hostname']
284 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
286 res['geni_status'] = 'ready'
288 res['geni_status'] = 'failed'
289 top_level_status = 'failed'
291 res['geni_error'] = ''
293 resources.append(res)
295 result['geni_status'] = top_level_status
296 result['geni_resources'] = resources
297 logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "
302 def get_user_record(hrn):
305 Returns the user record based on the hrn from the SFA DB .
307 :param hrn: user's hrn
309 :returns: user record from SFA database
313 return dbsession.query(RegRecord).filter_by(hrn=hrn).first()
315 def testbed_name(self):
318 Returns testbed's name.
319 :returns: testbed authority name.
325 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
326 def aggregate_version(self):
329 Returns the testbed's supported rspec advertisement and request
331 :returns: rspec versions supported ad a dictionary.
335 version_manager = VersionManager()
336 ad_rspec_versions = []
337 request_rspec_versions = []
338 for rspec_version in version_manager.versions:
339 if rspec_version.content_type in ['*', 'ad']:
340 ad_rspec_versions.append(rspec_version.to_dict())
341 if rspec_version.content_type in ['*', 'request']:
342 request_rspec_versions.append(rspec_version.to_dict())
344 'testbed': self.testbed_name(),
345 'geni_request_rspec_versions': request_rspec_versions,
346 'geni_ad_rspec_versions': ad_rspec_versions}
348 def _get_requested_leases_list(self, rspec):
350 Process leases in rspec depending on the rspec version (format)
351 type. Find the lease requests in the rspec and creates
352 a lease request list with the mandatory information ( nodes,
353 start time and duration) of the valid leases (duration above or
354 equal to the iotlab experiment minimum duration).
356 :param rspec: rspec request received.
358 :returns: list of lease requests found in the rspec
361 requested_lease_list = []
362 for lease in rspec.version.get_leases():
363 single_requested_lease = {}
364 logger.debug("IOTLABDRIVER.PY \t \
365 _get_requested_leases_list lease %s " % (lease))
367 if not lease.get('lease_id'):
368 if get_authority(lease['component_id']) == \
369 self.testbed_shell.root_auth:
370 single_requested_lease['hostname'] = \
371 iotlab_xrn_to_hostname(\
372 lease.get('component_id').strip())
373 single_requested_lease['start_time'] = \
374 lease.get('start_time')
375 single_requested_lease['duration'] = lease.get('duration')
376 #Check the experiment's duration is valid before adding
377 #the lease to the requested leases list
378 duration_in_seconds = \
379 int(single_requested_lease['duration'])
380 if duration_in_seconds >= self.testbed_shell.GetMinExperimentDurationInGranularity():
381 requested_lease_list.append(single_requested_lease)
383 return requested_lease_list
386 def _group_leases_by_start_time(requested_lease_list):
388 Create dict of leases by start_time, regrouping nodes reserved
389 at the same time, for the same amount of time so as to
390 define one job on OAR.
392 :param requested_lease_list: list of leases
393 :type requested_lease_list: list
394 :returns: Dictionary with key = start time, value = list of leases
395 with the same start time.
400 requested_xp_dict = {}
401 for lease in requested_lease_list:
403 #In case it is an asap experiment start_time is empty
404 if lease['start_time'] == '':
405 lease['start_time'] = '0'
407 if lease['start_time'] not in requested_xp_dict:
408 if isinstance(lease['hostname'], str):
409 lease['hostname'] = [lease['hostname']]
411 requested_xp_dict[lease['start_time']] = lease
414 job_lease = requested_xp_dict[lease['start_time']]
415 if lease['duration'] == job_lease['duration']:
416 job_lease['hostname'].append(lease['hostname'])
418 return requested_xp_dict
420 def _process_requested_xp_dict(self, rspec):
422 Turns the requested leases and information into a dictionary
423 of requested jobs, grouped by starting time.
425 :param rspec: RSpec received
430 requested_lease_list = self._get_requested_leases_list(rspec)
431 logger.debug("IOTLABDRIVER _process_requested_xp_dict \
432 requested_lease_list %s" % (requested_lease_list))
433 xp_dict = self._group_leases_by_start_time(requested_lease_list)
434 logger.debug("IOTLABDRIVER _process_requested_xp_dict xp_dict\
439 def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string,
441 """Answer to CreateSliver.
443 Creates the leases and slivers for the users from the information
444 found in the rspec string.
445 Launch experiment on OAR if the requested leases is valid. Delete
446 no longer requested leases.
449 :param creds: user's credentials
451 :param users: user record list
456 :returns: a valid Rspec for the slice which has just been
462 aggregate = IotlabAggregate(self)
464 slices = IotlabSlices(self)
465 peer = slices.get_peer(slice_hrn)
466 sfa_peer = slices.get_sfa_peer(slice_hrn)
469 if not isinstance(creds, list):
473 slice_record = users[0].get('slice_record', {})
474 logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\
475 creds %s \r\n \r\n users %s"
477 slice_record['user'] = {'keys': users[0]['keys'],
478 'email': users[0]['email'],
479 'hrn': slice_record['reg-researchers'][0]}
481 rspec = RSpec(rspec_string)
482 logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \
483 %s slice_record %s users %s"
484 % (rspec.version, slice_record, users))
486 # ensure site record exists?
487 # ensure slice record exists
488 #Removed options in verify_slice SA 14/08/12
489 #Removed peer record in verify_slice SA 18/07/13
490 sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer)
492 # ensure person records exists
493 #verify_persons returns added persons but the return value
495 #Removed peer record and sfa_peer in verify_persons SA 18/07/13
496 slices.verify_persons(slice_hrn, sfa_slice, users, options=options)
497 #requested_attributes returned by rspec.version.get_slice_attributes()
498 #unused, removed SA 13/08/12
499 #rspec.version.get_slice_attributes()
501 logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice))
503 # add/remove slice from nodes
505 #requested_slivers = [node.get('component_id') \
506 #for node in rspec.version.get_nodes_with_slivers()\
507 #if node.get('authority_id') is self.testbed_shell.root_auth]
508 #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
509 #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
510 #requested_slivers %s listnodes %s" \
511 #%(requested_slivers,l))
512 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
513 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
515 requested_xp_dict = self._process_requested_xp_dict(rspec)
517 logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_xp_dict %s "
518 % (requested_xp_dict))
519 #verify_slice_leases returns the leases , but the return value is unused
520 #here. Removed SA 13/08/12
521 slices.verify_slice_leases(sfa_slice,
522 requested_xp_dict, peer)
524 return aggregate.get_rspec(slice_xrn=slice_urn,
525 login=sfa_slice['login'],
526 version=rspec.version)
528 def delete_sliver(self, slice_urn, slice_hrn, creds, options):
530 Deletes the lease associated with the slice hrn and the credentials
531 if the slice belongs to iotlab. Answer to DeleteSliver.
533 :param slice_urn: urn of the slice
534 :param slice_hrn: name of the slice
535 :param creds: slice credenials
536 :type slice_urn: string
537 :type slice_hrn: string
538 :type creds: ? unused
540 :returns: 1 if the slice to delete was not found on iotlab,
541 True if the deletion was successful, False otherwise otherwise.
543 .. note:: Should really be named delete_leases because iotlab does
544 not have any slivers, but only deals with leases. However,
545 SFA api only have delete_sliver define so far. SA 13/05/2013
546 .. note:: creds are unused, and are not used either in the dummy driver
550 sfa_slice_list = self.testbed_shell.GetSlices(
551 slice_filter=slice_hrn,
552 slice_filter_type='slice_hrn')
554 if not sfa_slice_list:
557 #Delete all leases in the slice
558 for sfa_slice in sfa_slice_list:
559 logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice))
560 slices = IotlabSlices(self)
561 # determine if this is a peer slice
563 peer = slices.get_peer(slice_hrn)
565 logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
566 \r\n \t sfa_slice %s " % (peer, sfa_slice))
568 self.testbed_shell.DeleteSliceFromNodes(sfa_slice)
573 def list_resources (self, slice_urn, slice_hrn, creds, options):
576 List resources from the iotlab aggregate and returns a Rspec
577 advertisement with resources found when slice_urn and slice_hrn are
578 None (in case of resource discovery).
579 If a slice hrn and urn are provided, list experiment's slice
580 nodes in a rspec format. Answer to ListResources.
583 :param slice_urn: urn of the slice
584 :param slice_hrn: name of the slice
585 :param creds: slice credenials
586 :type slice_urn: string
587 :type slice_hrn: string
588 :type creds: ? unused
589 :param options: options used when listing resources (list_leases, info,
591 :returns: rspec string in xml
594 .. note:: creds are unused
597 #cached_requested = options.get('cached', True)
599 version_manager = VersionManager()
600 # get the rspec's return format from options
602 version_manager.get_version(options.get('geni_rspec_version'))
603 version_string = "rspec_%s" % (rspec_version)
605 #panos adding the info option to the caching key (can be improved)
606 if options.get('info'):
607 version_string = version_string + "_" + \
608 options.get('info', 'default')
610 # Adding the list_leases option to the caching key
611 if options.get('list_leases'):
612 version_string = version_string + "_" + \
613 options.get('list_leases', 'default')
615 # Adding geni_available to caching key
616 if options.get('geni_available'):
617 version_string = version_string + "_" + \
618 str(options.get('geni_available'))
620 # look in cache first
621 #if cached_requested and self.cache and not slice_hrn:
622 #rspec = self.cache.get(version_string)
624 #logger.debug("IotlabDriver.ListResources: \
625 #returning cached advertisement")
628 #panos: passing user-defined options
629 aggregate = IotlabAggregate(self)
631 rspec = aggregate.get_rspec(slice_xrn=slice_urn,
632 version=rspec_version, options=options)
635 #if self.cache and not slice_hrn:
636 #logger.debug("Iotlab.ListResources: stores advertisement in cache")
637 #self.cache.add(version_string, rspec)
642 def list_slices(self, creds, options):
643 """Answer to ListSlices.
645 List slices belonging to iotlab, returns slice urns list.
646 No caching used. Options unused but are defined in the SFA method
649 :returns: slice urns list
652 .. note:: creds are unused
654 # look in cache first
656 #slices = self.cache.get('slices')
658 #logger.debug("PlDriver.list_slices returns from cache")
663 slices = self.testbed_shell.GetSlices()
664 logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n"
666 slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
668 slice_urns = [hrn_to_urn(slice_hrn, 'slice')
669 for slice_hrn in slice_hrns]
673 #logger.debug ("IotlabDriver.list_slices stores value in cache")
674 #self.cache.add('slices', slice_urns)
679 def register(self, sfa_record, hrn, pub_key):
681 Adding new user, slice, node or site should not be handled
684 ..warnings:: should not be used. Different components are in charge of
685 doing this task. Adding nodes = OAR
686 Adding users = LDAP Iotlab
687 Adding slice = Import from LDAP users
690 :param sfa_record: record provided by the client of the
692 :type sfa_record: dict
693 :param pub_key: public key of the user
694 :type pub_key: string
696 .. note:: DOES NOTHING. Returns -1.
702 def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
704 No site or node record update allowed in Iotlab. The only modifications
705 authorized here are key deletion/addition on an existing user and
706 password change. On an existing user, CAN NOT BE MODIFIED: 'first_name',
707 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio',
708 'title', 'accepted_aup'. A slice is bound to its user, so modifying the
709 user's ssh key should nmodify the slice's GID after an import procedure.
711 :param old_sfa_record: what is in the db for this hrn
712 :param new_sfa_record: what was passed to the update call
713 :param new_key: the new user's public key
714 :param hrn: the user's sfa hrn
715 :type old_sfa_record: dict
716 :type new_sfa_record: dict
717 :type new_key: string
721 .. seealso:: update in driver.py.
724 pointer = old_sfa_record['pointer']
725 old_sfa_record_type = old_sfa_record['type']
727 # new_key implemented for users only
728 if new_key and old_sfa_record_type not in ['user']:
729 raise UnknownSfaType(old_sfa_record_type)
731 if old_sfa_record_type == "user":
733 all_fields = new_sfa_record
734 for key in all_fields.keys():
735 if key in ['key', 'password']:
736 update_fields[key] = all_fields[key]
739 # must check this key against the previous one if it exists
740 persons = self.testbed_shell.GetPersons([old_sfa_record])
742 keys = [person['pkey']]
743 #Get all the person's keys
744 keys_dict = self.testbed_shell.GetKeys(keys)
746 # Delete all stale keys, meaning the user has only one key
748 #TODO: do we really want to delete all the other keys?
749 #Is this a problem with the GID generation to have multiple
755 #remove all the other keys
756 for key in keys_dict:
757 self.testbed_shell.DeleteKey(person, key)
758 self.testbed_shell.AddPersonKey(
759 person, {'sshPublicKey': person['pkey']},
760 {'sshPublicKey': new_key})
763 def remove(self, sfa_record):
766 Removes users only. Mark the user as disabled in LDAP. The user and his
767 slice are then deleted from the db by running an import on the registry.
769 :param sfa_record: record is the existing sfa record in the db
770 :type sfa_record: dict
772 ..warning::As fas as the slice is concerned, here only the leases are
773 removed from the slice. The slice is record itself is not removed
778 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
780 TODO: return boolean for the slice part
782 sfa_record_type = sfa_record['type']
783 hrn = sfa_record['hrn']
784 if sfa_record_type == 'user':
786 #get user from iotlab ldap
787 person = self.testbed_shell.GetPersons(sfa_record)
788 #No registering at a given site in Iotlab.
789 #Once registered to the LDAP, all iotlab sites are
792 #Mark account as disabled in ldap
793 return self.testbed_shell.DeletePerson(sfa_record)
795 elif sfa_record_type == 'slice':
796 if self.testbed_shell.GetSlices(slice_filter=hrn,
797 slice_filter_type='slice_hrn'):
798 ret = self.testbed_shell.DeleteSlice(sfa_record)