2 Implements what a driver should provide for SFA to work.
4 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
5 from sfa.util.sfalogging import logger
6 from sfa.storage.alchemy import dbsession
7 from sfa.storage.model import RegRecord
9 from sfa.managers.driver import Driver
10 from sfa.rspecs.version_manager import VersionManager
11 from sfa.rspecs.rspec import RSpec
13 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
15 from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname
16 from sfa.iotlab.iotlabslices import IotlabSlices
19 from sfa.iotlab.iotlabapi import IotlabTestbedAPI
22 class IotlabDriver(Driver):
23 """ Iotlab Driver class inherited from Driver generic class.
25 Contains methods compliant with the SFA standard and the testbed
26 infrastructure (calls to LDAP and OAR).
28 .. seealso::: Driver class
31 def __init__(self, config):
34 Sets the iotlab SFA config parameters,
35 instanciates the testbed api and the iotlab database.
37 :param config: iotlab SFA configuration object
38 :type config: Config object
41 Driver.__init__(self, config)
43 self.iotlab_api = IotlabTestbedAPI(config)
46 def augment_records_with_testbed_info(self, record_list):
49 Adds specific testbed info to the records.
51 :param record_list: list of sfa dictionaries records
52 :type record_list: list
53 :returns: list of records with extended information in each record
57 return self.fill_record_info(record_list)
59 def fill_record_info(self, record_list):
62 For each SFA record, fill in the iotlab specific and SFA specific
65 :param record_list: list of sfa dictionaries records
66 :type record_list: list
67 :returns: list of records with extended information in each record
70 .. warning:: Should not be modifying record_list directly because modi
71 fication are kept outside the method's scope. Howerver, there is no
72 other way to do it given the way it's called in registry manager.
76 logger.debug("IOTLABDRIVER \tfill_record_info records %s "
78 if not isinstance(record_list, list):
79 record_list = [record_list]
82 for record in record_list:
83 #If the record is a SFA slice record, then add information
84 #about the user of this slice. This kind of
85 #information is in the Iotlab's DB.
86 if str(record['type']) == 'slice':
87 if 'reg_researchers' in record and \
88 isinstance(record['reg_researchers'], list):
89 record['reg_researchers'] = \
90 record['reg_researchers'][0].__dict__
92 {'PI': [record['reg_researchers']['hrn']],
93 'researcher': [record['reg_researchers']['hrn']],
94 'name': record['hrn'],
97 'person_ids': [record['reg_researchers']
99 # For client_helper.py compatibility
101 # For client_helper.py compatibility
103 # For client_helper.py compatibility
106 #Get iotlab slice record and oar job id if any.
107 recslice_list = self.iotlab_api.GetSlices(
108 slice_filter=str(record['hrn']),
109 slice_filter_type='slice_hrn')
111 logger.debug("IOTLABDRIVER \tfill_record_info \
112 TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\
113 %s " % (record['hrn'], record['oar_job_id']))
114 del record['reg_researchers']
116 for rec in recslice_list:
117 logger.debug("IOTLABDRIVER\r\n \t \
118 fill_record_info oar_job_id %s "
119 % (rec['oar_job_id']))
121 record['node_ids'] = [self.iotlab_api.root_auth +
122 hostname for hostname in
127 logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \
128 recslice_list %s \r\n \t RECORD %s \r\n \
129 \r\n" % (recslice_list, record))
131 if str(record['type']) == 'user':
132 #The record is a SFA user record.
133 #Get the information about his slice from Iotlab's DB
134 #and add it to the user record.
135 recslice_list = self.iotlab_api.GetSlices(
136 slice_filter=record['record_id'],
137 slice_filter_type='record_id_user')
139 logger.debug("IOTLABDRIVER.PY \t fill_record_info \
140 TYPE USER recslice_list %s \r\n \t RECORD %s \r\n"
141 % (recslice_list, record))
142 #Append slice record in records list,
143 #therefore fetches user and slice info again(one more loop)
144 #Will update PIs and researcher for the slice
146 recuser = recslice_list[0]['reg_researchers']
147 logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \
148 recuser %s \r\n \r\n" % (recuser))
150 recslice = recslice_list[0]
152 {'PI': [recuser['hrn']],
153 'researcher': [recuser['hrn']],
154 'name': record['hrn'],
157 'person_ids': [recuser['record_id']]})
159 for rec in recslice_list:
160 recslice['oar_job_id'].append(rec['oar_job_id'])
164 recslice.update({'type': 'slice',
165 'hrn': recslice_list[0]['hrn']})
167 #GetPersons takes [] as filters
168 user_iotlab = self.iotlab_api.GetPersons([record])
170 record.update(user_iotlab[0])
171 #For client_helper.py compatibility
176 record_list.append(recslice)
178 logger.debug("IOTLABDRIVER.PY \t \
179 fill_record_info ADDING SLICE\
180 INFO TO USER records %s" % (record_list))
182 except TypeError, error:
183 logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"
188 def sliver_status(self, slice_urn, slice_hrn):
190 Receive a status request for slice named urn/hrn
191 urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
192 shall return a structure as described in
193 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
194 NT : not sure if we should implement this or not, but used by sface.
196 :param slice_urn: slice urn
197 :type slice_urn: string
198 :param slice_hrn: slice hrn
199 :type slice_hrn: string
203 #First get the slice with the slice hrn
204 slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn,
205 slice_filter_type='slice_hrn')
207 if len(slice_list) == 0:
208 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
210 #Used for fetching the user info witch comes along the slice info
211 one_slice = slice_list[0]
213 #Make a list of all the nodes hostnames in use for this slice
214 slice_nodes_list = []
215 slice_nodes_list = one_slice['node_ids']
216 #Get all the corresponding nodes details
217 nodes_all = self.iotlab_api.GetNodes(
218 {'hostname': slice_nodes_list},
219 ['node_id', 'hostname', 'site', 'boot_state'])
220 nodeall_byhostname = dict([(one_node['hostname'], one_node)
221 for one_node in nodes_all])
223 for single_slice in slice_list:
225 top_level_status = 'empty'
228 ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status',
229 'geni_resources'], None)
231 # ['geni_urn','geni_error', 'pl_login','geni_status',
232 # 'geni_resources'], None)
233 # result['pl_login'] = one_slice['reg_researchers'][0].hrn
234 result['iotlab_login'] = one_slice['user']
235 logger.debug("Slabdriver - sliver_status Sliver status \
236 urn %s hrn %s single_slice %s \r\n "
237 % (slice_urn, slice_hrn, single_slice))
239 if 'node_ids' not in single_slice:
241 result['geni_status'] = top_level_status
242 result['geni_resources'] = []
245 top_level_status = 'ready'
247 #A job is running on Iotlab for this slice
248 # report about the local nodes that are in the slice only
250 result['geni_urn'] = slice_urn
253 for node_hostname in single_slice['node_ids']:
255 res['iotlab_hostname'] = node_hostname
256 res['iotlab_boot_state'] = \
257 nodeall_byhostname[node_hostname]['boot_state']
259 #res['pl_hostname'] = node['hostname']
260 #res['pl_boot_state'] = \
261 #nodeall_byhostname[node['hostname']]['boot_state']
262 #res['pl_last_contact'] = strftime(self.time_format, \
263 #gmtime(float(timestamp)))
265 slice_urn, type='slice',
266 id=nodeall_byhostname[node_hostname]['node_id']).urn
268 res['geni_urn'] = sliver_id
269 #node_name = node['hostname']
270 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
272 res['geni_status'] = 'ready'
274 res['geni_status'] = 'failed'
275 top_level_status = 'failed'
277 res['geni_error'] = ''
279 resources.append(res)
281 result['geni_status'] = top_level_status
282 result['geni_resources'] = resources
283 logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "
288 def get_user_record(hrn):
291 Returns the user record based on the hrn from the SFA DB .
293 :param hrn: user's hrn
295 :returns: user record from SFA database
299 return dbsession.query(RegRecord).filter_by(hrn=hrn).first()
301 def testbed_name(self):
304 Returns testbed's name.
305 :returns: testbed authority name.
311 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
312 def aggregate_version(self):
315 Returns the testbed's supported rspec advertisement and request
317 :returns: rspec versions supported ad a dictionary.
321 version_manager = VersionManager()
322 ad_rspec_versions = []
323 request_rspec_versions = []
324 for rspec_version in version_manager.versions:
325 if rspec_version.content_type in ['*', 'ad']:
326 ad_rspec_versions.append(rspec_version.to_dict())
327 if rspec_version.content_type in ['*', 'request']:
328 request_rspec_versions.append(rspec_version.to_dict())
330 'testbed': self.testbed_name(),
331 'geni_request_rspec_versions': request_rspec_versions,
332 'geni_ad_rspec_versions': ad_rspec_versions}
334 def _get_requested_leases_list(self, rspec):
336 Process leases in rspec depending on the rspec version (format)
337 type. Find the lease requests in the rspec and creates
338 a lease request list with the mandatory information ( nodes,
339 start time and duration) of the valid leases (duration above or
340 equal to the iotlab experiment minimum duration).
342 :param rspec: rspec request received.
344 :returns: list of lease requests found in the rspec
347 requested_lease_list = []
348 for lease in rspec.version.get_leases():
349 single_requested_lease = {}
350 logger.debug("IOTLABDRIVER.PY \t \
351 _get_requested_leases_list lease %s " % (lease))
353 if not lease.get('lease_id'):
354 if get_authority(lease['component_id']) == \
355 self.iotlab_api.root_auth:
356 single_requested_lease['hostname'] = \
357 iotlab_xrn_to_hostname(\
358 lease.get('component_id').strip())
359 single_requested_lease['start_time'] = \
360 lease.get('start_time')
361 single_requested_lease['duration'] = lease.get('duration')
362 #Check the experiment's duration is valid before adding
363 #the lease to the requested leases list
364 duration_in_seconds = \
365 int(single_requested_lease['duration'])
366 if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec():
367 requested_lease_list.append(single_requested_lease)
369 return requested_lease_list
372 def _group_leases_by_start_time(requested_lease_list):
374 Create dict of leases by start_time, regrouping nodes reserved
375 at the same time, for the same amount of time so as to
376 define one job on OAR.
378 :param requested_lease_list: list of leases
379 :type requested_lease_list: list
380 :returns: Dictionary with key = start time, value = list of leases
381 with the same start time.
386 requested_job_dict = {}
387 for lease in requested_lease_list:
389 #In case it is an asap experiment start_time is empty
390 if lease['start_time'] == '':
391 lease['start_time'] = '0'
393 if lease['start_time'] not in requested_job_dict:
394 if isinstance(lease['hostname'], str):
395 lease['hostname'] = [lease['hostname']]
397 requested_job_dict[lease['start_time']] = lease
400 job_lease = requested_job_dict[lease['start_time']]
401 if lease['duration'] == job_lease['duration']:
402 job_lease['hostname'].append(lease['hostname'])
404 return requested_job_dict
406 def _process_requested_jobs(self, rspec):
408 Turns the requested leases and information into a dictionary
409 of requested jobs, grouped by starting time.
411 :param rspec: RSpec received
416 requested_lease_list = self._get_requested_leases_list(rspec)
417 logger.debug("IOTLABDRIVER _process_requested_jobs \
418 requested_lease_list %s" % (requested_lease_list))
419 job_dict = self._group_leases_by_start_time(requested_lease_list)
420 logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\
425 def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string,
427 """Answer to CreateSliver.
429 Creates the leases and slivers for the users from the information
430 found in the rspec string.
431 Launch experiment on OAR if the requested leases is valid. Delete
432 no longer requested leases.
435 :param creds: user's credentials
437 :param users: user record list
442 :returns: a valid Rspec for the slice which has just been
448 aggregate = IotlabAggregate(self)
450 slices = IotlabSlices(self)
451 peer = slices.get_peer(slice_hrn)
452 sfa_peer = slices.get_sfa_peer(slice_hrn)
455 if not isinstance(creds, list):
459 slice_record = users[0].get('slice_record', {})
460 logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\
461 creds %s \r\n \r\n users %s"
463 slice_record['user'] = {'keys': users[0]['keys'],
464 'email': users[0]['email'],
465 'hrn': slice_record['reg-researchers'][0]}
467 rspec = RSpec(rspec_string)
468 logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \
469 %s slice_record %s users %s"
470 % (rspec.version, slice_record, users))
472 # ensure site record exists?
473 # ensure slice record exists
474 #Removed options in verify_slice SA 14/08/12
475 #Removed peer record in verify_slice SA 18/07/13
476 sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer)
478 # ensure person records exists
479 #verify_persons returns added persons but the return value
481 #Removed peer record and sfa_peer in verify_persons SA 18/07/13
482 slices.verify_persons(slice_hrn, sfa_slice, users, options=options)
483 #requested_attributes returned by rspec.version.get_slice_attributes()
484 #unused, removed SA 13/08/12
485 #rspec.version.get_slice_attributes()
487 logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice))
489 # add/remove slice from nodes
491 #requested_slivers = [node.get('component_id') \
492 #for node in rspec.version.get_nodes_with_slivers()\
493 #if node.get('authority_id') is self.iotlab_api.root_auth]
494 #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
495 #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
496 #requested_slivers %s listnodes %s" \
497 #%(requested_slivers,l))
498 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
499 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
501 requested_job_dict = self._process_requested_jobs(rspec)
503 logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "
504 % (requested_job_dict))
505 #verify_slice_leases returns the leases , but the return value is unused
506 #here. Removed SA 13/08/12
507 slices.verify_slice_leases(sfa_slice,
508 requested_job_dict, peer)
510 return aggregate.get_rspec(slice_xrn=slice_urn,
511 login=sfa_slice['login'],
512 version=rspec.version)
514 def delete_sliver(self, slice_urn, slice_hrn, creds, options):
516 Deletes the lease associated with the slice hrn and the credentials
517 if the slice belongs to iotlab. Answer to DeleteSliver.
519 :param slice_urn: urn of the slice
520 :param slice_hrn: name of the slice
521 :param creds: slice credenials
522 :type slice_urn: string
523 :type slice_hrn: string
524 :type creds: ? unused
526 :returns: 1 if the slice to delete was not found on iotlab,
527 True if the deletion was successful, False otherwise otherwise.
529 .. note:: Should really be named delete_leases because iotlab does
530 not have any slivers, but only deals with leases. However,
531 SFA api only have delete_sliver define so far. SA 13/05/2013
532 .. note:: creds are unused, and are not used either in the dummy driver
536 sfa_slice_list = self.iotlab_api.GetSlices(
537 slice_filter=slice_hrn,
538 slice_filter_type='slice_hrn')
540 if not sfa_slice_list:
543 #Delete all leases in the slice
544 for sfa_slice in sfa_slice_list:
545 logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice))
546 slices = IotlabSlices(self)
547 # determine if this is a peer slice
549 peer = slices.get_peer(slice_hrn)
551 logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
552 \r\n \t sfa_slice %s " % (peer, sfa_slice))
554 self.iotlab_api.DeleteSliceFromNodes(sfa_slice)
559 def list_resources (self, slice_urn, slice_hrn, creds, options):
562 List resources from the iotlab aggregate and returns a Rspec
563 advertisement with resources found when slice_urn and slice_hrn are
564 None (in case of resource discovery).
565 If a slice hrn and urn are provided, list experiment's slice
566 nodes in a rspec format. Answer to ListResources.
569 :param slice_urn: urn of the slice
570 :param slice_hrn: name of the slice
571 :param creds: slice credenials
572 :type slice_urn: string
573 :type slice_hrn: string
574 :type creds: ? unused
575 :param options: options used when listing resources (list_leases, info,
577 :returns: rspec string in xml
580 .. note:: creds are unused
583 #cached_requested = options.get('cached', True)
585 version_manager = VersionManager()
586 # get the rspec's return format from options
588 version_manager.get_version(options.get('geni_rspec_version'))
589 version_string = "rspec_%s" % (rspec_version)
591 #panos adding the info option to the caching key (can be improved)
592 if options.get('info'):
593 version_string = version_string + "_" + \
594 options.get('info', 'default')
596 # Adding the list_leases option to the caching key
597 if options.get('list_leases'):
598 version_string = version_string + "_" + \
599 options.get('list_leases', 'default')
601 # Adding geni_available to caching key
602 if options.get('geni_available'):
603 version_string = version_string + "_" + \
604 str(options.get('geni_available'))
606 # look in cache first
607 #if cached_requested and self.cache and not slice_hrn:
608 #rspec = self.cache.get(version_string)
610 #logger.debug("IotlabDriver.ListResources: \
611 #returning cached advertisement")
614 #panos: passing user-defined options
615 aggregate = IotlabAggregate(self)
617 rspec = aggregate.get_rspec(slice_xrn=slice_urn,
618 version=rspec_version, options=options)
621 #if self.cache and not slice_hrn:
622 #logger.debug("Iotlab.ListResources: stores advertisement in cache")
623 #self.cache.add(version_string, rspec)
628 def list_slices(self, creds, options):
629 """Answer to ListSlices.
631 List slices belonging to iotlab, returns slice urns list.
632 No caching used. Options unused but are defined in the SFA method
635 :returns: slice urns list
638 .. note:: creds are unused
640 # look in cache first
642 #slices = self.cache.get('slices')
644 #logger.debug("PlDriver.list_slices returns from cache")
649 slices = self.iotlab_api.GetSlices()
650 logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n"
652 slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
654 slice_urns = [hrn_to_urn(slice_hrn, 'slice')
655 for slice_hrn in slice_hrns]
659 #logger.debug ("IotlabDriver.list_slices stores value in cache")
660 #self.cache.add('slices', slice_urns)
665 def register(self, sfa_record, hrn, pub_key):
667 Adding new user, slice, node or site should not be handled
670 ..warnings:: should not be used. Different components are in charge of
671 doing this task. Adding nodes = OAR
672 Adding users = LDAP Iotlab
673 Adding slice = Import from LDAP users
676 :param sfa_record: record provided by the client of the
678 :type sfa_record: dict
679 :param pub_key: public key of the user
680 :type pub_key: string
682 .. note:: DOES NOTHING. Returns -1.
688 def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
690 No site or node record update allowed in Iotlab.
691 The only modifications authorized here are key deletion/addition
692 on an existing user and password change.
693 On an existing user, CAN NOT BE MODIFIED:
694 'first_name', 'last_name', 'email'
695 DOES NOT EXIST IN SENSLAB:
696 'phone', 'url', 'bio','title', 'accepted_aup',
697 A slice is bound to its user, so modifying the user's ssh key should
698 modify the slice's GID after an import procedure.
700 :param old_sfa_record: what is in the db for this hrn
701 :param new_sfa_record: what was passed to the Update call
702 :param new_key: the new user's public key
703 :param hrn: the user's sfa hrn
704 :type old_sfa_record: dictionary
705 :type new_sfa_record: dictionary
706 :type pub_key: string
710 .. seealso::: update in driver.py.
713 pointer = old_sfa_record['pointer']
714 old_sfa_record_type = old_sfa_record['type']
716 # new_key implemented for users only
717 if new_key and old_sfa_record_type not in ['user']:
718 raise UnknownSfaType(old_sfa_record_type)
720 if old_sfa_record_type == "user":
722 all_fields = new_sfa_record
723 for key in all_fields.keys():
724 if key in ['key', 'password']:
725 update_fields[key] = all_fields[key]
728 # must check this key against the previous one if it exists
729 persons = self.iotlab_api.GetPersons([old_sfa_record])
731 keys = [person['pkey']]
732 #Get all the person's keys
733 keys_dict = self.iotlab_api.GetKeys(keys)
735 # Delete all stale keys, meaning the user has only one key
737 #TODO: do we really want to delete all the other keys?
738 #Is this a problem with the GID generation to have multiple
744 #remove all the other keys
745 for key in keys_dict:
746 self.iotlab_api.DeleteKey(person, key)
747 self.iotlab_api.AddPersonKey(
748 person, {'sshPublicKey': person['pkey']},
749 {'sshPublicKey': new_key})
752 def remove(self, sfa_record):
755 Removes users only. Mark the user as disabled in
756 LDAP. The user and his slice are then deleted from the
757 db by running an import on the registry.
759 :param sfa_record: record is the existing sfa record in the db
760 :type sfa_record: dict
762 ..warning::As fas as the slice is concerned, here only the leases are
763 removed from the slice. The slice is record itself is not removed
767 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
769 TODO: return boolean for the slice part
771 sfa_record_type = sfa_record['type']
772 hrn = sfa_record['hrn']
773 if sfa_record_type == 'user':
775 #get user from iotlab ldap
776 person = self.iotlab_api.GetPersons(sfa_record)
777 #No registering at a given site in Iotlab.
778 #Once registered to the LDAP, all iotlab sites are
781 #Mark account as disabled in ldap
782 return self.iotlab_api.DeletePerson(sfa_record)
784 elif sfa_record_type == 'slice':
785 if self.iotlab_api.GetSlices(slice_filter=hrn,
786 slice_filter_type='slice_hrn'):
787 ret = self.iotlab_api.DeleteSlice(sfa_record)