1 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
2 from sfa.util.sfalogging import logger
3 from sfa.storage.alchemy import dbsession
4 from sfa.storage.model import RegRecord
6 from sfa.managers.driver import Driver
7 from sfa.rspecs.version_manager import VersionManager
8 from sfa.rspecs.rspec import RSpec
10 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
13 from sfa.iotlab.iotlabpostgres import IotlabDB
15 from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname
16 from sfa.iotlab.iotlabslices import IotlabSlices
19 from sfa.iotlab.iotlabapi import IotlabTestbedAPI
22 class IotlabDriver(Driver):
23 """ Iotlab Driver class inherited from Driver generic class.
25 Contains methods compliant with the SFA standard and the testbed
26 infrastructure (calls to LDAP and OAR).
28 .. seealso::: Driver class
31 def __init__(self, config):
34 Sets the iotlab SFA config parameters,
35 instanciates the testbed api and the iotlab database.
37 :param config: iotlab SFA configuration object
38 :type config: Config object
41 Driver.__init__ (self, config)
44 # self.db = IotlabDB(config, debug = False)
45 self.iotlab_api = IotlabTestbedAPI(config)
48 def augment_records_with_testbed_info (self, record_list ):
51 Adds specific testbed info to the records.
53 :param record_list: list of sfa dictionaries records
54 :type record_list: list
55 :returns: list of records with extended information in each record
59 return self.fill_record_info (record_list)
61 def fill_record_info(self, record_list):
63 For each SFA record, fill in the iotlab specific and SFA specific
66 :param record_list: list of sfa dictionaries records
67 :type record_list: list
68 :returns: list of records with extended information in each record
71 .. warnings:: Should not be modifying record_list directly because modi
72 fication are kept outside the method's scope. Howerver, there is no
73 other way to do it given the way it's called in registry manager.
77 logger.debug("IOTLABDRIVER \tfill_record_info records %s "
79 if not isinstance(record_list, list):
80 record_list = [record_list]
83 for record in record_list:
84 #If the record is a SFA slice record, then add information
85 #about the user of this slice. This kind of
86 #information is in the Iotlab's DB.
87 if str(record['type']) == 'slice':
88 if 'reg_researchers' in record and \
89 isinstance(record['reg_researchers'], list) :
90 record['reg_researchers'] = \
91 record['reg_researchers'][0].__dict__
92 record.update({'PI':[record['reg_researchers']['hrn']],
93 'researcher': [record['reg_researchers']['hrn']],
97 'person_ids': [record['reg_researchers']
99 # For client_helper.py compatibility
101 # For client_helper.py compatibility
103 # For client_helper.py compatibility
106 #Get iotlab slice record and oar job id if any.
107 recslice_list = self.iotlab_api.GetSlices(
108 slice_filter=str(record['hrn']),
109 slice_filter_type='slice_hrn')
111 logger.debug("IOTLABDRIVER \tfill_record_info \
112 TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\
113 %s " % (record['hrn'], record['oar_job_id']))
114 del record['reg_researchers']
116 for rec in recslice_list:
117 logger.debug("IOTLABDRIVER\r\n \t \
118 fill_record_info oar_job_id %s "
119 % (rec['oar_job_id']))
121 record['node_ids'] = [self.iotlab_api.root_auth +
122 hostname for hostname in
127 logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \
128 recslice_list %s \r\n \t RECORD %s \r\n \
129 \r\n" % (recslice_list, record))
131 if str(record['type']) == 'user':
132 #The record is a SFA user record.
133 #Get the information about his slice from Iotlab's DB
134 #and add it to the user record.
135 recslice_list = self.iotlab_api.GetSlices(
136 slice_filter=record['record_id'],
137 slice_filter_type='record_id_user')
139 logger.debug("IOTLABDRIVER.PY \t fill_record_info \
140 TYPE USER recslice_list %s \r\n \t RECORD %s \r\n"
141 % (recslice_list, record))
142 #Append slice record in records list,
143 #therefore fetches user and slice info again(one more loop)
144 #Will update PIs and researcher for the slice
146 recuser = recslice_list[0]['reg_researchers']
147 logger.debug( "IOTLABDRIVER.PY \t fill_record_info USER \
148 recuser %s \r\n \r\n" % (recuser))
150 recslice = recslice_list[0]
151 recslice.update({'PI':[recuser['hrn']],
152 'researcher': [recuser['hrn']],
153 'name':record['hrn'],
156 'person_ids':[recuser['record_id']]})
158 for rec in recslice_list:
159 recslice['oar_job_id'].append(rec['oar_job_id'])
163 recslice.update({'type':'slice', \
164 'hrn':recslice_list[0]['hrn']})
167 #GetPersons takes [] as filters
168 user_iotlab = self.iotlab_api.GetPersons([record])
171 record.update(user_iotlab[0])
172 #For client_helper.py compatibility
173 record.update( { 'geni_urn':'',
176 record_list.append(recslice)
178 logger.debug("IOTLABDRIVER.PY \t \
179 fill_record_info ADDING SLICE\
180 INFO TO USER records %s" % (record_list))
183 except TypeError, error:
184 logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"\
190 def sliver_status(self, slice_urn, slice_hrn):
193 Receive a status request for slice named urn/hrn
194 urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
195 shall return a structure as described in
196 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
197 NT : not sure if we should implement this or not, but used by sface.
199 :param slice_urn: slice urn
200 :type slice_urn: string
201 :param slice_hrn: slice hrn
202 :type slice_hrn: string
207 #First get the slice with the slice hrn
208 slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn,
209 slice_filter_type='slice_hrn')
211 if len(slice_list) == 0:
212 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
214 #Used for fetching the user info witch comes along the slice info
215 one_slice = slice_list[0]
218 #Make a list of all the nodes hostnames in use for this slice
219 slice_nodes_list = []
220 #for single_slice in slice_list:
221 #for node in single_slice['node_ids']:
222 #slice_nodes_list.append(node['hostname'])
223 #for node in one_slice:
224 #slice_nodes_list.append(node['hostname'])
225 slice_nodes_list = one_slice['node_ids']
226 #Get all the corresponding nodes details
227 nodes_all = self.iotlab_api.GetNodes({'hostname':slice_nodes_list},
228 ['node_id', 'hostname','site','boot_state'])
229 nodeall_byhostname = dict([(one_node['hostname'], one_node) \
230 for one_node in nodes_all])
234 for single_slice in slice_list:
237 top_level_status = 'empty'
240 ['geni_urn','geni_error', 'iotlab_login','geni_status',
241 'geni_resources'], None)
243 # ['geni_urn','geni_error', 'pl_login','geni_status',
244 # 'geni_resources'], None)
245 # result['pl_login'] = one_slice['reg_researchers'][0].hrn
246 result['iotlab_login'] = one_slice['user']
247 logger.debug("Slabdriver - sliver_status Sliver status \
248 urn %s hrn %s single_slice %s \r\n " \
249 %(slice_urn, slice_hrn, single_slice))
251 if 'node_ids' not in single_slice:
253 result['geni_status'] = top_level_status
254 result['geni_resources'] = []
257 top_level_status = 'ready'
259 #A job is running on Iotlab for this slice
260 # report about the local nodes that are in the slice only
262 result['geni_urn'] = slice_urn
265 for node_hostname in single_slice['node_ids']:
267 res['iotlab_hostname'] = node_hostname
268 res['iotlab_boot_state'] = \
269 nodeall_byhostname[node_hostname]['boot_state']
271 #res['pl_hostname'] = node['hostname']
272 #res['pl_boot_state'] = \
273 #nodeall_byhostname[node['hostname']]['boot_state']
274 #res['pl_last_contact'] = strftime(self.time_format, \
275 #gmtime(float(timestamp)))
276 sliver_id = Xrn(slice_urn, type='slice', \
277 id=nodeall_byhostname[node_hostname]['node_id']).urn
279 res['geni_urn'] = sliver_id
280 #node_name = node['hostname']
281 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
283 res['geni_status'] = 'ready'
285 res['geni_status'] = 'failed'
286 top_level_status = 'failed'
288 res['geni_error'] = ''
290 resources.append(res)
292 result['geni_status'] = top_level_status
293 result['geni_resources'] = resources
294 logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "\
299 def get_user_record(hrn):
302 Returns the user record based on the hrn from the SFA DB .
304 :param hrn: user's hrn
306 :return : user record from SFA database
310 return dbsession.query(RegRecord).filter_by(hrn = hrn).first()
313 def testbed_name (self):
316 Returns testbed's name.
323 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
324 def aggregate_version (self):
327 Returns the testbed's supported rspec advertisement and
333 version_manager = VersionManager()
334 ad_rspec_versions = []
335 request_rspec_versions = []
336 for rspec_version in version_manager.versions:
337 if rspec_version.content_type in ['*', 'ad']:
338 ad_rspec_versions.append(rspec_version.to_dict())
339 if rspec_version.content_type in ['*', 'request']:
340 request_rspec_versions.append(rspec_version.to_dict())
342 'testbed':self.testbed_name(),
343 'geni_request_rspec_versions': request_rspec_versions,
344 'geni_ad_rspec_versions': ad_rspec_versions,
349 def _get_requested_leases_list(self, rspec):
351 Process leases in rspec depending on the rspec version (format)
352 type. Find the lease requests in the rspec and creates
353 a lease request list with the mandatory information ( nodes,
354 start time and duration) of the valid leases (duration above or
355 equal to the iotlab experiment minimum duration).
357 :param rspec: rspec request received.
359 :returns: list of lease requests found in the rspec
362 requested_lease_list = []
363 for lease in rspec.version.get_leases():
364 single_requested_lease = {}
365 logger.debug("IOTLABDRIVER.PY \t \
366 _get_requested_leases_list lease %s " % (lease))
368 if not lease.get('lease_id'):
369 if get_authority(lease['component_id']) == \
370 self.iotlab_api.root_auth:
371 single_requested_lease['hostname'] = \
372 iotlab_xrn_to_hostname(\
373 lease.get('component_id').strip())
374 single_requested_lease['start_time'] = \
375 lease.get('start_time')
376 single_requested_lease['duration'] = lease.get('duration')
377 #Check the experiment's duration is valid before adding
378 #the lease to the requested leases list
379 duration_in_seconds = \
380 int(single_requested_lease['duration'])
381 if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec() :
382 requested_lease_list.append(single_requested_lease)
384 return requested_lease_list
387 def _group_leases_by_start_time(requested_lease_list):
389 Create dict of leases by start_time, regrouping nodes reserved
390 at the same time, for the same amount of time so as to
391 define one job on OAR.
393 :param requested_lease_list: list of leases
394 :type requested_lease_list: list
395 :returns: Dictionary with key = start time, value = list of leases
396 with the same start time.
401 requested_job_dict = {}
402 for lease in requested_lease_list:
404 #In case it is an asap experiment start_time is empty
405 if lease['start_time'] == '':
406 lease['start_time'] = '0'
408 if lease['start_time'] not in requested_job_dict:
409 if isinstance(lease['hostname'], str):
410 lease['hostname'] = [lease['hostname']]
413 requested_job_dict[lease['start_time']] = lease
416 job_lease = requested_job_dict[lease['start_time']]
417 if lease['duration'] == job_lease['duration'] :
418 job_lease['hostname'].append(lease['hostname'])
420 return requested_job_dict
422 def _process_requested_jobs(self, rspec):
424 Turns the requested leases and information into a dictionary
425 of requested jobs, grouped by starting time.
427 :param rspec: RSpec received
432 requested_lease_list = self._get_requested_leases_list(rspec)
433 logger.debug("IOTLABDRIVER _process_requested_jobs requested_lease_list \
434 %s"%(requested_lease_list))
435 job_dict = self._group_leases_by_start_time(requested_lease_list)
436 logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\
441 def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \
443 """Answer to CreateSliver.
445 Creates the leases and slivers for the users from the information
446 found in the rspec string.
447 Launch experiment on OAR if the requested leases is valid. Delete
448 no longer requested leases.
451 :param creds: user's credentials
453 :param users: user record list
458 :returns: a valid Rspec for the slice which has just been
464 aggregate = IotlabAggregate(self)
466 slices = IotlabSlices(self)
467 peer = slices.get_peer(slice_hrn)
468 sfa_peer = slices.get_sfa_peer(slice_hrn)
471 if not isinstance(creds, list):
475 slice_record = users[0].get('slice_record', {})
476 logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\
477 creds %s \r\n \r\n users %s" \
479 slice_record['user'] = {'keys':users[0]['keys'], \
480 'email':users[0]['email'], \
481 'hrn':slice_record['reg-researchers'][0]}
483 rspec = RSpec(rspec_string)
484 logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \
485 %s slice_record %s users %s" \
486 %(rspec.version,slice_record, users))
489 # ensure site record exists?
490 # ensure slice record exists
491 #Removed options to verify_slice SA 14/08/12
492 sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \
495 # ensure person records exists
496 #verify_persons returns added persons but since the return value
498 slices.verify_persons(slice_hrn, sfa_slice, users, peer, \
499 sfa_peer, options=options)
500 #requested_attributes returned by rspec.version.get_slice_attributes()
501 #unused, removed SA 13/08/12
502 #rspec.version.get_slice_attributes()
504 logger.debug("IOTLABDRIVER.PY create_sliver slice %s " %(sfa_slice))
506 # add/remove slice from nodes
508 #requested_slivers = [node.get('component_id') \
509 #for node in rspec.version.get_nodes_with_slivers()\
510 #if node.get('authority_id') is self.iotlab_api.root_auth]
511 #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
512 #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
513 #requested_slivers %s listnodes %s" \
514 #%(requested_slivers,l))
515 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
516 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
519 requested_job_dict = self._process_requested_jobs(rspec)
522 logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\
523 %(requested_job_dict))
524 #verify_slice_leases returns the leases , but the return value is unused
525 #here. Removed SA 13/08/12
526 slices.verify_slice_leases(sfa_slice, \
527 requested_job_dict, peer)
529 return aggregate.get_rspec(slice_xrn=slice_urn, \
530 login=sfa_slice['login'], version=rspec.version)
533 def delete_sliver (self, slice_urn, slice_hrn, creds, options):
535 Deletes the lease associated with the slice hrn and the credentials
536 if the slice belongs to iotlab. Answer to DeleteSliver.
538 :returns: 1 if the slice to delete was not found on iotlab,
539 True if the deletion was successful, False otherwise otherwise.
541 .. note:: Should really be named delete_leases because iotlab does
542 not have any slivers, but only deals with leases. However,
543 SFA api only have delete_sliver define so far. SA 13.05/2013
546 sfa_slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \
547 slice_filter_type = 'slice_hrn')
549 if not sfa_slice_list:
552 #Delete all leases in the slice
553 for sfa_slice in sfa_slice_list:
556 logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" %(sfa_slice))
557 slices = IotlabSlices(self)
558 # determine if this is a peer slice
560 peer = slices.get_peer(slice_hrn)
562 logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
563 \r\n \t sfa_slice %s " %(peer, sfa_slice))
566 self.iotlab_api.DeleteSliceFromNodes(sfa_slice)
572 def list_resources (self, slice_urn, slice_hrn, creds, options):
575 List resources from the iotlab aggregate and returns a Rspec
576 advertisement with resources found when slice_urn and slice_hrn are
577 None (in case of resource discovery).
578 If a slice hrn and urn are provided, list experiment's slice
579 nodes in a rspec format. Answer to ListResources.
582 :param options: options used when listing resources (list_leases, info,
584 :returns: rspec string in xml
589 #cached_requested = options.get('cached', True)
591 version_manager = VersionManager()
592 # get the rspec's return format from options
594 version_manager.get_version(options.get('geni_rspec_version'))
595 version_string = "rspec_%s" % (rspec_version)
597 #panos adding the info option to the caching key (can be improved)
598 if options.get('info'):
599 version_string = version_string + "_" + \
600 options.get('info', 'default')
602 # Adding the list_leases option to the caching key
603 if options.get('list_leases'):
604 version_string = version_string + "_" + \
605 options.get('list_leases', 'default')
607 # Adding geni_available to caching key
608 if options.get('geni_available'):
609 version_string = version_string + "_" + \
610 str(options.get('geni_available'))
612 # look in cache first
613 #if cached_requested and self.cache and not slice_hrn:
614 #rspec = self.cache.get(version_string)
616 #logger.debug("IotlabDriver.ListResources: \
617 #returning cached advertisement")
620 #panos: passing user-defined options
621 aggregate = IotlabAggregate(self)
623 rspec = aggregate.get_rspec(slice_xrn=slice_urn, \
624 version=rspec_version, options=options)
627 #if self.cache and not slice_hrn:
628 #logger.debug("Iotlab.ListResources: stores advertisement in cache")
629 #self.cache.add(version_string, rspec)
634 def list_slices (self, creds, options):
635 """Answer to ListSlices.
637 List slices belonging to iotlab, returns slice urns list.
638 No caching used. Options unused but are defined in the SFA method
641 :returns: slice urns list
645 # look in cache first
647 #slices = self.cache.get('slices')
649 #logger.debug("PlDriver.list_slices returns from cache")
654 slices = self.iotlab_api.GetSlices()
655 logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices))
656 slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
658 slice_urns = [hrn_to_urn(slice_hrn, 'slice') \
659 for slice_hrn in slice_hrns]
663 #logger.debug ("IotlabDriver.list_slices stores value in cache")
664 #self.cache.add('slices', slice_urns)
669 def register (self, sfa_record, hrn, pub_key):
671 Adding new user, slice, node or site should not be handled
674 ..warnings:: should not be used. Different components are in charge of
675 doing this task. Adding nodes = OAR
676 Adding users = LDAP Iotlab
677 Adding slice = Import from LDAP users
680 :param sfa_record: record provided by the client of the
682 :type sfa_record: dict
688 def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
690 No site or node record update allowed in Iotlab.
691 The only modifications authorized here are key deletion/addition
692 on an existing user and password change.
693 On an existing user, CAN NOT BE MODIFIED:
694 'first_name', 'last_name', 'email'
695 DOES NOT EXIST IN SENSLAB:
696 'phone', 'url', 'bio','title', 'accepted_aup',
697 A slice is bound to its user, so modifying the user's ssh key should
698 modify the slice's GID after an import procedure.
700 :param old_sfa_record: what is in the db for this hrn
701 :param new_sfa_record: what was passed to the Update call
703 .. seealso::: update in driver.py.
707 pointer = old_sfa_record['pointer']
708 old_sfa_record_type = old_sfa_record['type']
710 # new_key implemented for users only
711 if new_key and old_sfa_record_type not in [ 'user' ]:
712 raise UnknownSfaType(old_sfa_record_type)
715 if old_sfa_record_type == "user":
717 all_fields = new_sfa_record
718 for key in all_fields.keys():
719 if key in ['key', 'password']:
720 update_fields[key] = all_fields[key]
724 # must check this key against the previous one if it exists
725 persons = self.iotlab_api.GetPersons([old_sfa_record])
727 keys = [person['pkey']]
728 #Get all the person's keys
729 keys_dict = self.iotlab_api.GetKeys(keys)
731 # Delete all stale keys, meaning the user has only one key
733 #TODO: do we really want to delete all the other keys?
734 #Is this a problem with the GID generation to have multiple
740 #remove all the other keys
741 for key in keys_dict:
742 self.iotlab_api.DeleteKey(person, key)
743 self.iotlab_api.AddPersonKey(person, \
744 {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} )
745 #self.iotlab_api.AddPersonKey(person, {'key_type': 'ssh', \
750 def remove (self, sfa_record):
753 Removes users only. Mark the user as disabled in
754 LDAP. The user and his slice are then deleted from the
755 db by running an import on the registry.
759 :param sfa_record: record is the existing sfa record in the db
760 :type sfa_record: dict
762 ..warning::As fas as the slice is concerned, here only the leases are
763 removed from the slice. The slice is record itself is not removed
766 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
768 TODO: return boolean for the slice part
770 sfa_record_type = sfa_record['type']
771 hrn = sfa_record['hrn']
772 if sfa_record_type == 'user':
774 #get user from iotlab ldap
775 person = self.iotlab_api.GetPersons(sfa_record)
776 #No registering at a given site in Iotlab.
777 #Once registered to the LDAP, all iotlab sites are
780 #Mark account as disabled in ldap
781 return self.iotlab_api.DeletePerson(sfa_record)
783 elif sfa_record_type == 'slice':
784 if self.iotlab_api.GetSlices(slice_filter = hrn, \
785 slice_filter_type = 'slice_hrn'):
786 ret = self.iotlab_api.DeleteSlice(sfa_record)