1 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
2 from sfa.util.sfalogging import logger
3 from sfa.storage.alchemy import dbsession
4 from sfa.storage.model import RegRecord
6 from sfa.managers.driver import Driver
7 from sfa.rspecs.version_manager import VersionManager
8 from sfa.rspecs.rspec import RSpec
10 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
13 from sfa.iotlab.iotlabpostgres import IotlabDB
15 from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname
16 from sfa.iotlab.iotlabslices import IotlabSlices
19 from sfa.iotlab.iotlabapi import IotlabTestbedAPI
22 class IotlabDriver(Driver):
23 """ Iotlab Driver class inherited from Driver generic class.
25 Contains methods compliant with the SFA standard and the testbed
26 infrastructure (calls to LDAP and OAR).
28 .. seealso::: Driver class
31 def __init__(self, config):
34 Sets the iotlab SFA config parameters ,
35 instanciates the testbed api and the iotlab database.
37 :param config: iotlab SFA configuration object
38 :type config: Config object
40 Driver.__init__ (self, config)
43 self.db = IotlabDB(config, debug = False)
44 self.iotlab_api = IotlabTestbedAPI(config)
47 def augment_records_with_testbed_info (self, record_list ):
50 Adds specific testbed info to the records.
52 :param record_list: list of sfa dictionaries records
53 :type record_list: list
54 :returns: list of records with extended information in each record
57 return self.fill_record_info (record_list)
59 def fill_record_info(self, record_list):
61 For each SFA record, fill in the iotlab specific and SFA specific
64 :param record_list: list of sfa dictionaries records
65 :type record_list: list
66 :returns: list of records with extended information in each record
69 .. warnings:: Should not be modifying record_list directly because modi
70 fication are kept outside the method's scope. Howerver, there is no
71 other way to do it given the way it's called in registry manager.
74 logger.debug("IOTLABDRIVER \tfill_record_info records %s " %(record_list))
75 if not isinstance(record_list, list):
76 record_list = [record_list]
80 for record in record_list:
81 #If the record is a SFA slice record, then add information
82 #about the user of this slice. This kind of
83 #information is in the Iotlab's DB.
84 if str(record['type']) == 'slice':
85 if 'reg_researchers' in record and \
86 isinstance(record['reg_researchers'], list) :
87 record['reg_researchers'] = \
88 record['reg_researchers'][0].__dict__
89 record.update({'PI':[record['reg_researchers']['hrn']],
90 'researcher': [record['reg_researchers']['hrn']],
94 'person_ids':[record['reg_researchers']['record_id']],
95 'geni_urn':'', #For client_helper.py compatibility
96 'keys':'', #For client_helper.py compatibility
97 'key_ids':''}) #For client_helper.py compatibility
100 #Get iotlab slice record and oar job id if any.
101 recslice_list = self.iotlab_api.GetSlices(slice_filter = \
103 slice_filter_type = 'slice_hrn')
106 logger.debug("IOTLABDRIVER \tfill_record_info \
107 TYPE SLICE RECUSER record['hrn'] %s ecord['oar_job_id']\
108 %s " %(record['hrn'], record['oar_job_id']))
109 del record['reg_researchers']
111 for rec in recslice_list:
112 logger.debug("IOTLABDRIVER\r\n \t \
113 fill_record_info oar_job_id %s " \
114 %(rec['oar_job_id']))
116 record['node_ids'] = [ self.iotlab_api.root_auth + \
117 hostname for hostname in rec['node_ids']]
122 logger.debug( "IOTLABDRIVER.PY \t fill_record_info SLICE \
123 recslice_list %s \r\n \t RECORD %s \r\n \
124 \r\n" %(recslice_list, record))
126 if str(record['type']) == 'user':
127 #The record is a SFA user record.
128 #Get the information about his slice from Iotlab's DB
129 #and add it to the user record.
130 recslice_list = self.iotlab_api.GetSlices(\
131 slice_filter = record['record_id'],\
132 slice_filter_type = 'record_id_user')
134 logger.debug( "IOTLABDRIVER.PY \t fill_record_info TYPE USER \
135 recslice_list %s \r\n \t RECORD %s \r\n" \
136 %(recslice_list , record))
137 #Append slice record in records list,
138 #therefore fetches user and slice info again(one more loop)
139 #Will update PIs and researcher for the slice
141 recuser = recslice_list[0]['reg_researchers']
142 logger.debug( "IOTLABDRIVER.PY \t fill_record_info USER \
143 recuser %s \r\n \r\n" %(recuser))
145 recslice = recslice_list[0]
146 recslice.update({'PI':[recuser['hrn']],
147 'researcher': [recuser['hrn']],
148 'name':record['hrn'],
151 'person_ids':[recuser['record_id']]})
153 for rec in recslice_list:
154 recslice['oar_job_id'].append(rec['oar_job_id'])
158 recslice.update({'type':'slice', \
159 'hrn':recslice_list[0]['hrn']})
162 #GetPersons takes [] as filters
163 user_iotlab = self.iotlab_api.GetPersons([record])
166 record.update(user_iotlab[0])
167 #For client_helper.py compatibility
168 record.update( { 'geni_urn':'',
171 record_list.append(recslice)
173 logger.debug("IOTLABDRIVER.PY \tfill_record_info ADDING SLICE\
174 INFO TO USER records %s" %(record_list))
177 except TypeError, error:
178 logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"\
184 def sliver_status(self, slice_urn, slice_hrn):
186 Receive a status request for slice named urn/hrn
187 urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
188 shall return a structure as described in
189 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
190 NT : not sure if we should implement this or not, but used by sface.
192 :param slice_urn: slice urn
193 :type slice_urn: string
194 :param slice_hrn: slice hrn
195 :type slice_hrn: string
200 #First get the slice with the slice hrn
201 slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \
202 slice_filter_type = 'slice_hrn')
204 if len(slice_list) is 0:
205 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
207 #Used for fetching the user info witch comes along the slice info
208 one_slice = slice_list[0]
211 #Make a list of all the nodes hostnames in use for this slice
212 slice_nodes_list = []
213 #for single_slice in slice_list:
214 #for node in single_slice['node_ids']:
215 #slice_nodes_list.append(node['hostname'])
216 #for node in one_slice:
217 #slice_nodes_list.append(node['hostname'])
218 slice_nodes_list = one_slice['node_ids']
219 #Get all the corresponding nodes details
220 nodes_all = self.iotlab_api.GetNodes({'hostname':slice_nodes_list},
221 ['node_id', 'hostname','site','boot_state'])
222 nodeall_byhostname = dict([(one_node['hostname'], one_node) \
223 for one_node in nodes_all])
227 for single_slice in slice_list:
230 top_level_status = 'empty'
233 ['geni_urn','geni_error', 'pl_login','geni_status','geni_resources'], None)
234 result['pl_login'] = one_slice['reg_researchers'][0].hrn
235 logger.debug("Slabdriver - sliver_status Sliver status \
236 urn %s hrn %s single_slice %s \r\n " \
237 %(slice_urn, slice_hrn, single_slice))
239 if 'node_ids' not in single_slice:
241 result['geni_status'] = top_level_status
242 result['geni_resources'] = []
245 top_level_status = 'ready'
247 #A job is running on Iotlab for this slice
248 # report about the local nodes that are in the slice only
250 result['geni_urn'] = slice_urn
253 for node_hostname in single_slice['node_ids']:
255 res['iotlab_hostname'] = node_hostname
256 res['iotlab_boot_state'] = nodeall_byhostname[node_hostname]['boot_state']
258 #res['pl_hostname'] = node['hostname']
259 #res['pl_boot_state'] = \
260 #nodeall_byhostname[node['hostname']]['boot_state']
261 #res['pl_last_contact'] = strftime(self.time_format, \
262 #gmtime(float(timestamp)))
263 sliver_id = Xrn(slice_urn, type='slice', \
264 id=nodeall_byhostname[node_hostname]['node_id'], \
265 authority=self.hrn).urn
267 res['geni_urn'] = sliver_id
268 #node_name = node['hostname']
269 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
271 res['geni_status'] = 'ready'
273 res['geni_status'] = 'failed'
274 top_level_status = 'failed'
276 res['geni_error'] = ''
278 resources.append(res)
280 result['geni_status'] = top_level_status
281 result['geni_resources'] = resources
282 logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "\
287 def get_user_record(hrn):
289 Returns the user record based on the hrn from the SFA DB .
291 :param hrn: user's hrn
293 :return : user record from SFA database
297 return dbsession.query(RegRecord).filter_by(hrn = hrn).first()
300 def testbed_name (self):
302 Returns testbed's name.
308 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
309 def aggregate_version (self):
312 Returns the testbed's supported rspec advertisement and
317 version_manager = VersionManager()
318 ad_rspec_versions = []
319 request_rspec_versions = []
320 for rspec_version in version_manager.versions:
321 if rspec_version.content_type in ['*', 'ad']:
322 ad_rspec_versions.append(rspec_version.to_dict())
323 if rspec_version.content_type in ['*', 'request']:
324 request_rspec_versions.append(rspec_version.to_dict())
326 'testbed':self.testbed_name(),
327 'geni_request_rspec_versions': request_rspec_versions,
328 'geni_ad_rspec_versions': ad_rspec_versions,
333 def _get_requested_leases_list(self, rspec):
335 Process leases in rspec depending on the rspec version (format)
336 type. Find the lease requests in the rspec and creates
337 a lease request list with the mandatory information ( nodes,
338 start time and duration) of the valid leases (duration above or equal
339 to the iotlab experiment minimum duration).
341 :param rspec: rspec request received.
343 :returns: list of lease requests found in the rspec
346 requested_lease_list = []
347 for lease in rspec.version.get_leases():
348 single_requested_lease = {}
349 logger.debug("IOTLABDRIVER.PY \t_get_requested_leases_list lease %s " %(lease))
351 if not lease.get('lease_id'):
352 if get_authority(lease['component_id']) == \
353 self.iotlab_api.root_auth:
354 single_requested_lease['hostname'] = \
355 iotlab_xrn_to_hostname(\
356 lease.get('component_id').strip())
357 single_requested_lease['start_time'] = \
358 lease.get('start_time')
359 single_requested_lease['duration'] = lease.get('duration')
360 #Check the experiment's duration is valid before adding
361 #the lease to the requested leases list
362 duration_in_seconds = \
363 int(single_requested_lease['duration'])
364 if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec() :
365 requested_lease_list.append(single_requested_lease)
367 return requested_lease_list
370 def _group_leases_by_start_time(requested_lease_list):
372 Create dict of leases by start_time, regrouping nodes reserved
373 at the same time, for the same amount of time so as to
374 define one job on OAR.
376 :param requested_lease_list: list of leases
377 :type requested_lease_list: list
378 :returns: Dictionary with key = start time, value = list of leases
379 with the same start time.
383 requested_job_dict = {}
384 for lease in requested_lease_list:
386 #In case it is an asap experiment start_time is empty
387 if lease['start_time'] == '':
388 lease['start_time'] = '0'
390 if lease['start_time'] not in requested_job_dict:
391 if isinstance(lease['hostname'], str):
392 lease['hostname'] = [lease['hostname']]
395 requested_job_dict[lease['start_time']] = lease
398 job_lease = requested_job_dict[lease['start_time']]
399 if lease['duration'] == job_lease['duration'] :
400 job_lease['hostname'].append(lease['hostname'])
402 return requested_job_dict
404 def _process_requested_jobs(self, rspec):
406 Turns the requested leases and information into a dictionary
407 of requested jobs, grouped by starting time.
409 :param rspec: RSpec received
413 requested_lease_list = self._get_requested_leases_list(rspec)
414 logger.debug("IOTLABDRIVER _process_requested_jobs requested_lease_list \
415 %s"%(requested_lease_list))
416 job_dict = self._group_leases_by_start_time(requested_lease_list)
417 logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\
422 def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \
425 Answer to CreateSliver.
426 Creates the leases and slivers for the users from the information
427 found in the rspec string.
428 Launch experiment on OAR if the requested leases is valid. Delete
429 no longer requested leases.
432 :param creds: user's credentials
434 :param users: user record list
439 :returns: a valid Rspec for the slice which has just been
445 aggregate = IotlabAggregate(self)
447 slices = IotlabSlices(self)
448 peer = slices.get_peer(slice_hrn)
449 sfa_peer = slices.get_sfa_peer(slice_hrn)
452 if not isinstance(creds, list):
456 slice_record = users[0].get('slice_record', {})
457 logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\
458 creds %s \r\n \r\n users %s" \
460 slice_record['user'] = {'keys':users[0]['keys'], \
461 'email':users[0]['email'], \
462 'hrn':slice_record['reg-researchers'][0]}
464 rspec = RSpec(rspec_string)
465 logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \
466 %s slice_record %s users %s" \
467 %(rspec.version,slice_record, users))
470 # ensure site record exists?
471 # ensure slice record exists
472 #Removed options to verify_slice SA 14/08/12
473 sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \
476 # ensure person records exists
477 #verify_persons returns added persons but since the return value
479 slices.verify_persons(slice_hrn, sfa_slice, users, peer, \
480 sfa_peer, options=options)
481 #requested_attributes returned by rspec.version.get_slice_attributes()
482 #unused, removed SA 13/08/12
483 #rspec.version.get_slice_attributes()
485 logger.debug("IOTLABDRIVER.PY create_sliver slice %s " %(sfa_slice))
487 # add/remove slice from nodes
489 #requested_slivers = [node.get('component_id') \
490 #for node in rspec.version.get_nodes_with_slivers()\
491 #if node.get('authority_id') is self.iotlab_api.root_auth]
492 #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
493 #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
494 #requested_slivers %s listnodes %s" \
495 #%(requested_slivers,l))
496 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
497 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
500 requested_job_dict = self._process_requested_jobs(rspec)
503 logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\
504 %(requested_job_dict))
505 #verify_slice_leases returns the leases , but the return value is unused
506 #here. Removed SA 13/08/12
507 slices.verify_slice_leases(sfa_slice, \
508 requested_job_dict, peer)
510 return aggregate.get_rspec(slice_xrn=slice_urn, \
511 login=sfa_slice['login'], version=rspec.version)
514 def delete_sliver (self, slice_urn, slice_hrn, creds, options):
516 Deletes the lease associated with the slice hrn and the credentials
517 if the slice belongs to iotlab. Answer to DeleteSliver.
519 :returns: 1 if the slice to delete was not found on iotlab,
520 True if the deletion was successful, False otherwise otherwise.
522 .. note:: Should really be named delete_leases because iotlab does
523 not have any slivers, but only deals with leases. However, SFA api only
524 have delete_sliver define so far. SA 13.05/2013
527 sfa_slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \
528 slice_filter_type = 'slice_hrn')
530 if not sfa_slice_list:
533 #Delete all leases in the slice
534 for sfa_slice in sfa_slice_list:
537 logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" %(sfa_slice))
538 slices = IotlabSlices(self)
539 # determine if this is a peer slice
541 peer = slices.get_peer(slice_hrn)
543 logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
544 \r\n \t sfa_slice %s " %(peer, sfa_slice))
547 self.iotlab_api.DeleteSliceFromNodes(sfa_slice)
553 def list_resources (self, slice_urn, slice_hrn, creds, options):
555 List resources from the iotlab aggregate and returns a Rspec
556 advertisement with resources found when slice_urn and slice_hrn are None
557 (in case of resource discovery).
558 If a slice hrn and urn are provided, list experiment's slice
559 nodes in a rspec format. Answer to ListResources.
561 :param options: options used when listing resources (list_leases, info,
563 :returns: rspec string in xml
567 #cached_requested = options.get('cached', True)
569 version_manager = VersionManager()
570 # get the rspec's return format from options
572 version_manager.get_version(options.get('geni_rspec_version'))
573 version_string = "rspec_%s" % (rspec_version)
575 #panos adding the info option to the caching key (can be improved)
576 if options.get('info'):
577 version_string = version_string + "_" + \
578 options.get('info', 'default')
580 # Adding the list_leases option to the caching key
581 if options.get('list_leases'):
582 version_string = version_string + "_" + \
583 options.get('list_leases', 'default')
585 # Adding geni_available to caching key
586 if options.get('geni_available'):
587 version_string = version_string + "_" + \
588 str(options.get('geni_available'))
590 # look in cache first
591 #if cached_requested and self.cache and not slice_hrn:
592 #rspec = self.cache.get(version_string)
594 #logger.debug("IotlabDriver.ListResources: \
595 #returning cached advertisement")
598 #panos: passing user-defined options
599 aggregate = IotlabAggregate(self)
601 rspec = aggregate.get_rspec(slice_xrn=slice_urn, \
602 version=rspec_version, options=options)
605 #if self.cache and not slice_hrn:
606 #logger.debug("Iotlab.ListResources: stores advertisement in cache")
607 #self.cache.add(version_string, rspec)
612 def list_slices (self, creds, options):
614 Answer to ListSlices.
615 List slices belonging to iotlab, returns slice urns list.
616 No caching used. Options unused but are defined in the SFA method
619 :returns: slice urns list
623 # look in cache first
625 #slices = self.cache.get('slices')
627 #logger.debug("PlDriver.list_slices returns from cache")
632 slices = self.iotlab_api.GetSlices()
633 logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices))
634 slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
636 slice_urns = [hrn_to_urn(slice_hrn, 'slice') \
637 for slice_hrn in slice_hrns]
641 #logger.debug ("IotlabDriver.list_slices stores value in cache")
642 #self.cache.add('slices', slice_urns)
647 def register (self, sfa_record, hrn, pub_key):
649 Adding new user, slice, node or site should not be handled
652 ..warnings:: should not be used. Different components are in charge of
653 doing this task. Adding nodes = OAR
654 Adding users = LDAP Iotlab
655 Adding slice = Import from LDAP users
658 :param sfa_record: record provided by the client of the
660 :type sfa_record: dict
665 def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
666 """No site or node record update allowed in Iotlab.
667 The only modifications authorized here are key deletion/addition
668 on an existing user and password change.
669 On an existing user, CAN NOT BE MODIFIED:
670 'first_name', 'last_name', 'email'
671 DOES NOT EXIST IN SENSLAB:
672 'phone', 'url', 'bio','title', 'accepted_aup',
673 A slice is bound to its user, so modifying the user's ssh key should
674 modify the slice's GID after an import procedure.
676 :param old_sfa_record: what is in the db for this hrn
677 :param new_sfa_record: what was passed to the Update call
679 .. seealso::: update in driver.py.
682 pointer = old_sfa_record['pointer']
683 old_sfa_record_type = old_sfa_record['type']
685 # new_key implemented for users only
686 if new_key and old_sfa_record_type not in [ 'user' ]:
687 raise UnknownSfaType(old_sfa_record_type)
690 if old_sfa_record_type == "user":
692 all_fields = new_sfa_record
693 for key in all_fields.keys():
694 if key in ['key', 'password']:
695 update_fields[key] = all_fields[key]
699 # must check this key against the previous one if it exists
700 persons = self.iotlab_api.GetPersons([old_sfa_record])
702 keys = [person['pkey']]
703 #Get all the person's keys
704 keys_dict = self.iotlab_api.GetKeys(keys)
706 # Delete all stale keys, meaning the user has only one key
708 #TODO: do we really want to delete all the other keys?
709 #Is this a problem with the GID generation to have multiple
715 #remove all the other keys
716 for key in keys_dict:
717 self.iotlab_api.DeleteKey(person, key)
718 self.iotlab_api.AddPersonKey(person, \
719 {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} )
720 #self.iotlab_api.AddPersonKey(person, {'key_type': 'ssh', \
725 def remove (self, sfa_record):
727 Removes users only. Mark the user as disabled in
728 LDAP. The user and his slice are then deleted from the db by running an
729 import on the registry.
733 :param sfa_record: record is the existing sfa record in the db
734 :type sfa_record: dict
736 ..warning::As fas as the slice is concerned, here only the leases are
737 removed from the slice. The slice is record itself is not removed from
739 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
741 TODO: return boolean for the slice part
743 sfa_record_type = sfa_record['type']
744 hrn = sfa_record['hrn']
745 if sfa_record_type == 'user':
747 #get user from iotlab ldap
748 person = self.iotlab_api.GetPersons(sfa_record)
749 #No registering at a given site in Iotlab.
750 #Once registered to the LDAP, all iotlab sites are
753 #Mark account as disabled in ldap
754 return self.iotlab_api.DeletePerson(sfa_record)
756 elif sfa_record_type == 'slice':
757 if self.iotlab_api.GetSlices(slice_filter = hrn, \
758 slice_filter_type = 'slice_hrn'):
759 ret = self.iotlab_api.DeleteSlice(sfa_record)