1 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
2 from sfa.util.sfalogging import logger
3 from sfa.storage.alchemy import dbsession
4 from sfa.storage.model import RegRecord
8 from sfa.managers.driver import Driver
9 from sfa.rspecs.version_manager import VersionManager
10 from sfa.rspecs.rspec import RSpec
12 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
15 from sfa.senslab.slabpostgres import SlabDB
18 from sfa.senslab.slabaggregate import SlabAggregate, slab_xrn_to_hostname
20 from sfa.senslab.slabslices import SlabSlices
23 from sfa.senslab.slabapi import SlabTestbedAPI
27 class SlabDriver(Driver):
28 """ Senslab Driver class inherited from Driver generic class.
30 Contains methods compliant with the SFA standard and the testbed
31 infrastructure (calls to LDAP and OAR).
33 .. seealso:: Driver class
36 def __init__(self, config):
39 Sets the senslab SFA config parameters ,
40 instanciates the testbed api and the senslab database.
42 :param config: senslab SFA configuration object
43 :type config: Config object
45 Driver.__init__ (self, config)
48 self.db = SlabDB(config, debug = False)
49 self.slab_api = SlabTestbedAPI(config)
52 def augment_records_with_testbed_info (self, record_list ):
55 Adds specific testbed info to the records.
57 :param record_list: list of sfa dictionaries records
58 :type record_list: list
59 :return: list of records with extended information in each record
62 return self.fill_record_info (record_list)
64 def fill_record_info(self, record_list):
66 For each SFA record, fill in the senslab specific and SFA specific
69 :param record_list: list of sfa dictionaries records
70 :type record_list: list
71 :return: list of records with extended information in each record
74 .. warnings:: Should not be modifying record_list directly because modi
75 fication are kept outside the method's scope. Howerver, there is no
76 other way to do it given the way it's called in registry manager.
79 logger.debug("SLABDRIVER \tfill_record_info records %s " %(record_list))
80 if not isinstance(record_list, list):
81 record_list = [record_list]
85 for record in record_list:
86 #If the record is a SFA slice record, then add information
87 #about the user of this slice. This kind of
88 #information is in the Senslab's DB.
89 if str(record['type']) == 'slice':
90 if 'reg_researchers' in record and \
91 isinstance(record['reg_researchers'], list) :
92 record['reg_researchers'] = \
93 record['reg_researchers'][0].__dict__
94 record.update({'PI':[record['reg_researchers']['hrn']],
95 'researcher': [record['reg_researchers']['hrn']],
99 'person_ids':[record['reg_researchers']['record_id']],
100 'geni_urn':'', #For client_helper.py compatibility
101 'keys':'', #For client_helper.py compatibility
102 'key_ids':''}) #For client_helper.py compatibility
105 #Get slab slice record and oar job id if any.
106 recslice_list = self.slab_api.GetSlices(slice_filter = \
108 slice_filter_type = 'slice_hrn')
111 logger.debug("SLABDRIVER \tfill_record_info \
112 TYPE SLICE RECUSER record['hrn'] %s ecord['oar_job_id']\
113 %s " %(record['hrn'], record['oar_job_id']))
114 del record['reg_researchers']
116 for rec in recslice_list:
117 logger.debug("SLABDRIVER\r\n \t \
118 fill_record_info oar_job_id %s " \
119 %(rec['oar_job_id']))
121 record['node_ids'] = [ self.slab_api.root_auth + \
122 hostname for hostname in rec['node_ids']]
127 logger.debug( "SLABDRIVER.PY \t fill_record_info SLICE \
128 recslice_list %s \r\n \t RECORD %s \r\n \
129 \r\n" %(recslice_list, record))
131 if str(record['type']) == 'user':
132 #The record is a SFA user record.
133 #Get the information about his slice from Senslab's DB
134 #and add it to the user record.
135 recslice_list = self.slab_api.GetSlices(\
136 slice_filter = record['record_id'],\
137 slice_filter_type = 'record_id_user')
139 logger.debug( "SLABDRIVER.PY \t fill_record_info TYPE USER \
140 recslice_list %s \r\n \t RECORD %s \r\n" \
141 %(recslice_list , record))
142 #Append slice record in records list,
143 #therefore fetches user and slice info again(one more loop)
144 #Will update PIs and researcher for the slice
146 recuser = recslice_list[0]['reg_researchers']
147 logger.debug( "SLABDRIVER.PY \t fill_record_info USER \
148 recuser %s \r\n \r\n" %(recuser))
150 recslice = recslice_list[0]
151 recslice.update({'PI':[recuser['hrn']],
152 'researcher': [recuser['hrn']],
153 'name':record['hrn'],
156 'person_ids':[recuser['record_id']]})
158 for rec in recslice_list:
159 recslice['oar_job_id'].append(rec['oar_job_id'])
163 recslice.update({'type':'slice', \
164 'hrn':recslice_list[0]['hrn']})
167 #GetPersons takes [] as filters
168 user_slab = self.slab_api.GetPersons([record])
171 record.update(user_slab[0])
172 #For client_helper.py compatibility
173 record.update( { 'geni_urn':'',
176 record_list.append(recslice)
178 logger.debug("SLABDRIVER.PY \tfill_record_info ADDING SLICE\
179 INFO TO USER records %s" %(record_list))
182 except TypeError, error:
183 logger.log_exc("SLABDRIVER \t fill_record_info EXCEPTION %s"\
189 def sliver_status(self, slice_urn, slice_hrn):
191 Receive a status request for slice named urn/hrn
192 urn:publicid:IDN+senslab+nturro_slice hrn senslab.nturro_slice
193 shall return a structure as described in
194 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
195 NT : not sure if we should implement this or not, but used by sface.
197 :param slice_urn: slice urn
198 :type slice_urn: string
199 :param slice_hrn: slice hrn
200 :type slice_hrn: string
205 #First get the slice with the slice hrn
206 slice_list = self.slab_api.GetSlices(slice_filter = slice_hrn, \
207 slice_filter_type = 'slice_hrn')
209 if len(slice_list) is 0:
210 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
212 #Used for fetching the user info witch comes along the slice info
213 one_slice = slice_list[0]
216 #Make a list of all the nodes hostnames in use for this slice
217 slice_nodes_list = []
218 #for single_slice in slice_list:
219 #for node in single_slice['node_ids']:
220 #slice_nodes_list.append(node['hostname'])
221 #for node in one_slice:
222 #slice_nodes_list.append(node['hostname'])
223 slice_nodes_list = one_slice['node_ids']
224 #Get all the corresponding nodes details
225 nodes_all = self.slab_api.GetNodes({'hostname':slice_nodes_list},
226 ['node_id', 'hostname','site','boot_state'])
227 nodeall_byhostname = dict([(one_node['hostname'], one_node) \
228 for one_node in nodes_all])
232 for single_slice in slice_list:
235 top_level_status = 'empty'
238 ['geni_urn','geni_error', 'pl_login','geni_status','geni_resources'], None)
239 result['pl_login'] = one_slice['reg_researchers'][0].hrn
240 logger.debug("Slabdriver - sliver_status Sliver status \
241 urn %s hrn %s single_slice %s \r\n " \
242 %(slice_urn, slice_hrn, single_slice))
244 if 'node_ids' not in single_slice:
246 result['geni_status'] = top_level_status
247 result['geni_resources'] = []
250 top_level_status = 'ready'
252 #A job is running on Senslab for this slice
253 # report about the local nodes that are in the slice only
255 result['geni_urn'] = slice_urn
258 for node_hostname in single_slice['node_ids']:
260 res['slab_hostname'] = node_hostname
261 res['slab_boot_state'] = nodeall_byhostname[node_hostname]['boot_state']
263 #res['pl_hostname'] = node['hostname']
264 #res['pl_boot_state'] = \
265 #nodeall_byhostname[node['hostname']]['boot_state']
266 #res['pl_last_contact'] = strftime(self.time_format, \
267 #gmtime(float(timestamp)))
268 sliver_id = Xrn(slice_urn, type='slice', \
269 id=nodeall_byhostname[node_hostname]['node_id'], \
270 authority=self.hrn).urn
272 res['geni_urn'] = sliver_id
273 #node_name = node['hostname']
274 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
276 res['geni_status'] = 'ready'
278 res['geni_status'] = 'failed'
279 top_level_status = 'failed'
281 res['geni_error'] = ''
283 resources.append(res)
285 result['geni_status'] = top_level_status
286 result['geni_resources'] = resources
287 logger.debug("SLABDRIVER \tsliver_statusresources %s res %s "\
292 def get_user_record(hrn):
294 Returns the user record based on the hrn from the SFA DB .
296 :param hrn: user's hrn
298 :return : user record from SFA database
302 return dbsession.query(RegRecord).filter_by(hrn = hrn).first()
305 def testbed_name (self):
307 Returns testbed's name.
313 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
314 def aggregate_version (self):
317 Returns the testbed's supported rspec advertisement and
322 version_manager = VersionManager()
323 ad_rspec_versions = []
324 request_rspec_versions = []
325 for rspec_version in version_manager.versions:
326 if rspec_version.content_type in ['*', 'ad']:
327 ad_rspec_versions.append(rspec_version.to_dict())
328 if rspec_version.content_type in ['*', 'request']:
329 request_rspec_versions.append(rspec_version.to_dict())
331 'testbed':self.testbed_name(),
332 'geni_request_rspec_versions': request_rspec_versions,
333 'geni_ad_rspec_versions': ad_rspec_versions,
338 def _get_requested_leases_list(self, rspec):
340 Process leases in rspec depending on the rspec version (format)
341 type. Find the lease requests in the rspec and creates
342 a lease request list with the mandatory information ( nodes,
343 start time and duration) of the valid leases (duration above or equal
344 to the senslab experiment minimum duration).
346 :param rspec: rspec request received.
348 :return: list of lease requests found in the rspec
351 requested_lease_list = []
352 for lease in rspec.version.get_leases():
353 single_requested_lease = {}
354 logger.debug("SLABDRIVER.PY \tcreate_sliver lease %s " %(lease))
356 if not lease.get('lease_id'):
357 if get_authority(lease['component_id']) == \
358 self.slab_api.root_auth:
359 single_requested_lease['hostname'] = \
360 slab_xrn_to_hostname(\
361 lease.get('component_id').strip())
362 single_requested_lease['start_time'] = \
363 lease.get('start_time')
364 single_requested_lease['duration'] = lease.get('duration')
365 #Check the experiment's duration is valid before adding
366 #the lease to the requested leases list
367 duration_in_seconds = \
368 int(single_requested_lease['duration'])
369 if duration_in_seconds > self.slab_api.GetMinExperimentDurationInSec() :
370 requested_lease_list.append(single_requested_lease)
372 return requested_lease_list
375 def _group_leases_by_start_time(requested_lease_list):
377 Create dict of leases by start_time, regrouping nodes reserved
378 at the same time, for the same amount of time so as to
379 define one job on OAR.
381 :param requested_lease_list: list of leases
382 :type requested_lease_list: list
383 :return: Dictionary with key = start time, value = list of leases
384 with the same start time.
388 requested_job_dict = {}
389 for lease in requested_lease_list:
391 #In case it is an asap experiment start_time is empty
392 if lease['start_time'] == '':
393 lease['start_time'] = '0'
395 if lease['start_time'] not in requested_job_dict:
396 if isinstance(lease['hostname'], str):
397 lease['hostname'] = [lease['hostname']]
400 requested_job_dict[lease['start_time']] = lease
403 job_lease = requested_job_dict[lease['start_time']]
404 if lease['duration'] == job_lease['duration'] :
405 job_lease['hostname'].append(lease['hostname'])
407 return requested_job_dict
409 def _process_requested_jobs(self, rspec):
411 Turns the requested leases and information into a dictionary
412 of requested jobs, grouped by starting time.
414 :param rspec: RSpec received
418 requested_lease_list = self._get_requested_leases_list(rspec)
419 logger.debug("SLABDRIVER _process_requested_jobs requested_lease_list \
420 %s"%(requested_lease_list))
421 job_dict = self._group_leases_by_start_time(requested_lease_list)
422 logger.debug("SLABDRIVER _process_requested_jobs job_dict\
427 def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \
430 Answer to CreateSliver.
431 Creates the leases and slivers for the users from the information
432 found in the rspec string.
433 Launch experiment on OAR if the requested leases is valid. Delete
434 no longer requested leases.
437 :param creds: user's credentials
439 :param users: user record list
444 :return: a valid Rspec for the slice which has just been
450 aggregate = SlabAggregate(self)
452 slices = SlabSlices(self)
453 peer = slices.get_peer(slice_hrn)
454 sfa_peer = slices.get_sfa_peer(slice_hrn)
457 if not isinstance(creds, list):
461 slice_record = users[0].get('slice_record', {})
462 logger.debug("SLABDRIVER.PY \t ===============create_sliver \t\
463 creds %s \r\n \r\n users %s" \
465 slice_record['user'] = {'keys':users[0]['keys'], \
466 'email':users[0]['email'], \
467 'hrn':slice_record['reg-researchers'][0]}
469 rspec = RSpec(rspec_string)
470 logger.debug("SLABDRIVER.PY \t create_sliver \trspec.version \
471 %s slice_record %s users %s" \
472 %(rspec.version,slice_record, users))
475 # ensure site record exists?
476 # ensure slice record exists
477 #Removed options to verify_slice SA 14/08/12
478 sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \
481 # ensure person records exists
482 #verify_persons returns added persons but since the return value
484 slices.verify_persons(slice_hrn, sfa_slice, users, peer, \
485 sfa_peer, options=options)
486 #requested_attributes returned by rspec.version.get_slice_attributes()
487 #unused, removed SA 13/08/12
488 #rspec.version.get_slice_attributes()
490 logger.debug("SLABDRIVER.PY create_sliver slice %s " %(sfa_slice))
492 # add/remove slice from nodes
494 #requested_slivers = [node.get('component_id') \
495 #for node in rspec.version.get_nodes_with_slivers()\
496 #if node.get('authority_id') is self.slab_api.root_auth]
497 #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
498 #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
499 #requested_slivers %s listnodes %s" \
500 #%(requested_slivers,l))
501 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
502 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
505 requested_job_dict = self._process_requested_jobs(rspec)
508 logger.debug("SLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\
509 %(requested_job_dict))
510 #verify_slice_leases returns the leases , but the return value is unused
511 #here. Removed SA 13/08/12
512 slices.verify_slice_leases(sfa_slice, \
513 requested_job_dict, peer)
515 return aggregate.get_rspec(slice_xrn=slice_urn, \
516 login=sfa_slice['login'], version=rspec.version)
519 def delete_sliver (self, slice_urn, slice_hrn, creds, options):
521 Deletes the lease associated with the slice hrn and the credentials
522 if the slice belongs to senslab. Answer to DeleteSliver.
524 :return: 1 if the slice to delete was not found on senslab,
525 True if the deletion was successful, False otherwise otherwise.
527 .. note:: Should really be named delete_leases because senslab does
528 not have any slivers, but only deals with leases. However, SFA api only
529 have delete_sliver define so far. SA 13.05/2013
532 sfa_slice_list = self.slab_api.GetSlices(slice_filter = slice_hrn, \
533 slice_filter_type = 'slice_hrn')
535 if not sfa_slice_list:
538 #Delete all leases in the slice
539 for sfa_slice in sfa_slice_list:
542 logger.debug("SLABDRIVER.PY delete_sliver slice %s" %(sfa_slice))
543 slices = SlabSlices(self)
544 # determine if this is a peer slice
546 peer = slices.get_peer(slice_hrn)
548 logger.debug("SLABDRIVER.PY delete_sliver peer %s \
549 \r\n \t sfa_slice %s " %(peer, sfa_slice))
552 self.slab_api.DeleteSliceFromNodes(sfa_slice)
558 def list_resources (self, slice_urn, slice_hrn, creds, options):
560 List resources from the senslab aggregate and returns a Rspec
561 advertisement with resources found when slice_urn and slice_hrn are None
562 (in case of resource discovery).
563 If a slice hrn and urn are provided, list experiment's slice
564 nodes in a rspec format. Answer to ListResources.
566 :param options: options used when listing resources (list_leases, info,
568 :return: rspec string in xml
572 #cached_requested = options.get('cached', True)
574 version_manager = VersionManager()
575 # get the rspec's return format from options
577 version_manager.get_version(options.get('geni_rspec_version'))
578 version_string = "rspec_%s" % (rspec_version)
580 #panos adding the info option to the caching key (can be improved)
581 if options.get('info'):
582 version_string = version_string + "_" + \
583 options.get('info', 'default')
585 # Adding the list_leases option to the caching key
586 if options.get('list_leases'):
587 version_string = version_string + "_" + \
588 options.get('list_leases', 'default')
590 # Adding geni_available to caching key
591 if options.get('geni_available'):
592 version_string = version_string + "_" + \
593 str(options.get('geni_available'))
595 # look in cache first
596 #if cached_requested and self.cache and not slice_hrn:
597 #rspec = self.cache.get(version_string)
599 #logger.debug("SlabDriver.ListResources: \
600 #returning cached advertisement")
603 #panos: passing user-defined options
604 aggregate = SlabAggregate(self)
606 rspec = aggregate.get_rspec(slice_xrn=slice_urn, \
607 version=rspec_version, options=options)
610 #if self.cache and not slice_hrn:
611 #logger.debug("Slab.ListResources: stores advertisement in cache")
612 #self.cache.add(version_string, rspec)
617 def list_slices (self, creds, options):
619 Answer to ListSlices.
620 List slices belonging to senslab, returns slice urns list.
621 No caching used. Options unused but are defined in the SFA method
624 :return: slice urns list
628 # look in cache first
630 #slices = self.cache.get('slices')
632 #logger.debug("PlDriver.list_slices returns from cache")
637 slices = self.slab_api.GetSlices()
638 logger.debug("SLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices))
639 slice_hrns = [slab_slice['hrn'] for slab_slice in slices]
641 slice_urns = [hrn_to_urn(slice_hrn, 'slice') \
642 for slice_hrn in slice_hrns]
646 #logger.debug ("SlabDriver.list_slices stores value in cache")
647 #self.cache.add('slices', slice_urns)
652 def register (self, sfa_record, hrn, pub_key):
654 Adding new user, slice, node or site should not be handled
657 ..warnings:: should not be used. Different components are in charge of
658 doing this task. Adding nodes = OAR
659 Adding users = LDAP Senslab
660 Adding slice = Import from LDAP users
663 :param sfa_record: record provided by the client of the
665 :type sfa_record: dict
670 def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
671 """No site or node record update allowed in Senslab.
672 The only modifications authorized here are key deletion/addition
673 on an existing user and password change.
674 On an existing user, CAN NOT BE MODIFIED:
675 'first_name', 'last_name', 'email'
676 DOES NOT EXIST IN SENSLAB:
677 'phone', 'url', 'bio','title', 'accepted_aup',
678 A slice is bound to its user, so modifying the user's ssh key should
679 modify the slice's GID after an import procedure.
681 :param old_sfa_record: what is in the db for this hrn
682 :param new_sfa_record: what was passed to the Update call
684 ..seealso:: update in driver.py.
687 pointer = old_sfa_record['pointer']
688 old_sfa_record_type = old_sfa_record['type']
690 # new_key implemented for users only
691 if new_key and old_sfa_record_type not in [ 'user' ]:
692 raise UnknownSfaType(old_sfa_record_type)
695 if old_sfa_record_type == "user":
697 all_fields = new_sfa_record
698 for key in all_fields.keys():
699 if key in ['key', 'password']:
700 update_fields[key] = all_fields[key]
704 # must check this key against the previous one if it exists
705 persons = self.slab_api.GetPersons([old_sfa_record])
707 keys = [person['pkey']]
708 #Get all the person's keys
709 keys_dict = self.slab_api.GetKeys(keys)
711 # Delete all stale keys, meaning the user has only one key
713 #TODO: do we really want to delete all the other keys?
714 #Is this a problem with the GID generation to have multiple
720 #remove all the other keys
721 for key in keys_dict:
722 self.slab_api.DeleteKey(person, key)
723 self.slab_api.AddPersonKey(person, \
724 {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} )
725 #self.slab_api.AddPersonKey(person, {'key_type': 'ssh', \
730 def remove (self, sfa_record):
732 Removes users only. Mark the user as disabled in
733 LDAP. The user and his slice are then deleted from the db by running an
734 import on the registry.
738 :param sfa_record: record is the existing sfa record in the db
739 :type sfa_record: dict
741 ..warning::As fas as the slice is concerned, here only the leases are
742 removed from the slice. The slice is record itself is not removed from
744 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
746 TODO: return boolean for the slice part
748 sfa_record_type = sfa_record['type']
749 hrn = sfa_record['hrn']
750 if sfa_record_type == 'user':
752 #get user from senslab ldap
753 person = self.slab_api.GetPersons(sfa_record)
754 #No registering at a given site in Senslab.
755 #Once registered to the LDAP, all senslab sites are
758 #Mark account as disabled in ldap
759 return self.slab_api.DeletePerson(sfa_record)
761 elif sfa_record_type == 'slice':
762 if self.slab_api.GetSlices(slice_filter = hrn, \
763 slice_filter_type = 'slice_hrn'):
764 ret = self.slab_api.DeleteSlice(sfa_record)