2 Implements what a driver should provide for SFA to work.
4 from sfa.util.faults import SliverDoesNotExist, Forbidden
5 from sfa.util.sfalogging import logger
7 from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
8 from sfa.util.sfatime import utcparse, datetime_to_string
9 from sfa.trust.certificate import Keypair, convert_public_key
11 from sfa.trust.hierarchy import Hierarchy
12 from sfa.trust.gid import create_uuid
14 from sfa.managers.driver import Driver
15 from sfa.rspecs.version_manager import VersionManager
16 from sfa.rspecs.rspec import RSpec
18 from sfa.iotlab.iotlabxrn import IotlabXrn, xrn_object
19 from sfa.util.xrn import Xrn, hrn_to_urn, get_authority, urn_to_hrn
20 from sfa.iotlab.iotlabaggregate import IotlabAggregate
21 from sfa.iotlab.iotlabxrn import xrn_to_hostname
22 from sfa.iotlab.iotlabslices import IotlabSlices
24 from sfa.trust.credential import Credential
25 from sfa.storage.model import SliverAllocation
27 from sfa.iotlab.iotlabshell import IotlabShell
28 from sqlalchemy.orm import joinedload
30 class IotlabDriver(Driver):
31 """ Iotlab Driver class inherited from Driver generic class.
33 Contains methods compliant with the SFA standard and the testbed
34 infrastructure (calls to LDAP and OAR).
36 .. seealso::: Driver class
39 def __init__(self, api):
42 Sets the iotlab SFA config parameters,
43 instanciates the testbed api and the iotlab database.
45 :param config: iotlab SFA configuration object
46 :type config: Config object
49 Driver.__init__(self, api)
52 self.testbed_shell = IotlabShell(config)
55 def GetPeers(self, peer_filter=None ):
56 """ Gathers registered authorities in SFA DB and looks for specific peer
57 if peer_filter is specified.
58 :param peer_filter: name of the site authority looked for.
59 :type peer_filter: string
60 :returns: list of records.
65 existing_hrns_by_types = {}
66 logger.debug("IOTLAB_API \tGetPeers peer_filter %s " % (peer_filter))
67 all_records = self.api.dbsession().query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
69 for record in all_records:
70 existing_records[(record.hrn, record.type)] = record
71 if record.type not in existing_hrns_by_types:
72 existing_hrns_by_types[record.type] = [record.hrn]
74 existing_hrns_by_types[record.type].append(record.hrn)
76 logger.debug("IOTLAB_API \tGetPeer\texisting_hrns_by_types %s "
77 % (existing_hrns_by_types))
82 records_list.append(existing_records[(peer_filter,
85 for hrn in existing_hrns_by_types['authority']:
86 records_list.append(existing_records[(hrn, 'authority')])
88 logger.debug("IOTLAB_API \tGetPeer \trecords_list %s "
94 return_records = records_list
95 logger.debug("IOTLAB_API \tGetPeer return_records %s "
99 def GetKeys(self, key_filter=None):
100 """Returns a dict of dict based on the key string. Each dict entry
101 contains the key id, the ssh key, the user's email and the
103 If key_filter is specified and is an array of key identifiers,
104 only keys matching the filter will be returned.
106 Admin may query all keys. Non-admins may only query their own keys.
109 :returns: dict with ssh key as key and dicts as value.
112 query = self.api.dbsession().query(RegKey)
113 if key_filter is None:
114 keys = query.options(joinedload('reg_user')).all()
116 constraint = RegKey.key.in_(key_filter)
117 keys = query.options(joinedload('reg_user')).filter(constraint).all()
121 key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
122 'email': key.reg_user.email,
123 'hrn': key.reg_user.hrn}
125 #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
126 #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
127 #for user in ldap_rslt)
129 logger.debug("IOTLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
134 def AddPerson(self, record):
137 Adds a new account. Any fields specified in records are used,
138 otherwise defaults are used. Creates an appropriate login by calling
141 :param record: dictionary with the sfa user's properties.
142 :returns: a dicitonary with the status. If successful, the dictionary
143 boolean is set to True and there is a 'uid' key with the new login
144 added to LDAP, otherwise the bool is set to False and a key
145 'message' is in the dictionary, with the error message.
149 ret = self.testbed_shell.ldap.LdapAddUser(record)
151 if ret['bool'] is True:
152 record['hrn'] = self.testbed_shell.root_auth + '.' + ret['uid']
153 logger.debug("IOTLAB_API AddPerson return code %s record %s "
155 self.__add_person_to_db(record)
158 def __add_person_to_db(self, user_dict):
160 Add a federated user straight to db when the user issues a lease
161 request with iotlab nodes and that he has not registered with iotlab
162 yet (that is he does not have a LDAP entry yet).
163 Uses parts of the routines in IotlabImport when importing user from
164 LDAP. Called by AddPerson, right after LdapAddUser.
165 :param user_dict: Must contain email, hrn and pkey to get a GID
166 and be added to the SFA db.
167 :type user_dict: dict
170 query = self.api.dbsession().query(RegUser)
171 check_if_exists = query.filter_by(email = user_dict['email']).first()
173 if not check_if_exists:
174 logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
176 hrn = user_dict['hrn']
177 person_urn = hrn_to_urn(hrn, 'user')
178 pubkey = user_dict['pkey']
180 pkey = convert_public_key(pubkey)
182 #key not good. create another pkey
183 logger.warn('__add_person_to_db: unable to convert public \
185 pkey = Keypair(create=True)
188 if pubkey is not None and pkey is not None :
189 hierarchy = Hierarchy()
190 person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
192 if user_dict['email']:
193 logger.debug("__add_person_to_db \r\n \r\n \
194 IOTLAB IMPORTER PERSON EMAIL OK email %s "\
195 %(user_dict['email']))
196 person_gid.set_email(user_dict['email'])
198 user_record = RegUser(hrn=hrn , pointer= '-1', \
199 authority=get_authority(hrn), \
200 email=user_dict['email'], gid = person_gid)
201 user_record.reg_keys = [RegKey(user_dict['pkey'])]
202 user_record.just_created()
203 self.api.dbsession().add (user_record)
204 self.api.dbsession().commit()
209 def _sql_get_slice_info(self, slice_filter):
211 Get the slice record based on the slice hrn. Fetch the record of the
212 user associated with the slice by using joinedload based on the
213 reg_researcher relationship.
215 :param slice_filter: the slice hrn we are looking for
216 :type slice_filter: string
217 :returns: the slice record enhanced with the user's information if the
218 slice was found, None it wasn't.
220 :rtype: dict or None.
222 #DO NOT USE RegSlice - reg_researchers to get the hrn
223 #of the user otherwise will mess up the RegRecord in
224 #Resolve, don't know why - SA 08/08/2012
226 #Only one entry for one user = one slice in testbed_xp table
227 #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
228 raw_slicerec = self.api.dbsession().query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
229 #raw_slicerec = self.api.dbsession().query(RegRecord).filter_by(hrn = slice_filter).first()
232 #raw_slicerec.reg_researchers
233 raw_slicerec = raw_slicerec.__dict__
234 logger.debug(" IOTLAB_API \t _sql_get_slice_info slice_filter %s \
235 raw_slicerec %s" % (slice_filter, raw_slicerec))
236 slicerec = raw_slicerec
237 #only one researcher per slice so take the first one
238 #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
239 #del slicerec['reg_researchers']['_sa_instance_state']
245 def _sql_get_slice_info_from_user(self, slice_filter):
247 Get the slice record based on the user recordid by using a joinedload
248 on the relationship reg_slices_as_researcher. Format the sql record
249 into a dict with the mandatory fields for user and slice.
250 :returns: dict with slice record and user record if the record was found
251 based on the user's id, None if not..
252 :rtype:dict or None..
254 #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
255 raw_slicerec = self.api.dbsession().query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
256 #raw_slicerec = self.api.dbsession().query(RegRecord).filter_by(record_id = slice_filter).first()
257 #Put it in correct order
258 user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
259 'classtype', 'authority', 'gid', 'record_id',
260 'date_created', 'type', 'email', 'pointer']
261 slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
262 'classtype', 'authority', 'gid', 'record_id',
263 'date_created', 'type', 'pointer']
265 #raw_slicerec.reg_slices_as_researcher
266 raw_slicerec = raw_slicerec.__dict__
269 dict([(k, raw_slicerec[
270 'reg_slices_as_researcher'][0].__dict__[k])
271 for k in slice_needed_fields])
272 slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
273 for k in user_needed_fields])
274 #TODO Handle multiple slices for one user SA 10/12/12
275 #for now only take the first slice record associated to the rec user
276 ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
277 #del raw_slicerec['reg_slices_as_researcher']
278 #slicerec['reg_researchers'] = raw_slicerec
279 ##del slicerec['_sa_instance_state']
288 def _get_slice_records(self, slice_filter=None,
289 slice_filter_type=None):
291 Get the slice record depending on the slice filter and its type.
292 :param slice_filter: Can be either the slice hrn or the user's record
294 :type slice_filter: string
295 :param slice_filter_type: describes the slice filter type used, can be
296 slice_hrn or record_id_user
298 :returns: the slice record
300 .. seealso::_sql_get_slice_info_from_user
301 .. seealso:: _sql_get_slice_info
304 #Get list of slices based on the slice hrn
305 if slice_filter_type == 'slice_hrn':
307 #if get_authority(slice_filter) == self.root_auth:
308 #login = slice_filter.split(".")[1].split("_")[0]
310 slicerec = self._sql_get_slice_info(slice_filter)
316 #Get slice based on user id
317 if slice_filter_type == 'record_id_user':
319 slicerec = self._sql_get_slice_info_from_user(slice_filter)
322 fixed_slicerec_dict = slicerec
323 #At this point if there is no login it means
324 #record_id_user filter has been used for filtering
326 ##If theslice record is from iotlab
327 #if fixed_slicerec_dict['peer_authority'] is None:
328 #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
329 #return login, fixed_slicerec_dict
330 return fixed_slicerec_dict
336 def GetSlices(self, slice_filter=None, slice_filter_type=None,
338 """Get the slice records from the iotlab db and add lease information
341 :param slice_filter: can be the slice hrn or slice record id in the db
342 depending on the slice_filter_type.
343 :param slice_filter_type: defines the type of the filtering used, Can be
344 either 'slice_hrn' or "record_id'.
345 :type slice_filter: string
346 :type slice_filter_type: string
347 :returns: a slice dict if slice_filter and slice_filter_type
348 are specified and a matching entry is found in the db. The result
349 is put into a list.Or a list of slice dictionnaries if no filters
356 authorized_filter_types_list = ['slice_hrn', 'record_id_user']
357 return_slicerec_dictlist = []
359 #First try to get information on the slice based on the filter provided
360 if slice_filter_type in authorized_filter_types_list:
361 fixed_slicerec_dict = self._get_slice_records(slice_filter,
363 # if the slice was not found in the sfa db
364 if fixed_slicerec_dict is None:
365 return return_slicerec_dictlist
367 slice_hrn = fixed_slicerec_dict['hrn']
369 logger.debug(" IOTLAB_API \tGetSlices login %s \
370 slice record %s slice_filter %s \
371 slice_filter_type %s " % (login,
372 fixed_slicerec_dict, slice_filter,
376 #Now we have the slice record fixed_slicerec_dict, get the
377 #jobs associated to this slice
380 leases_list = self.testbed_shell.GetLeases(login=login)
381 #If no job is running or no job scheduled
382 #return only the slice record
383 if leases_list == [] and fixed_slicerec_dict:
384 return_slicerec_dictlist.append(fixed_slicerec_dict)
386 # if the jobs running don't belong to the user/slice we are looking
388 leases_hrn = [lease['slice_hrn'] for lease in leases_list]
389 if slice_hrn not in leases_hrn:
390 return_slicerec_dictlist.append(fixed_slicerec_dict)
391 #If several jobs for one slice , put the slice record into
392 # each lease information dict
393 for lease in leases_list:
395 logger.debug("IOTLAB_API.PY \tGetSlices slice_filter %s \
396 \t lease['slice_hrn'] %s"
397 % (slice_filter, lease['slice_hrn']))
398 if lease['slice_hrn'] == slice_hrn:
399 slicerec_dict['oar_job_id'] = lease['lease_id']
400 #Update lease dict with the slice record
401 if fixed_slicerec_dict:
402 fixed_slicerec_dict['oar_job_id'] = []
403 fixed_slicerec_dict['oar_job_id'].append(
404 slicerec_dict['oar_job_id'])
405 slicerec_dict.update(fixed_slicerec_dict)
406 #slicerec_dict.update({'hrn':\
407 #str(fixed_slicerec_dict['slice_hrn'])})
408 slicerec_dict['slice_hrn'] = lease['slice_hrn']
409 slicerec_dict['hrn'] = lease['slice_hrn']
410 slicerec_dict['user'] = lease['user']
411 slicerec_dict.update(
413 {'hostname': lease['reserved_nodes']}})
414 slicerec_dict.update({'node_ids': lease['reserved_nodes']})
418 return_slicerec_dictlist.append(slicerec_dict)
419 logger.debug("IOTLAB_API.PY \tGetSlices \
420 OHOHOHOH %s" %(return_slicerec_dictlist))
422 logger.debug("IOTLAB_API.PY \tGetSlices \
423 slicerec_dict %s return_slicerec_dictlist %s \
424 lease['reserved_nodes'] \
425 %s" % (slicerec_dict, return_slicerec_dictlist,
426 lease['reserved_nodes']))
428 logger.debug("IOTLAB_API.PY \tGetSlices RETURN \
429 return_slicerec_dictlist %s"
430 % (return_slicerec_dictlist))
432 return return_slicerec_dictlist
436 #Get all slices from the iotlab sfa database ,
437 #put them in dict format
438 #query_slice_list = dbsession.query(RegRecord).all()
440 self.api.dbsession().query(RegSlice).options(joinedload('reg_researchers')).all()
442 for record in query_slice_list:
443 tmp = record.__dict__
444 tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
445 #del tmp['reg_researchers']['_sa_instance_state']
446 return_slicerec_dictlist.append(tmp)
447 #return_slicerec_dictlist.append(record.__dict__)
449 #Get all the jobs reserved nodes
450 leases_list = self.testbed_shell.GetReservedNodes()
452 for fixed_slicerec_dict in return_slicerec_dictlist:
454 #Check if the slice belongs to a iotlab user
455 if fixed_slicerec_dict['peer_authority'] is None:
456 owner = fixed_slicerec_dict['hrn'].split(
457 ".")[1].split("_")[0]
460 for lease in leases_list:
461 if owner == lease['user']:
462 slicerec_dict['oar_job_id'] = lease['lease_id']
464 #for reserved_node in lease['reserved_nodes']:
465 logger.debug("IOTLAB_API.PY \tGetSlices lease %s "
467 slicerec_dict.update(fixed_slicerec_dict)
468 slicerec_dict.update({'node_ids':
469 lease['reserved_nodes']})
470 slicerec_dict.update({'list_node_ids':
472 lease['reserved_nodes']}})
474 #slicerec_dict.update({'hrn':\
475 #str(fixed_slicerec_dict['slice_hrn'])})
476 #return_slicerec_dictlist.append(slicerec_dict)
477 fixed_slicerec_dict.update(slicerec_dict)
479 logger.debug("IOTLAB_API.PY \tGetSlices RETURN \
480 return_slicerec_dictlist %s \t slice_filter %s " \
481 %(return_slicerec_dictlist, slice_filter))
483 return return_slicerec_dictlist
485 def AddSlice(self, slice_record, user_record):
488 Add slice to the local iotlab sfa tables if the slice comes
489 from a federated site and is not yet in the iotlab sfa DB,
490 although the user has already a LDAP login.
491 Called by verify_slice during lease/sliver creation.
493 :param slice_record: record of slice, must contain hrn, gid, slice_id
494 and authority of the slice.
495 :type slice_record: dictionary
496 :param user_record: record of the user
497 :type user_record: RegUser
501 sfa_record = RegSlice(hrn=slice_record['hrn'],
502 gid=slice_record['gid'],
503 pointer=slice_record['slice_id'],
504 authority=slice_record['authority'])
505 logger.debug("IOTLAB_API.PY AddSlice sfa_record %s user_record %s"
506 % (sfa_record, user_record))
507 sfa_record.just_created()
508 self.api.dbsession().add(sfa_record)
509 self.api.dbsession().commit()
510 #Update the reg-researcher dependance table
511 sfa_record.reg_researchers = [user_record]
512 self.api.dbsession().commit()
516 def augment_records_with_testbed_info(self, record_list):
519 Adds specific testbed info to the records.
521 :param record_list: list of sfa dictionaries records
522 :type record_list: list
523 :returns: list of records with extended information in each record
527 return self.fill_record_info(record_list)
529 def fill_record_info(self, record_list):
532 For each SFA record, fill in the iotlab specific and SFA specific
533 fields in the record.
535 :param record_list: list of sfa dictionaries records
536 :type record_list: list
537 :returns: list of records with extended information in each record
540 .. warning:: Should not be modifying record_list directly because modi
541 fication are kept outside the method's scope. Howerver, there is no
542 other way to do it given the way it's called in registry manager.
546 logger.debug("IOTLABDRIVER \tfill_record_info records %s "
548 if not isinstance(record_list, list):
549 record_list = [record_list]
552 for record in record_list:
554 if str(record['type']) == 'node':
555 # look for node info using GetNodes
556 # the record is about one node only
557 filter_dict = {'hrn': [record['hrn']]}
558 node_info = self.testbed_shell.GetNodes(filter_dict)
559 # the node_info is about one node only, but it is formatted
561 record.update(node_info[0])
562 logger.debug("IOTLABDRIVER.PY \t \
563 fill_record_info NODE" % (record))
565 #If the record is a SFA slice record, then add information
566 #about the user of this slice. This kind of
567 #information is in the Iotlab's DB.
568 if str(record['type']) == 'slice':
569 if 'reg_researchers' in record and isinstance(record
572 record['reg_researchers'] = \
573 record['reg_researchers'][0].__dict__
575 {'PI': [record['reg_researchers']['hrn']],
576 'researcher': [record['reg_researchers']['hrn']],
577 'name': record['hrn'],
580 'person_ids': [record['reg_researchers']
582 # For client_helper.py compatibility
584 # For client_helper.py compatibility
586 # For client_helper.py compatibility
589 #Get iotlab slice record and oar job id if any.
590 recslice_list = self.GetSlices(
591 slice_filter=str(record['hrn']),
592 slice_filter_type='slice_hrn')
594 logger.debug("IOTLABDRIVER \tfill_record_info \
595 TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\
596 %s " % (record['hrn'], record['oar_job_id']))
597 del record['reg_researchers']
599 for rec in recslice_list:
600 logger.debug("IOTLABDRIVER\r\n \t \
601 fill_record_info oar_job_id %s "
602 % (rec['oar_job_id']))
604 record['node_ids'] = [self.testbed_shell.root_auth +
605 '.' + hostname for hostname
610 logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \
611 recslice_list %s \r\n \t RECORD %s \r\n \
612 \r\n" % (recslice_list, record))
614 if str(record['type']) == 'user':
615 #The record is a SFA user record.
616 #Get the information about his slice from Iotlab's DB
617 #and add it to the user record.
618 recslice_list = self.GetSlices(
619 slice_filter=record['record_id'],
620 slice_filter_type='record_id_user')
622 logger.debug("IOTLABDRIVER.PY \t fill_record_info \
623 TYPE USER recslice_list %s \r\n \t RECORD %s \r\n"
624 % (recslice_list, record))
625 #Append slice record in records list,
626 #therefore fetches user and slice info again(one more loop)
627 #Will update PIs and researcher for the slice
629 recuser = recslice_list[0]['reg_researchers']
630 logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \
631 recuser %s \r\n \r\n" % (recuser))
633 recslice = recslice_list[0]
635 {'PI': [recuser['hrn']],
636 'researcher': [recuser['hrn']],
637 'name': record['hrn'],
640 'person_ids': [recuser['record_id']]})
642 for rec in recslice_list:
643 recslice['oar_job_id'].append(rec['oar_job_id'])
647 recslice.update({'type': 'slice',
648 'hrn': recslice_list[0]['hrn']})
650 #GetPersons takes [] as filters
651 user_iotlab = self.testbed_shell.GetPersons([record])
653 record.update(user_iotlab[0])
654 #For client_helper.py compatibility
659 record_list.append(recslice)
661 logger.debug("IOTLABDRIVER.PY \t \
662 fill_record_info ADDING SLICE\
663 INFO TO USER records %s" % (record_list))
665 except TypeError, error:
666 logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"
671 def sliver_status(self, slice_urn, slice_hrn):
673 Receive a status request for slice named urn/hrn
674 urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice
675 shall return a structure as described in
676 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
677 NT : not sure if we should implement this or not, but used by sface.
679 :param slice_urn: slice urn
680 :type slice_urn: string
681 :param slice_hrn: slice hrn
682 :type slice_hrn: string
686 #First get the slice with the slice hrn
687 slice_list = self.GetSlices(slice_filter=slice_hrn,
688 slice_filter_type='slice_hrn')
690 if len(slice_list) == 0:
691 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
693 #Used for fetching the user info witch comes along the slice info
694 one_slice = slice_list[0]
696 #Make a list of all the nodes hostnames in use for this slice
697 slice_nodes_list = []
698 slice_nodes_list = one_slice['node_ids']
699 #Get all the corresponding nodes details
700 nodes_all = self.testbed_shell.GetNodes(
701 {'hostname': slice_nodes_list},
702 ['node_id', 'hostname', 'site', 'boot_state'])
703 nodeall_byhostname = dict([(one_node['hostname'], one_node)
704 for one_node in nodes_all])
706 for single_slice in slice_list:
708 top_level_status = 'empty'
711 ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status',
712 'geni_resources'], None)
714 # ['geni_urn','geni_error', 'pl_login','geni_status',
715 # 'geni_resources'], None)
716 # result['pl_login'] = one_slice['reg_researchers'][0].hrn
717 result['iotlab_login'] = one_slice['user']
718 logger.debug("Slabdriver - sliver_status Sliver status \
719 urn %s hrn %s single_slice %s \r\n "
720 % (slice_urn, slice_hrn, single_slice))
722 if 'node_ids' not in single_slice:
724 result['geni_status'] = top_level_status
725 result['geni_resources'] = []
728 top_level_status = 'ready'
730 #A job is running on Iotlab for this slice
731 # report about the local nodes that are in the slice only
733 result['geni_urn'] = slice_urn
736 for node_hostname in single_slice['node_ids']:
738 res['iotlab_hostname'] = node_hostname
739 res['iotlab_boot_state'] = \
740 nodeall_byhostname[node_hostname]['boot_state']
742 #res['pl_hostname'] = node['hostname']
743 #res['pl_boot_state'] = \
744 #nodeall_byhostname[node['hostname']]['boot_state']
745 #res['pl_last_contact'] = strftime(self.time_format, \
746 #gmtime(float(timestamp)))
748 slice_urn, type='slice',
749 id=nodeall_byhostname[node_hostname]['node_id']).urn
751 res['geni_urn'] = sliver_id
752 #node_name = node['hostname']
753 if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive':
755 res['geni_status'] = 'ready'
757 res['geni_status'] = 'failed'
758 top_level_status = 'failed'
760 res['geni_error'] = ''
762 resources.append(res)
764 result['geni_status'] = top_level_status
765 result['geni_resources'] = resources
766 logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "
770 def get_user_record(self, hrn):
773 Returns the user record based on the hrn from the SFA DB .
775 :param hrn: user's hrn
777 :returns: user record from SFA database
781 return self.api.dbsession().query(RegRecord).filter_by(hrn=hrn).first()
783 def testbed_name(self):
786 Returns testbed's name.
787 :returns: testbed authority name.
794 def _get_requested_leases_list(self, rspec):
796 Process leases in rspec depending on the rspec version (format)
797 type. Find the lease requests in the rspec and creates
798 a lease request list with the mandatory information ( nodes,
799 start time and duration) of the valid leases (duration above or
800 equal to the iotlab experiment minimum duration).
802 :param rspec: rspec request received.
804 :returns: list of lease requests found in the rspec
807 requested_lease_list = []
808 for lease in rspec.version.get_leases():
809 single_requested_lease = {}
810 logger.debug("IOTLABDRIVER.PY \t \
811 _get_requested_leases_list lease %s " % (lease))
813 if not lease.get('lease_id'):
814 if get_authority(lease['component_id']) == \
815 self.testbed_shell.root_auth:
816 single_requested_lease['hostname'] = \
818 lease.get('component_id').strip())
819 single_requested_lease['start_time'] = \
820 lease.get('start_time')
821 single_requested_lease['duration'] = lease.get('duration')
822 #Check the experiment's duration is valid before adding
823 #the lease to the requested leases list
824 duration_in_seconds = \
825 int(single_requested_lease['duration'])
826 if duration_in_seconds >= self.testbed_shell.GetMinExperimentDurationInGranularity():
827 requested_lease_list.append(single_requested_lease)
829 return requested_lease_list
832 def _group_leases_by_start_time(requested_lease_list):
834 Create dict of leases by start_time, regrouping nodes reserved
835 at the same time, for the same amount of time so as to
836 define one job on OAR.
838 :param requested_lease_list: list of leases
839 :type requested_lease_list: list
840 :returns: Dictionary with key = start time, value = list of leases
841 with the same start time.
846 requested_xp_dict = {}
847 for lease in requested_lease_list:
849 #In case it is an asap experiment start_time is empty
850 if lease['start_time'] == '':
851 lease['start_time'] = '0'
853 if lease['start_time'] not in requested_xp_dict:
854 if isinstance(lease['hostname'], str):
855 lease['hostname'] = [lease['hostname']]
857 requested_xp_dict[lease['start_time']] = lease
860 job_lease = requested_xp_dict[lease['start_time']]
861 if lease['duration'] == job_lease['duration']:
862 job_lease['hostname'].append(lease['hostname'])
864 return requested_xp_dict
866 def _process_requested_xp_dict(self, rspec):
868 Turns the requested leases and information into a dictionary
869 of requested jobs, grouped by starting time.
871 :param rspec: RSpec received
876 requested_lease_list = self._get_requested_leases_list(rspec)
877 logger.debug("IOTLABDRIVER _process_requested_xp_dict \
878 requested_lease_list %s" % (requested_lease_list))
879 xp_dict = self._group_leases_by_start_time(requested_lease_list)
880 logger.debug("IOTLABDRIVER _process_requested_xp_dict xp_dict\
887 def delete(self, slice_urns, options={}):
889 Deletes the lease associated with the slice hrn and the credentials
890 if the slice belongs to iotlab. Answer to DeleteSliver.
892 :param slice_urn: urn of the slice
893 :type slice_urn: string
896 :returns: 1 if the slice to delete was not found on iotlab,
897 True if the deletion was successful, False otherwise otherwise.
899 .. note:: Should really be named delete_leases because iotlab does
900 not have any slivers, but only deals with leases. However,
901 SFA api only have delete_sliver define so far. SA 13/05/2013
902 .. note:: creds are unused, and are not used either in the dummy driver
905 # collect sliver ids so we can update sliver allocation states after
906 # we remove the slivers.
907 aggregate = IotlabAggregate(self)
908 slivers = aggregate.get_slivers(slice_urns)
910 # slice_id = slivers[0]['slice_id']
913 sliver_jobs_dict = {}
914 for sliver in slivers:
915 node_ids.append(sliver['node_id'])
916 sliver_ids.append(sliver['sliver_id'])
917 job_id = sliver['sliver_id'].split('+')[-1].split('-')[0]
918 sliver_jobs_dict[job_id] = sliver['sliver_id']
919 logger.debug("IOTLABDRIVER.PY delete_sliver slivers %s slice_urns %s"
920 % (slivers, slice_urns))
921 slice_hrn = urn_to_hrn(slice_urns[0])[0]
923 sfa_slice_list = self.GetSlices(slice_filter=slice_hrn,
924 slice_filter_type='slice_hrn')
926 if not sfa_slice_list:
929 #Delete all leases in the slice
930 for sfa_slice in sfa_slice_list:
931 logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice))
932 slices = IotlabSlices(self)
933 # determine if this is a peer slice
935 peer = slices.get_peer(slice_hrn)
937 logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \
938 \r\n \t sfa_slice %s " % (peer, sfa_slice))
939 oar_bool_ans = self.testbed_shell.DeleteSliceFromNodes(
941 for job_id in oar_bool_ans:
942 # if the job has not been successfully deleted
943 # don't delete the associated sliver
944 # remove it from the sliver list
945 if oar_bool_ans[job_id] is False:
946 sliver = sliver_jobs_dict[job_id]
947 sliver_ids.remove(sliver)
950 dbsession = self.api.dbsession()
951 SliverAllocation.delete_allocations(sliver_ids, dbsession)
953 logger.log_exc("IOTLABDRIVER.PY delete error ")
955 # prepare return struct
957 for sliver in slivers:
959 {'geni_sliver_urn': sliver['sliver_id'],
960 'geni_allocation_status': 'geni_unallocated',
961 'geni_expires': datetime_to_string(utcparse(sliver['expires']))})
964 # def list_resources (self, slice_urn, slice_hrn, creds, options):
967 # List resources from the iotlab aggregate and returns a Rspec
968 # advertisement with resources found when slice_urn and slice_hrn are
969 # None (in case of resource discovery).
970 # If a slice hrn and urn are provided, list experiment's slice
971 # nodes in a rspec format. Answer to ListResources.
974 # :param slice_urn: urn of the slice
975 # :param slice_hrn: name of the slice
976 # :param creds: slice credenials
977 # :type slice_urn: string
978 # :type slice_hrn: string
979 # :type creds: ? unused
980 # :param options: options used when listing resources (list_leases, info,
982 # :returns: rspec string in xml
985 # .. note:: creds are unused
988 # #cached_requested = options.get('cached', True)
990 # version_manager = VersionManager()
991 # # get the rspec's return format from options
993 # version_manager.get_version(options.get('geni_rspec_version'))
994 # version_string = "rspec_%s" % (rspec_version)
996 # #panos adding the info option to the caching key (can be improved)
997 # if options.get('info'):
998 # version_string = version_string + "_" + \
999 # options.get('info', 'default')
1001 # # Adding the list_leases option to the caching key
1002 # if options.get('list_leases'):
1003 # version_string = version_string + "_" + \
1004 # options.get('list_leases', 'default')
1006 # # Adding geni_available to caching key
1007 # if options.get('geni_available'):
1008 # version_string = version_string + "_" + \
1009 # str(options.get('geni_available'))
1011 # # look in cache first
1012 # #if cached_requested and self.cache and not slice_hrn:
1013 # #rspec = self.cache.get(version_string)
1015 # #logger.debug("IotlabDriver.ListResources: \
1016 # #returning cached advertisement")
1019 # #panos: passing user-defined options
1020 # aggregate = IotlabAggregate(self)
1022 # rspec = aggregate.get_rspec(slice_xrn=slice_urn,
1023 # version=rspec_version, options=options)
1025 # # cache the result
1026 # #if self.cache and not slice_hrn:
1027 # #logger.debug("Iotlab.ListResources: stores advertisement in cache")
1028 # #self.cache.add(version_string, rspec)
1033 def list_slices(self, creds, options):
1034 """Answer to ListSlices.
1036 List slices belonging to iotlab, returns slice urns list.
1037 No caching used. Options unused but are defined in the SFA method
1040 :returns: slice urns list
1043 .. note:: creds and options are unused - SA 12/12/13
1045 # look in cache first
1047 #slices = self.cache.get('slices')
1049 #logger.debug("PlDriver.list_slices returns from cache")
1054 slices = self.GetSlices()
1055 logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n"
1057 slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
1059 slice_urns = [hrn_to_urn(slice_hrn, 'slice')
1060 for slice_hrn in slice_hrns]
1064 #logger.debug ("IotlabDriver.list_slices stores value in cache")
1065 #self.cache.add('slices', slice_urns)
1070 def register(self, sfa_record, hrn, pub_key):
1072 Adding new user, slice, node or site should not be handled
1075 ..warnings:: should not be used. Different components are in charge of
1076 doing this task. Adding nodes = OAR
1077 Adding users = LDAP Iotlab
1078 Adding slice = Import from LDAP users
1081 :param sfa_record: record provided by the client of the
1083 :type sfa_record: dict
1084 :param pub_key: public key of the user
1085 :type pub_key: string
1087 .. note:: DOES NOTHING. Returns -1.
1093 def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
1095 No site or node record update allowed in Iotlab. The only modifications
1096 authorized here are key deletion/addition on an existing user and
1097 password change. On an existing user, CAN NOT BE MODIFIED: 'first_name',
1098 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio',
1099 'title', 'accepted_aup'. A slice is bound to its user, so modifying the
1100 user's ssh key should nmodify the slice's GID after an import procedure.
1102 :param old_sfa_record: what is in the db for this hrn
1103 :param new_sfa_record: what was passed to the update call
1104 :param new_key: the new user's public key
1105 :param hrn: the user's sfa hrn
1106 :type old_sfa_record: dict
1107 :type new_sfa_record: dict
1108 :type new_key: string
1112 .. warning:: SA 12/12/13 - Removed. should be done in iotlabimporter
1113 since users, keys and slice are managed by the LDAP.
1116 # pointer = old_sfa_record['pointer']
1117 # old_sfa_record_type = old_sfa_record['type']
1119 # # new_key implemented for users only
1120 # if new_key and old_sfa_record_type not in ['user']:
1121 # raise UnknownSfaType(old_sfa_record_type)
1123 # if old_sfa_record_type == "user":
1124 # update_fields = {}
1125 # all_fields = new_sfa_record
1126 # for key in all_fields.keys():
1127 # if key in ['key', 'password']:
1128 # update_fields[key] = all_fields[key]
1131 # # must check this key against the previous one if it exists
1132 # persons = self.testbed_shell.GetPersons([old_sfa_record])
1133 # person = persons[0]
1134 # keys = [person['pkey']]
1135 # #Get all the person's keys
1136 # keys_dict = self.GetKeys(keys)
1138 # # Delete all stale keys, meaning the user has only one key
1140 # #TODO: do we really want to delete all the other keys?
1141 # #Is this a problem with the GID generation to have multiple
1142 # #keys? SA 30/05/13
1143 # key_exists = False
1144 # if key in keys_dict:
1147 # #remove all the other keys
1148 # for key in keys_dict:
1149 # self.testbed_shell.DeleteKey(person, key)
1150 # self.testbed_shell.AddPersonKey(
1151 # person, {'sshPublicKey': person['pkey']},
1152 # {'sshPublicKey': new_key})
1153 logger.warning ("UNDEFINED - Update should be done by the \
1157 def remove(self, sfa_record):
1160 Removes users only. Mark the user as disabled in LDAP. The user and his
1161 slice are then deleted from the db by running an import on the registry.
1163 :param sfa_record: record is the existing sfa record in the db
1164 :type sfa_record: dict
1166 ..warning::As fas as the slice is concerned, here only the leases are
1167 removed from the slice. The slice is record itself is not removed
1172 TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013,
1174 TODO: return boolean for the slice part
1176 sfa_record_type = sfa_record['type']
1177 hrn = sfa_record['hrn']
1178 if sfa_record_type == 'user':
1180 #get user from iotlab ldap
1181 person = self.testbed_shell.GetPersons(sfa_record)
1182 #No registering at a given site in Iotlab.
1183 #Once registered to the LDAP, all iotlab sites are
1186 #Mark account as disabled in ldap
1187 return self.testbed_shell.DeletePerson(sfa_record)
1189 elif sfa_record_type == 'slice':
1190 if self.GetSlices(slice_filter=hrn,
1191 slice_filter_type='slice_hrn'):
1192 ret = self.testbed_shell.DeleteSlice(sfa_record)
1195 def check_sliver_credentials(self, creds, urns):
1196 """Check that the sliver urns belongs to the slice specified in the
1199 :param urns: list of sliver urns.
1201 :param creds: slice credentials.
1202 :type creds: Credential object.
1206 # build list of cred object hrns
1207 slice_cred_names = []
1209 slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn()
1210 slicename = IotlabXrn(xrn=slice_cred_hrn).iotlab_slicename()
1211 slice_cred_names.append(slicename)
1213 # look up slice name of slivers listed in urns arg
1217 sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts()
1219 slice_ids.append(int(sliver_id_parts[0]))
1224 raise Forbidden("sliver urn not provided")
1226 slices = self.GetSlices(slice_ids)
1227 sliver_names = [single_slice['name'] for single_slice in slices]
1229 # make sure we have a credential for every specified sliver
1230 for sliver_name in sliver_names:
1231 if sliver_name not in slice_cred_names:
1232 msg = "Valid credential not found for target: %s" % sliver_name
1233 raise Forbidden(msg)
1235 ########################################
1236 ########## aggregate oriented
1237 ########################################
1239 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
1240 def aggregate_version(self):
1243 Returns the testbed's supported rspec advertisement and request
1245 :returns: rspec versions supported ad a dictionary.
1249 version_manager = VersionManager()
1250 ad_rspec_versions = []
1251 request_rspec_versions = []
1252 for rspec_version in version_manager.versions:
1253 if rspec_version.content_type in ['*', 'ad']:
1254 ad_rspec_versions.append(rspec_version.to_dict())
1255 if rspec_version.content_type in ['*', 'request']:
1256 request_rspec_versions.append(rspec_version.to_dict())
1258 'testbed': self.testbed_name(),
1259 'geni_request_rspec_versions': request_rspec_versions,
1260 'geni_ad_rspec_versions': ad_rspec_versions}
1262 # first 2 args are None in case of resource discovery
1263 def list_resources (self, version=None, options={}):
1264 aggregate = IotlabAggregate(self)
1265 rspec = aggregate.list_resources(version=version, options=options)
1268 def describe(self, urns, version, options={}):
1269 aggregate = IotlabAggregate(self)
1270 return aggregate.describe(urns, version=version, options=options)
1272 def status (self, urns, options={}):
1273 aggregate = IotlabAggregate(self)
1274 desc = aggregate.describe(urns, version='GENI 3')
1275 status = {'geni_urn': desc['geni_urn'],
1276 'geni_slivers': desc['geni_slivers']}
1280 def allocate (self, urn, rspec_string, expiration, options={}):
1282 aggregate = IotlabAggregate(self)
1284 slices = IotlabSlices(self)
1285 peer = slices.get_peer(xrn.get_hrn())
1286 sfa_peer = slices.get_sfa_peer(xrn.get_hrn())
1290 users = options.get('geni_users', [])
1292 sfa_users = options.get('sfa_users', [])
1294 slice_record = sfa_users[0].get('slice_record', [])
1297 rspec = RSpec(rspec_string)
1298 # requested_attributes = rspec.version.get_slice_attributes()
1300 # ensure site record exists
1301 # site = slices.verify_site(xrn.hrn, slice_record, peer, sfa_peer, options=options)
1302 # ensure slice record exists
1304 current_slice = slices.verify_slice(xrn.hrn, slice_record, sfa_peer)
1305 logger.debug("IOTLABDRIVER.PY \t ===============allocate \t\
1306 \r\n \r\n current_slice %s" % (current_slice))
1307 # ensure person records exists
1309 # oui c'est degueulasse, le slice_record se retrouve modifie
1310 # dans la methode avec les infos du user, els infos sont propagees
1311 # dans verify_slice_leases
1312 persons = slices.verify_persons(xrn.hrn, slice_record, users,
1314 # ensure slice attributes exists
1315 # slices.verify_slice_attributes(slice, requested_attributes,
1318 # add/remove slice from nodes
1319 requested_xp_dict = self._process_requested_xp_dict(rspec)
1321 logger.debug("IOTLABDRIVER.PY \tallocate requested_xp_dict %s "
1322 % (requested_xp_dict))
1323 request_nodes = rspec.version.get_nodes_with_slivers()
1325 for start_time in requested_xp_dict:
1326 lease = requested_xp_dict[start_time]
1327 for hostname in lease['hostname']:
1328 nodes_list.append(hostname)
1330 # nodes = slices.verify_slice_nodes(slice_record,request_nodes, peer)
1331 logger.debug("IOTLABDRIVER.PY \tallocate nodes_list %s slice_record %s"
1332 % (nodes_list, slice_record))
1335 rspec_requested_leases = rspec.version.get_leases()
1336 leases = slices.verify_slice_leases(slice_record, requested_xp_dict, peer)
1337 logger.debug("IOTLABDRIVER.PY \tallocate leases %s \
1338 rspec_requested_leases %s" % (leases,
1339 rspec_requested_leases))
1340 # update sliver allocations
1341 for hostname in nodes_list:
1342 client_id = hostname
1343 node_urn = xrn_object(self.testbed_shell.root_auth, hostname).urn
1344 component_id = node_urn
1345 slice_urn = current_slice['reg-urn']
1346 for lease in leases:
1347 if hostname in lease['reserved_nodes']:
1348 index = lease['reserved_nodes'].index(hostname)
1349 sliver_hrn = '%s.%s-%s' % (self.hrn, lease['lease_id'],
1350 lease['resource_ids'][index] )
1351 sliver_id = Xrn(sliver_hrn, type='sliver').urn
1352 record = SliverAllocation(sliver_id=sliver_id, client_id=client_id,
1353 component_id=component_id,
1354 slice_urn = slice_urn,
1355 allocation_state='geni_allocated')
1356 record.sync(self.api.dbsession())
1358 return aggregate.describe([xrn.get_urn()], version=rspec.version)
1360 def provision(self, urns, options={}):
1362 slices = IotlabSlices(self)
1363 aggregate = IotlabAggregate(self)
1364 slivers = aggregate.get_slivers(urns)
1365 current_slice = slivers[0]
1366 peer = slices.get_peer(current_slice['hrn'])
1367 sfa_peer = slices.get_sfa_peer(current_slice['hrn'])
1368 users = options.get('geni_users', [])
1369 # persons = slices.verify_persons(current_slice['hrn'],
1370 # current_slice, users, peer, sfa_peer, options=options)
1371 # slices.handle_peer(None, None, persons, peer)
1372 # update sliver allocation states and set them to geni_provisioned
1373 sliver_ids = [sliver['sliver_id'] for sliver in slivers]
1374 dbsession = self.api.dbsession()
1375 SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',
1377 version_manager = VersionManager()
1378 rspec_version = version_manager.get_version(options[
1379 'geni_rspec_version'])
1380 return self.describe(urns, rspec_version, options=options)