2 File providing methods to generate valid RSpecs for the Iotlab testbed.
3 Contains methods to get information on slice, slivers, nodes and leases,
4 formatting them and turn it into a RSpec.
6 from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority
8 from sfa.rspecs.rspec import RSpec
9 #from sfa.rspecs.elements.location import Location
10 from sfa.rspecs.elements.hardware_type import HardwareType
11 from sfa.rspecs.elements.login import Login
12 from sfa.rspecs.elements.services import ServicesElement
13 from sfa.rspecs.elements.sliver import Sliver
14 from sfa.rspecs.elements.lease import Lease
15 from sfa.rspecs.elements.granularity import Granularity
16 from sfa.rspecs.version_manager import VersionManager
18 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, \
19 IotlabNode, IotlabLocation
21 from sfa.util.sfalogging import logger
22 from sfa.util.xrn import Xrn
28 class IotlabAggregate:
29 """Aggregate manager class for Iotlab. """
42 def __init__(self, driver):
45 def get_slice_and_slivers(self, slice_xrn, login=None):
47 Get the slices and the associated leases if any from the iotlab
48 testbed. One slice can have mutliple leases.
49 For each slice, get the nodes in the associated lease
50 and create a sliver with the necessary info and insert it into the
51 sliver dictionary, keyed on the node hostnames.
52 Returns a dict of slivers based on the sliver's node_id.
56 :param slice_xrn: xrn of the slice
57 :param login: user's login on iotlab ldap
59 :type slice_xrn: string
61 :returns: a list of slices dict and a list of Sliver object
64 .. note: There is no real slivers in iotlab, only leases. The goal
65 is to be consistent with the SFA standard.
71 return (sfa_slice, slivers)
72 slice_urn = hrn_to_urn(slice_xrn, 'slice')
73 slice_hrn, _ = urn_to_hrn(slice_xrn)
74 slice_name = slice_hrn
76 # GetSlices always returns a list, even if there is only one element
77 slices = self.driver.testbed_shell.GetSlices(slice_filter=str(slice_name),
78 slice_filter_type='slice_hrn',
81 logger.debug("IotlabAggregate api \tget_slice_and_slivers \
82 slice_hrn %s \r\n slices %s self.driver.hrn %s"
83 % (slice_hrn, slices, self.driver.hrn))
85 return (sfa_slice, slivers)
87 # sort slivers by node id , if there is a job
88 #and therefore, node allocated to this slice
89 # for sfa_slice in slices:
92 node_ids_list = sfa_slice['node_ids']
94 logger.log_exc("IOTLABAGGREGATE \t \
95 get_slice_and_slivers No nodes in the slice \
100 for node in node_ids_list:
101 sliver_xrn = Xrn(slice_urn, type='sliver', id=node)
102 sliver_xrn.set_authority(self.driver.hrn)
103 sliver = Sliver({'sliver_id': sliver_xrn.urn,
104 'name': sfa_slice['hrn'],
105 'type': 'iotlab-node',
108 slivers[node] = sliver
110 #Add default sliver attribute :
111 #connection information for iotlab
112 # if get_authority(sfa_slice['hrn']) == self.driver.testbed_shell.root_auth:
113 # tmp = sfa_slice['hrn'].split('.')
114 # ldap_username = tmp[1].split('_')[0]
116 # slivers['default_sliver'] = {'ssh': ssh_access,
117 # 'login': ldap_username}
119 ldap_username = self.find_ldap_username_from_slice(sfa_slice)
121 if ldap_username is not None:
123 slivers['default_sliver'] = {'ssh': ssh_access,
124 'login': ldap_username}
127 logger.debug("IOTLABAGGREGATE api get_slice_and_slivers slivers %s "
129 return (slices, slivers)
131 def find_ldap_username_from_slice(self, sfa_slice):
132 researchers = [sfa_slice['reg_researchers'][0].__dict__]
135 ret = self.driver.testbed_shell.GetPersons(researchers)
137 ldap_username = ret[0]['uid']
143 def get_nodes(self, slices=None, slivers=[], options=None):
144 # def node_to_rspec_node(self, node, sites, node_tags,
145 # grain=None, options={}):
146 """Returns the nodes in the slice using the rspec format, with all the
149 Fetch the nodes ids in the slices dictionary and get all the nodes
150 properties from OAR. Makes a rspec dicitonary out of this and returns
151 it. If the slice does not have any job running or scheduled, that is
152 it has no reserved nodes, then returns an empty list.
154 :param slices: list of slices (record dictionaries)
155 :param slivers: the list of slivers in all the slices
156 :type slices: list of dicts
157 :type slivers: list of Sliver object (dictionaries)
158 :returns: An empty list if the slice has no reserved nodes, a rspec
159 list with all the nodes and their properties (a dict per node)
163 .. seealso:: get_slice_and_slivers
167 # NT: the semantic of this function is not clear to me :
168 # if slice is not defined, then all the nodes should be returned
169 # if slice is defined, we should return only the nodes that
170 # are part of this slice
171 # but what is the role of the slivers parameter ?
172 # So i assume that slice['node_ids'] will be the same as slivers for us
175 geni_available = options.get('geni_available')
176 if geni_available == True:
177 filter_nodes['boot_state'] = ['Alive']
179 # slice_nodes_list = []
180 # if slices is not None:
181 # for one_slice in slices:
183 # slice_nodes_list = one_slice['node_ids']
184 # # if we are dealing with a slice that has no node just
185 # # return an empty list. In iotlab a slice can have multiple
186 # # jobs scheduled, so it either has at least one lease or
191 # get the granularity in second for the reservation system
192 # grain = self.driver.testbed_shell.GetLeaseGranularity()
194 nodes = self.driver.testbed_shell.GetNodes()
198 #if slices, this means we got to list all the nodes given to this slice
199 # Make a list of all the nodes in the slice before getting their
203 # logger.debug("IOTLABAGGREGATE api get_nodes slices %s "
206 # reserved_nodes = self.driver.testbed_shell.GetNodesCurrentlyInUse()
207 # logger.debug("IOTLABAGGREGATE api get_nodes slice_nodes_list %s "
208 # % (slice_nodes_list))
210 nodes_dict[node['node_id']] = node
211 # if slice_nodes_list == [] or node['hostname'] in slice_nodes_list:
213 # rspec_node = IotlabNode()
214 # # xxx how to retrieve site['login_base']
215 # #site_id=node['site_id']
216 # #site=sites_dict[site_id]
218 # rspec_node['mobile'] = node['mobile']
219 # rspec_node['archi'] = node['archi']
220 # rspec_node['radio'] = node['radio']
222 # iotlab_xrn = xrn_object(self.driver.testbed_shell.root_auth,
224 # rspec_node['component_id'] = iotlab_xrn.urn
225 # rspec_node['component_name'] = node['hostname']
226 # rspec_node['component_manager_id'] = \
227 # hrn_to_urn(self.driver.testbed_shell.root_auth,
230 # # Iotlab's nodes are federated : there is only one authority
231 # # for all Iotlab sites, registered in SFA.
232 # # Removing the part including the site
233 # # in authority_id SA 27/07/12
234 # rspec_node['authority_id'] = rspec_node['component_manager_id']
236 # # do not include boot state (<available> element)
237 # #in the manifest rspec
240 # rspec_node['boot_state'] = node['boot_state']
241 # if node['hostname'] in reserved_nodes:
242 # rspec_node['boot_state'] = "Reserved"
243 # rspec_node['exclusive'] = 'true'
244 # rspec_node['hardware_types'] = [HardwareType({'name': \
248 # location = IotlabLocation({'country':'France', 'site': \
250 # rspec_node['location'] = location
253 # position = IotlabPosition()
254 # for field in position :
256 # position[field] = node[field]
257 # except KeyError, error :
258 # logger.log_exc("IOTLABAGGREGATE\t get_nodes \
259 # position %s "% (error))
261 # rspec_node['position'] = position
262 # #rspec_node['interfaces'] = []
265 # granularity = Granularity({'grain': grain})
266 # rspec_node['granularity'] = granularity
267 # rspec_node['tags'] = []
268 # if node['hostname'] in slivers:
270 # sliver = slivers[node['hostname']]
271 # rspec_node['sliver_id'] = sliver['sliver_id']
272 # rspec_node['client_id'] = node['hostname']
273 # rspec_node['slivers'] = [sliver]
275 # # slivers always provide the ssh service
276 # login = Login({'authentication': 'ssh-keys', \
277 # 'hostname': node['hostname'], 'port':'22', \
278 # 'username': sliver['name']})
279 # service = Services({'login': login})
280 # rspec_node['services'] = [service]
281 # rspec_nodes.append(rspec_node)
283 # return (rspec_nodes)
286 def node_to_rspec_node(self, node, grain):
287 rspec_node = IotlabNode()
288 # xxx how to retrieve site['login_base']
289 #site_id=node['site_id']
290 #site=sites_dict[site_id]
292 rspec_node['mobile'] = node['mobile']
293 rspec_node['archi'] = node['archi']
294 rspec_node['radio'] = node['radio']
296 iotlab_xrn = xrn_object(self.driver.testbed_shell.root_auth,
298 rspec_node['component_id'] = iotlab_xrn.urn
299 rspec_node['component_name'] = node['hostname']
300 rspec_node['component_manager_id'] = \
301 hrn_to_urn(self.driver.testbed_shell.root_auth,
304 # Iotlab's nodes are federated : there is only one authority
305 # for all Iotlab sites, registered in SFA.
306 # Removing the part including the site
307 # in authority_id SA 27/07/12
308 rspec_node['authority_id'] = rspec_node['component_manager_id']
310 # do not include boot state (<available> element)
311 #in the manifest rspec
314 rspec_node['boot_state'] = node['boot_state']
315 # if node['hostname'] in reserved_nodes:
316 # rspec_node['boot_state'] = "Reserved"
317 rspec_node['exclusive'] = 'true'
318 rspec_node['hardware_types'] = [HardwareType({'name': \
321 location = IotlabLocation({'country':'France', 'site': \
323 rspec_node['location'] = location
326 position = IotlabPosition()
327 for field in position :
329 position[field] = node[field]
330 except KeyError, error :
331 logger.log_exc("IOTLABAGGREGATE\t get_nodes \
332 position %s "% (error))
334 rspec_node['position'] = position
338 granularity = Granularity({'grain': grain})
339 rspec_node['granularity'] = granularity
340 rspec_node['tags'] = []
341 # if node['hostname'] in slivers:
343 # sliver = slivers[node['hostname']]
344 # rspec_node['sliver_id'] = sliver['sliver_id']
345 # rspec_node['client_id'] = node['hostname']
346 # rspec_node['slivers'] = [sliver]
348 # # slivers always provide the ssh service
349 # login = Login({'authentication': 'ssh-keys', \
350 # 'hostname': node['hostname'], 'port':'22', \
351 # 'username': sliver['name']})
352 # service = Services({'login': login})
353 # rspec_node['services'] = [service]
357 def get_all_leases(self, ldap_username):
360 Get list of lease dictionaries which all have the mandatory keys
361 ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration').
362 All the leases running or scheduled are returned.
364 :param ldap_username: if ldap uid is not None, looks for the leases
365 belonging to this user.
366 :type ldap_username: string
367 :returns: rspec lease dictionary with keys lease_id, component_id,
368 slice_id, start_time, duration.
371 .. note::There is no filtering of leases within a given time frame.
372 All the running or scheduled leases are returned. options
373 removed SA 15/05/2013
378 #now = int(time.time())
379 #lease_filter = {'clip': now }
382 #lease_filter.update({'name': slice_record['name']})
384 #leases = self.driver.testbed_shell.GetLeases(lease_filter)
386 logger.debug("IOTLABAGGREGATE get_all_leases ldap_username %s "
388 leases = self.driver.testbed_shell.GetLeases(login=ldap_username)
389 grain = self.driver.testbed_shell.GetLeaseGranularity()
393 #as many leases as there are nodes in the job
394 for node in lease['reserved_nodes']:
395 rspec_lease = Lease()
396 rspec_lease['lease_id'] = lease['lease_id']
397 #site = node['site_id']
398 iotlab_xrn = xrn_object(self.driver.testbed_shell.root_auth,
400 rspec_lease['component_id'] = iotlab_xrn.urn
401 #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\
402 #site, node['hostname'])
404 rspec_lease['slice_id'] = lease['slice_id']
406 #No info on the slice used in testbed_xp table
408 rspec_lease['start_time'] = lease['t_from']
409 rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \
411 rspec_leases.append(rspec_lease)
414 def get_rspec(self, slice_xrn=None, login=None, version=None,
418 - a full advertisement rspec with the testbed resources if slice_xrn is
419 not specified.If a lease option is given, also returns the leases
420 scheduled on the testbed.
421 - a manifest Rspec with the leases and nodes in slice's leases if
422 slice_xrn is not None.
424 :param slice_xrn: srn of the slice
425 :type slice_xrn: string
426 :param login: user'uid (ldap login) on iotlab
428 :param version: can be set to sfa or iotlab
429 :type version: RSpecVersion
430 :param options: used to specify if the leases should also be included in
442 version_manager = VersionManager()
443 version = version_manager.get_version(version)
444 logger.debug("IotlabAggregate \t get_rspec ***version %s \
445 version.type %s version.version %s options %s \r\n"
446 % (version, version.type, version.version, options))
448 if slice_xrn is None:
449 rspec_version = version_manager._get_version(version.type,
450 version.version, 'ad')
453 rspec_version = version_manager._get_version(
454 version.type, version.version, 'manifest')
456 slices, slivers = self.get_slice_and_slivers(slice_xrn, login)
457 if slice_xrn and slices is not None:
458 #Get user associated with this slice
459 #for one_slice in slices :
460 ldap_username = self.find_ldap_username_from_slice(slices[0])
461 # ldap_username = slices[0]['reg_researchers'][0].__dict__['hrn']
462 # # ldap_username = slices[0]['user']
463 # tmp = ldap_username.split('.')
464 # ldap_username = tmp[1]
465 logger.debug("IotlabAggregate \tget_rspec **** \
466 LDAP USERNAME %s \r\n" \
468 #at this point sliver may be empty if no iotlab job
469 #is running for this user/slice.
470 rspec = RSpec(version=rspec_version, user_options=options)
472 logger.debug("\r\n \r\n IotlabAggregate \tget_rspec *** \
473 slice_xrn %s slices %s\r\n \r\n"
474 % (slice_xrn, slices))
476 if options is not None:
477 lease_option = options['list_leases']
479 #If no options are specified, at least print the resources
482 #lease_option = 'all'
484 if lease_option in ['all', 'resources']:
485 #if not options.get('list_leases') or options.get('list_leases')
486 #and options['list_leases'] != 'leases':
487 nodes = self.get_nodes(slices, slivers)
489 logger.debug("IotlabAggregate \t lease_option %s \
490 get rspec ******* nodes %s"
491 % (lease_option, nodes))
493 sites_set = set([node['location']['site'] for node in nodes])
495 #In case creating a job, slice_xrn is not set to None
496 rspec.version.add_nodes(nodes)
497 if slice_xrn and slices is not None:
498 # #Get user associated with this slice
499 # #for one_slice in slices :
500 # ldap_username = slices[0]['reg_researchers']
501 # # ldap_username = slices[0]['user']
502 # tmp = ldap_username.split('.')
503 # ldap_username = tmp[1]
504 # # ldap_username = tmp[1].split('_')[0]
506 logger.debug("IotlabAggregate \tget_rspec **** \
507 version type %s ldap_ user %s \r\n" \
508 % (version.type, ldap_username))
509 if version.type == "Iotlab":
510 rspec.version.add_connection_information(
511 ldap_username, sites_set)
513 default_sliver = slivers.get('default_sliver', [])
514 if default_sliver and len(nodes) is not 0:
515 #default_sliver_attribs = default_sliver.get('tags', [])
516 logger.debug("IotlabAggregate \tget_rspec **** \
517 default_sliver%s \r\n" % (default_sliver))
518 for attrib in default_sliver:
519 rspec.version.add_default_sliver_attribute(
520 attrib, default_sliver[attrib])
522 if lease_option in ['all','leases']:
523 leases = self.get_all_leases(ldap_username)
524 rspec.version.add_leases(leases)
525 logger.debug("IotlabAggregate \tget_rspec **** \
526 FINAL RSPEC %s \r\n" % (rspec.toxml()))
530 def list_resources(self, version = None, options={}):
532 version_manager = VersionManager()
533 version = version_manager.get_version(version)
534 rspec_version = version_manager._get_version(version.type, version.version, 'ad')
535 rspec = RSpec(version=rspec_version, user_options=options)
537 if not options.get('list_leases') or options['list_leases'] != 'leases':
539 nodes_dict = self.get_nodes(options)
545 # site_ids.append(node['site_id'])
546 # interface_ids.extend(node['interface_ids'])
547 # tag_ids.extend(node['node_tag_ids'])
548 # nodes_dict[node['node_id']] = node
549 # sites = self.get_sites({'site_id': site_ids})
550 # interfaces = self.get_interfaces({'interface_id':interface_ids})
551 # node_tags = self.get_node_tags({'node_tag_id': tag_ids})
552 # pl_initscripts = self.get_pl_initscripts()
553 # convert nodes to rspec nodes
554 grain = self.driver.testbed_shell.GetLeaseGranularity()
556 for node_id in nodes_dict:
557 node = nodes_dict[node_id]
558 # rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
559 rspec_node = self.node_to_rspec_node(node, grain)
560 rspec_nodes.append(rspec_node)
561 rspec.version.add_nodes(rspec_nodes)
564 # links = self.get_links(sites, nodes_dict, interfaces)
565 # rspec.version.add_links(links)
567 if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
568 leases = self.get_all_leases()
569 rspec.version.add_leases(leases)
574 def describe(self, urns, version=None, options={}):
575 version_manager = VersionManager()
576 version = version_manager.get_version(version)
577 rspec_version = version_manager._get_version(version.type,
578 version.version, 'manifest')
579 rspec = RSpec(version=rspec_version, user_options=options)
583 slivers = self.get_slivers(urns, options)
585 rspec_expires = datetime_to_string(utcparse(slivers[0]['expires']))
587 rspec_expires = datetime_to_string(utcparse(time.time()))
588 rspec.xml.set('expires', rspec_expires)
590 # lookup the sliver allocations
592 sliver_ids = [sliver['sliver_id'] for sliver in slivers]
593 constraint = SliverAllocation.sliver_id.in_(sliver_ids)
594 sliver_allocations = self.driver.api.dbsession().query(SliverAllocation).filter(constraint)
595 sliver_allocation_dict = {}
596 for sliver_allocation in sliver_allocations:
597 geni_urn = sliver_allocation.slice_urn
598 sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
602 for sliver in slivers:
603 nodes_dict[sliver['node_id']] = sliver
605 for sliver in slivers:
606 rspec_node = self.sliver_to_rspec_node(sliver, sliver_allocation_dict)
607 rspec_nodes.append(rspec_node)
608 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict)
609 geni_slivers.append(geni_sliver)
610 rspec.version.add_nodes(rspec_nodes)
612 return {'geni_urn': geni_urn,
613 'geni_rspec': rspec.toxml(),
614 'geni_slivers': geni_slivers}