2 from collections import defaultdict
3 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, get_authority, get_leaf
4 from sfa.util.sfatime import utcparse, datetime_to_string
5 from sfa.util.sfalogging import logger
6 from sfa.util.faults import SliverDoesNotExist
7 from sfa.rspecs.rspec import RSpec
8 from sfa.rspecs.elements.hardware_type import HardwareType
9 from sfa.rspecs.elements.node import NodeElement
10 from sfa.rspecs.elements.link import Link
11 from sfa.rspecs.elements.sliver import Sliver
12 from sfa.rspecs.elements.login import Login
13 from sfa.rspecs.elements.location import Location
14 from sfa.rspecs.elements.interface import Interface
15 from sfa.rspecs.elements.services import ServicesElement
16 from sfa.rspecs.elements.pltag import PLTag
17 from sfa.rspecs.elements.lease import Lease
18 from sfa.rspecs.elements.granularity import Granularity
19 from sfa.rspecs.version_manager import VersionManager
21 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn
22 from sfa.planetlab.vlink import get_tc_rate
23 from sfa.planetlab.topology import Topology
24 from sfa.storage.model import SliverAllocation
31 def __init__(self, driver):
34 def get_nodes(self, options=None):
35 if options is None: options={}
36 filter = {'peer_id': None}
37 geni_available = options.get('geni_available')
38 if geni_available == True:
39 filter['boot_state'] = 'boot'
40 nodes = self.driver.shell.GetNodes(filter)
44 def get_sites(self, filter=None):
45 if filter is None: filter={}
47 for site in self.driver.shell.GetSites(filter):
48 sites[site['site_id']] = site
51 def get_interfaces(self, filter=None):
52 if filter is None: filter={}
54 for interface in self.driver.shell.GetInterfaces(filter):
56 if interface['bwlimit']:
57 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
58 interfaces[interface['interface_id']] = interface
61 def get_links(self, sites, nodes, interfaces):
65 for (site_id1, site_id2) in topology:
66 site_id1 = int(site_id1)
67 site_id2 = int(site_id2)
69 if not site_id1 in sites or site_id2 not in sites:
71 site1 = sites[site_id1]
72 site2 = sites[site_id2]
74 site1_hrn = self.driver.hrn + '.' + site1['login_base']
75 site2_hrn = self.driver.hrn + '.' + site2['login_base']
77 for s1_node_id in site1['node_ids']:
78 for s2_node_id in site2['node_ids']:
79 if s1_node_id not in nodes or s2_node_id not in nodes:
81 node1 = nodes[s1_node_id]
82 node2 = nodes[s2_node_id]
84 # just get first interface of the first node
85 if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
86 if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
87 if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
88 if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
90 if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
91 if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
94 link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
95 link['interface1'] = if1
96 link['interface2'] = if2
97 link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
98 link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
99 link['component_manager_id'] = hrn_to_urn(self.driver.hrn, 'authority+am')
104 def get_node_tags(self, filter=None):
105 if filter is None: filter={}
107 for node_tag in self.driver.shell.GetNodeTags(filter, ['tagname', 'value', 'node_id', 'node_tag_id'] ):
108 node_tags[node_tag['node_tag_id']] = node_tag
111 def get_pl_initscripts(self, filter=None):
112 if filter is None: filter={}
114 filter.update({'enabled': True})
115 for initscript in self.driver.shell.GetInitScripts(filter):
116 pl_initscripts[initscript['initscript_id']] = initscript
117 return pl_initscripts
119 def get_slivers(self, urns, options=None):
120 if options is None: options={}
127 if xrn.type == 'sliver':
128 # id: slice_id-node_id
130 sliver_id_parts = xrn.get_sliver_id_parts()
131 slice_id = int(sliver_id_parts[0])
132 node_id = int(sliver_id_parts[1])
133 slice_ids.add(slice_id)
134 node_ids.append(node_id)
138 slice_hrn = xrn.get_hrn()
141 filter['peer_id'] = None
143 filter['slice_id'] = list(slice_ids)
145 fields = ['slice_id', 'name', 'hrn', 'person_ids', 'node_ids', 'slice_tag_ids', 'expires']
146 all_slices = self.driver.shell.GetSlices(filter, fields)
148 slices = [slice for slice in all_slices if slice['hrn'] == slice_hrn]
154 logger.error("PlAggregate.get_slivers : no slice found with hrn {}".format(slice_hrn))
156 logger.error("PlAggregate.get_slivers : no sliver found with urns {}".format(urns))
159 slice['hrn'] = slice_hrn
165 person_ids.extend(slice['person_ids'])
167 persons = self.driver.shell.GetPersons(person_ids)
172 for person in persons:
173 key_ids.extend(person['key_ids'])
176 key_list = self.driver.shell.GetKeys(key_ids)
178 keys[key['key_id']] = key
180 # construct user key info
182 for person in persons:
183 person_urn = hrn_to_urn(self.driver.shell.GetPersonHrn(int(person['person_id'])), 'user')
185 'login': slice['name'],
186 'user_urn': person_urn,
187 'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys]
192 node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']]
193 slice['node_ids'] = node_ids
194 pltags_dict = self.get_pltags_by_node_id(slice)
195 nodes_dict = self.get_slice_nodes(slice, options)
197 for node in nodes_dict.values():
200 node['slice-tags'] = pltags_dict['slice-global']
202 # this is where we chould maybe add the nodegroup slice tags,
203 # but it's tedious...
206 node['slice-tags'] += pltags_dict[node['node_id']]
207 sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
208 node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
209 node['urn'] = node['sliver_id']
210 node['services_user'] = users
213 logger.warning("PlAggregate.get_slivers : slice(s) found but with no sliver {}".format(urns))
216 def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=None, grain=None, options=None):
217 if pl_initscripts is None: pl_initscripts=[]
218 if options is None: options={}
219 rspec_node = NodeElement()
220 # xxx how to retrieve site['login_base']
221 site=sites[node['site_id']]
222 rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
223 rspec_node['component_name'] = node['hostname']
224 rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
225 rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
226 # do not include boot state (<available> element) in the manifest rspec
227 rspec_node['boot_state'] = node['boot_state']
228 if node['boot_state'] == 'boot':
229 rspec_node['available'] = 'true'
231 rspec_node['available'] = 'false'
233 #distinguish between Shared and Reservable nodes
234 if node['node_type'] == 'reservable':
235 rspec_node['exclusive'] = 'true'
237 rspec_node['exclusive'] = 'false'
239 rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
240 HardwareType({'name': 'pc'})]
241 # only doing this because protogeni rspec needs
242 # to advertise available initscripts
243 rspec_node['pl_initscripts'] = pl_initscripts.values()
244 # add site/interface info to nodes.
245 # assumes that sites, interfaces and tags have already been prepared.
246 if site['longitude'] and site['latitude']:
247 location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
248 rspec_node['location'] = location
250 granularity = Granularity({'grain': grain})
251 rspec_node['granularity'] = granularity
252 rspec_node['interfaces'] = []
254 for if_id in node['interface_ids']:
255 interface = Interface(interfaces[if_id])
256 interface['ipv4'] = interface['ip']
257 interface['component_id'] = PlXrn(auth=self.driver.hrn,
258 interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
259 # interfaces in the manifest need a client id
261 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
262 rspec_node['interfaces'].append(interface)
264 # this is what describes a particular node
265 node_level_tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags]
266 rspec_node['tags'] = node_level_tags
269 def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, sliver_pltags, \
270 pl_initscripts, sliver_allocations):
271 # get the granularity in second for the reservation system
272 grain = self.driver.shell.GetLeaseGranularity()
273 rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
274 for pltag in sliver_pltags:
275 logger.debug("Need to expose {}".format(pltag))
276 # xxx how to retrieve site['login_base']
277 rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
278 # remove interfaces from manifest
279 rspec_node['interfaces'] = []
281 rspec_sliver = Sliver({'sliver_id': sliver['urn'],
282 'name': sliver['name'],
283 'type': 'plab-vserver',
284 'tags': sliver_pltags,
286 rspec_node['sliver_id'] = rspec_sliver['sliver_id']
287 if sliver['urn'] in sliver_allocations:
288 rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id
289 if sliver_allocations[sliver['urn']].component_id:
290 rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id
291 rspec_node['slivers'] = [rspec_sliver]
293 # slivers always provide the ssh service
294 login = Login({'authentication': 'ssh-keys',
295 'hostname': sliver['hostname'],
297 'username': sliver['name'],
298 'login': sliver['name']
300 service = ServicesElement({'login': login,
301 'services_user': sliver['services_user']})
302 rspec_node['services'] = [service]
305 def get_pltags_by_node_id(self, slice):
307 slice_tag_ids.extend(slice['slice_tag_ids'])
308 tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids},
309 ['tagname', 'value', 'node_id', 'nodegroup_id'])
311 pltags_dict = defaultdict(list)
315 tag['scope'] = 'sliver'
316 pltags_dict[tag['node_id']].append(PLTag(tag))
317 # restricted to a nodegroup
318 # for now such tags are not exposed to describe
319 # xxx we should also expose the nodegroup name in this case to be complete..
320 elif tag['nodegroup_id']:
321 tag['scope'] = 'nodegroup'
322 pltags_dict['nodegroup'].append(PLTag(tag))
323 # this tag is global to the slice
325 tag['scope'] = 'slice'
326 pltags_dict['slice-global'].append(PLTag(tag))
329 def get_slice_nodes(self, slice, options=None):
330 if options is None: options={}
332 filter = {'peer_id': None}
334 if slice and slice.get('node_ids'):
335 filter['node_id'] = slice['node_ids']
337 # there are no nodes to look up
339 tags_filter=filter.copy()
340 geni_available = options.get('geni_available')
341 if geni_available == True:
342 filter['boot_state'] = 'boot'
343 nodes = self.driver.shell.GetNodes(filter)
345 nodes_dict[node['node_id']] = node
348 def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations=None):
349 if sliver_allocations is None: sliver_allocations={}
350 if rspec_node['sliver_id'] in sliver_allocations:
351 # set sliver allocation and operational status
352 sliver_allocation = sliver_allocations[rspec_node['sliver_id']]
353 if sliver_allocation:
354 allocation_status = sliver_allocation.allocation_state
355 if allocation_status == 'geni_allocated':
356 op_status = 'geni_pending_allocation'
357 elif allocation_status == 'geni_provisioned':
358 if rspec_node['boot_state'] == 'boot':
359 op_status = 'geni_ready'
361 op_status = 'geni_failed'
363 op_status = 'geni_unknown'
365 allocation_status = 'geni_unallocated'
367 allocation_status = 'geni_unallocated'
368 op_status = 'geni_failed'
370 geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
371 'geni_expires': rspec_node['expires'],
372 'geni_allocation_status' : allocation_status,
373 'geni_operational_status': op_status,
378 def get_leases(self, slice=None, options=None):
379 if options is None: options={}
381 now = int(time.time())
383 filter.update({'clip':now})
385 filter.update({'name':slice['name']})
386 return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
387 leases = self.driver.shell.GetLeases(filter)
388 grain = self.driver.shell.GetLeaseGranularity()
392 site_ids.append(lease['site_id'])
395 sites_dict = self.get_sites({'site_id': site_ids})
400 rspec_lease = Lease()
402 # xxx how to retrieve site['login_base']
403 site_id=lease['site_id']
404 site=sites_dict[site_id]
406 rspec_lease['component_id'] = hrn_to_urn(self.driver.shell.GetNodeHrn(lease['hostname']), 'node')
407 slice_hrn = self.driver.shell.GetSliceHrn(lease['slice_id'])
408 slice_urn = hrn_to_urn(slice_hrn, 'slice')
409 rspec_lease['slice_id'] = slice_urn
410 rspec_lease['start_time'] = lease['t_from']
411 rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
412 rspec_leases.append(rspec_lease)
416 def list_resources(self, version = None, options=None):
417 if options is None: options={}
419 version_manager = VersionManager()
420 version = version_manager.get_version(version)
421 rspec_version = version_manager._get_version(version.type, version.version, 'ad')
422 rspec = RSpec(version=rspec_version, user_options=options)
424 if not options.get('list_leases') or options['list_leases'] != 'leases':
426 nodes = self.get_nodes(options)
432 site_ids.append(node['site_id'])
433 interface_ids.extend(node['interface_ids'])
434 tag_ids.extend(node['node_tag_ids'])
435 nodes_dict[node['node_id']] = node
436 sites = self.get_sites({'site_id': site_ids})
437 interfaces = self.get_interfaces({'interface_id':interface_ids})
438 node_tags = self.get_node_tags({'node_tag_id': tag_ids})
439 pl_initscripts = self.get_pl_initscripts()
440 # convert nodes to rspec nodes
443 rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
444 rspec_nodes.append(rspec_node)
445 rspec.version.add_nodes(rspec_nodes)
448 links = self.get_links(sites, nodes_dict, interfaces)
449 rspec.version.add_links(links)
451 if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
452 leases = self.get_leases()
453 rspec.version.add_leases(leases)
457 def describe(self, urns, version=None, options=None):
458 if options is None: options={}
459 version_manager = VersionManager()
460 version = version_manager.get_version(version)
461 rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
462 rspec = RSpec(version=rspec_version, user_options=options)
466 slivers = self.get_slivers(urns, options)
468 rspec_expires = datetime_to_string(utcparse(slivers[0]['expires']))
470 rspec_expires = datetime_to_string(utcparse(time.time()))
471 rspec.xml.set('expires', rspec_expires)
473 # lookup the sliver allocations
475 sliver_ids = [sliver['sliver_id'] for sliver in slivers]
476 constraint = SliverAllocation.sliver_id.in_(sliver_ids)
477 sliver_allocations = self.driver.api.dbsession().query(SliverAllocation).filter(constraint)
478 sliver_allocation_dict = {}
479 for sliver_allocation in sliver_allocations:
480 geni_urn = sliver_allocation.slice_urn
481 sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
483 if not options.get('list_leases') or options['list_leases'] != 'leases':
489 for sliver in slivers:
490 site_ids.append(sliver['site_id'])
491 interface_ids.extend(sliver['interface_ids'])
492 tag_ids.extend(sliver['node_tag_ids'])
493 nodes_dict[sliver['node_id']] = sliver
494 sites = self.get_sites({'site_id': site_ids})
495 interfaces = self.get_interfaces({'interface_id':interface_ids})
496 node_tags = self.get_node_tags({'node_tag_id': tag_ids})
497 pl_initscripts = self.get_pl_initscripts()
499 for sliver in slivers:
500 if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
502 sliver_pltags = sliver['slice-tags']
503 rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, sliver_pltags,
504 pl_initscripts, sliver_allocation_dict)
505 logger.debug('rspec of type {}'.format(rspec_node.__class__.__name__))
506 # manifest node element shouldn't contain available attribute
507 rspec_node.pop('available')
508 rspec_nodes.append(rspec_node)
509 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict)
510 geni_slivers.append(geni_sliver)
511 rspec.version.add_nodes(rspec_nodes)
513 # add sliver defaults
514 #default_sliver = slivers.get(None, [])
516 # default_sliver_attribs = default_sliver.get('tags', [])
517 # for attrib in default_sliver_attribs:
518 # rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
521 links = self.get_links(sites, nodes_dict, interfaces)
522 rspec.version.add_links(links)
524 if not options.get('list_leases') or options['list_leases'] != 'resources':
526 leases = self.get_leases(slivers[0])
527 rspec.version.add_leases(leases)
530 return {'geni_urn': geni_urn,
531 'geni_rspec': rspec.toxml(),
532 'geni_slivers': geni_slivers}