2 from collections import defaultdict
3 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
4 from sfa.util.sfatime import utcparse, datetime_to_string
5 from sfa.util.sfalogging import logger
6 from sfa.util.faults import SliverDoesNotExist
7 from sfa.rspecs.rspec import RSpec
8 from sfa.rspecs.elements.hardware_type import HardwareType
9 from sfa.rspecs.elements.node import NodeElement
10 from sfa.rspecs.elements.link import Link
11 from sfa.rspecs.elements.sliver import Sliver
12 from sfa.rspecs.elements.login import Login
13 from sfa.rspecs.elements.location import Location
14 from sfa.rspecs.elements.interface import Interface
15 from sfa.rspecs.elements.services import ServicesElement
16 from sfa.rspecs.elements.pltag import PLTag
17 from sfa.rspecs.elements.lease import Lease
18 from sfa.rspecs.elements.granularity import Granularity
19 from sfa.rspecs.version_manager import VersionManager
21 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
22 from sfa.planetlab.vlink import get_tc_rate
23 from sfa.planetlab.topology import Topology
24 from sfa.storage.alchemy import dbsession
25 from sfa.storage.model import SliverAllocation
32 def __init__(self, driver):
35 def get_nodes(self, options={}):
36 filter = {'peer_id': None}
37 geni_available = options.get('geni_available')
38 if geni_available == True:
39 filter['boot_state'] = 'boot'
40 nodes = self.driver.shell.GetNodes(filter)
44 def get_sites(self, filter={}):
46 for site in self.driver.shell.GetSites(filter):
47 sites[site['site_id']] = site
50 def get_interfaces(self, filter={}):
52 for interface in self.driver.shell.GetInterfaces(filter):
54 if interface['bwlimit']:
55 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
56 interfaces[interface['interface_id']] = interface
59 def get_links(self, sites, nodes, interfaces):
63 for (site_id1, site_id2) in topology:
64 site_id1 = int(site_id1)
65 site_id2 = int(site_id2)
67 if not site_id1 in sites or site_id2 not in sites:
69 site1 = sites[site_id1]
70 site2 = sites[site_id2]
72 site1_hrn = self.driver.hrn + '.' + site1['login_base']
73 site2_hrn = self.driver.hrn + '.' + site2['login_base']
75 for s1_node_id in site1['node_ids']:
76 for s2_node_id in site2['node_ids']:
77 if s1_node_id not in nodes or s2_node_id not in nodes:
79 node1 = nodes[s1_node_id]
80 node2 = nodes[s2_node_id]
82 # just get first interface of the first node
83 if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
84 if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
85 if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
86 if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
88 if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
89 if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
92 link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
93 link['interface1'] = if1
94 link['interface2'] = if2
95 link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
96 link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
97 link['component_manager_id'] = hrn_to_urn(self.driver.hrn, 'authority+am')
102 def get_node_tags(self, filter={}):
104 for node_tag in self.driver.shell.GetNodeTags(filter):
105 node_tags[node_tag['node_tag_id']] = node_tag
108 def get_pl_initscripts(self, filter={}):
110 filter.update({'enabled': True})
111 for initscript in self.driver.shell.GetInitScripts(filter):
112 pl_initscripts[initscript['initscript_id']] = initscript
113 return pl_initscripts
115 def get_slivers(self, urns, options={}):
121 if xrn.type == 'sliver':
122 # id: slice_id-node_id
124 sliver_id_parts = xrn.get_sliver_id_parts()
125 slice_id = int(sliver_id_parts[0])
126 node_id = int(sliver_id_parts[1])
127 slice_ids.add(slice_id)
128 node_ids.append(node_id)
132 names.add(xrn.pl_slicename())
136 filter['name'] = list(names)
138 filter['slice_id'] = list(slice_ids)
140 slices = self.driver.shell.GetSlices(filter)
144 slice['hrn'] = PlXrn(auth=self.driver.hrn, slicename=slice['name']).hrn
150 person_ids.extend(slice['person_ids'])
152 persons = self.driver.shell.GetPersons(person_ids)
157 for person in persons:
158 key_ids.extend(person['key_ids'])
161 key_list = self.driver.shell.GetKeys(key_ids)
163 keys[key['key_id']] = key
165 # construct user key info
167 for person in persons:
168 name = person['email'][0:person['email'].index('@')]
170 'login': slice['name'],
171 'user_urn': Xrn('%s.%s' % (self.driver.hrn, name), type='user').urn,
172 'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys]
177 node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']]
178 slice['node_ids'] = node_ids
179 tags_dict = self.get_slice_tags(slice)
180 nodes_dict = self.get_slice_nodes(slice, options)
182 for node in nodes_dict.values():
184 node['tags'] = tags_dict[node['node_id']]
185 sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
186 node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
187 node['urn'] = node['sliver_id']
188 node['services_user'] = users
192 def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
193 rspec_node = NodeElement()
194 # xxx how to retrieve site['login_base']
195 site=sites[node['site_id']]
196 rspec_node['component_id'] = PlXrn(self.driver.hrn, hostname=node['hostname']).get_urn()
197 rspec_node['component_name'] = node['hostname']
198 rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
199 rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
200 # do not include boot state (<available> element) in the manifest rspec
201 rspec_node['boot_state'] = node['boot_state']
202 if node['boot_state'] == 'boot':
203 rspec_node['available'] = 'true'
205 rspec_node['available'] = 'false'
206 rspec_node['exclusive'] = 'false'
207 rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
208 HardwareType({'name': 'pc'})]
209 # only doing this because protogeni rspec needs
210 # to advertise available initscripts
211 rspec_node['pl_initscripts'] = pl_initscripts.values()
212 # add site/interface info to nodes.
213 # assumes that sites, interfaces and tags have already been prepared.
214 if site['longitude'] and site['latitude']:
215 location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
216 rspec_node['location'] = location
218 granularity = Granularity({'grain': grain})
219 rspec_node['granularity'] = granularity
220 rspec_node['interfaces'] = []
222 for if_id in node['interface_ids']:
223 interface = Interface(interfaces[if_id])
224 interface['ipv4'] = interface['ip']
225 interface['component_id'] = PlXrn(auth=self.driver.hrn,
226 interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
227 # interfaces in the manifest need a client id
229 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
230 rspec_node['interfaces'].append(interface)
232 tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags]
233 rspec_node['tags'] = tags
236 def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, \
237 pl_initscripts, sliver_allocations):
238 # get the granularity in second for the reservation system
239 grain = self.driver.shell.GetLeaseGranularity()
240 rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
241 # xxx how to retrieve site['login_base']
242 rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
243 # remove interfaces from manifest
244 rspec_node['interfaces'] = []
246 rspec_sliver = Sliver({'sliver_id': sliver['urn'],
247 'name': sliver['name'],
248 'type': 'plab-vserver',
250 rspec_node['sliver_id'] = rspec_sliver['sliver_id']
251 rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id
252 if sliver_allocations[sliver['urn']].component_id:
253 rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id
254 rspec_node['slivers'] = [rspec_sliver]
256 # slivers always provide the ssh service
257 login = Login({'authentication': 'ssh-keys',
258 'hostname': sliver['hostname'],
260 'username': sliver['name'],
261 'login': sliver['name']
263 service = ServicesElement({'login': login,
264 'services_user': sliver['services_user']})
265 rspec_node['services'] = [service]
268 def get_slice_tags(self, slice):
270 slice_tag_ids.extend(slice['slice_tag_ids'])
271 tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
273 tags_dict = defaultdict(list)
275 tags_dict[tag['node_id']] = tag
278 def get_slice_nodes(self, slice, options={}):
280 filter = {'peer_id': None}
282 if slice and slice.get('node_ids'):
283 filter['node_id'] = slice['node_ids']
285 # there are no nodes to look up
287 tags_filter=filter.copy()
288 geni_available = options.get('geni_available')
289 if geni_available == True:
290 filter['boot_state'] = 'boot'
291 nodes = self.driver.shell.GetNodes(filter)
293 nodes_dict[node['node_id']] = node
296 def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}):
297 if rspec_node['sliver_id'] in sliver_allocations:
298 # set sliver allocation and operational status
299 sliver_allocation = sliver_allocations[rspec_node['sliver_id']]
300 if sliver_allocation:
301 allocation_status = sliver_allocation.allocation_state
302 if allocation_status == 'geni_allocated':
303 op_status = 'geni_pending_allocation'
304 elif allocation_status == 'geni_provisioned':
305 if rspec_node['boot_state'] == 'boot':
306 op_status = 'geni_ready'
308 op_status = 'geni_failed'
310 op_status = 'geni_unknown'
312 allocation_status = 'geni_unallocated'
314 geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
315 'geni_expires': rspec_node['expires'],
316 'geni_allocation_status' : allocation_status,
317 'geni_operational_status': op_status,
322 def get_leases(self, slice_xrn=None, slice=None, options={}):
324 if slice_xrn and not slice:
327 now = int(time.time())
329 filter.update({'clip':now})
331 filter.update({'name':slice['name']})
332 return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
333 leases = self.driver.shell.GetLeases(filter)
334 grain = self.driver.shell.GetLeaseGranularity()
338 site_ids.append(lease['site_id'])
341 sites_dict = self.get_sites({'site_id': site_ids})
346 rspec_lease = Lease()
348 # xxx how to retrieve site['login_base']
349 site_id=lease['site_id']
350 site=sites_dict[site_id]
352 rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
354 slice_urn = slice_xrn
355 slice_hrn = urn_to_hrn(slice_urn)
357 slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
358 slice_urn = hrn_to_urn(slice_hrn, 'slice')
359 rspec_lease['slice_id'] = slice_urn
360 rspec_lease['start_time'] = lease['t_from']
361 rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
362 rspec_leases.append(rspec_lease)
366 def list_resources(self, version = None, options={}):
368 version_manager = VersionManager()
369 version = version_manager.get_version(version)
370 rspec_version = version_manager._get_version(version.type, version.version, 'ad')
371 rspec = RSpec(version=rspec_version, user_options=options)
373 if not options.get('list_leases') or options['list_leases'] != 'leases':
375 nodes = self.get_nodes(options)
381 site_ids.append(node['site_id'])
382 interface_ids.extend(node['interface_ids'])
383 tag_ids.extend(node['node_tag_ids'])
384 nodes_dict[node['node_id']] = node
385 sites = self.get_sites({'site_id': site_ids})
386 interfaces = self.get_interfaces({'interface_id':interface_ids})
387 node_tags = self.get_node_tags({'node_tag_id': tag_ids})
388 pl_initscripts = self.get_pl_initscripts()
389 # convert nodes to rspec nodes
392 rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
393 rspec_nodes.append(rspec_node)
394 rspec.version.add_nodes(rspec_nodes)
397 links = self.get_links(sites, nodes_dict, interfaces)
398 rspec.version.add_links(links)
400 if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
401 leases = self.get_leases(slice_xrn, slice)
402 rspec.version.add_leases(leases)
406 def describe(self, urns, version=None, options={}):
407 version_manager = VersionManager()
408 version = version_manager.get_version(version)
409 rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
410 rspec = RSpec(version=rspec_version, user_options=options)
414 slivers = self.get_slivers(urns, options)
416 rspec_expires = datetime_to_string(utcparse(slivers[0]['expires']))
418 rspec_expires = datetime_to_string(utcparse(time.time()))
419 rspec.xml.set('expires', rspec_expires)
421 # lookup the sliver allocations
423 sliver_ids = [sliver['sliver_id'] for sliver in slivers]
424 constraint = SliverAllocation.sliver_id.in_(sliver_ids)
425 sliver_allocations = dbsession.query(SliverAllocation).filter(constraint)
426 sliver_allocation_dict = {}
427 for sliver_allocation in sliver_allocations:
428 geni_urn = sliver_allocation.slice_urn
429 sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
431 if not options.get('list_leases') or options['list_leases'] != 'leases':
437 for sliver in slivers:
438 site_ids.append(sliver['site_id'])
439 interface_ids.extend(sliver['interface_ids'])
440 tag_ids.extend(sliver['node_tag_ids'])
441 nodes_dict[sliver['node_id']] = sliver
442 sites = self.get_sites({'site_id': site_ids})
443 interfaces = self.get_interfaces({'interface_id':interface_ids})
444 node_tags = self.get_node_tags({'node_tag_id': tag_ids})
445 pl_initscripts = self.get_pl_initscripts()
447 for sliver in slivers:
448 if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
450 rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags,
451 pl_initscripts, sliver_allocation_dict)
452 # manifest node element shouldn't contain available attribute
453 rspec_node.pop('available')
454 rspec_nodes.append(rspec_node)
455 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict)
456 geni_slivers.append(geni_sliver)
457 rspec.version.add_nodes(rspec_nodes)
459 # add sliver defaults
460 #default_sliver = slivers.get(None, [])
462 # default_sliver_attribs = default_sliver.get('tags', [])
463 # for attrib in default_sliver_attribs:
464 # rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
467 links = self.get_links(sites, nodes_dict, interfaces)
468 rspec.version.add_links(links)
470 if not options.get('list_leases') or options['list_leases'] != 'resources':
472 leases = self.get_leases(slivers[0])
473 rspec.version.add_leases(leases)
476 return {'geni_urn': geni_urn,
477 'geni_rspec': rspec.toxml(),
478 'geni_slivers': geni_slivers}