node elements in the manifest rspec should not contain available element
[sfa.git] / sfa / planetlab / plaggregate.py
1 #!/usr/bin/python
2 from collections import defaultdict
3 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
4 from sfa.util.sfatime import utcparse, datetime_to_string
5 from sfa.util.sfalogging import logger
6 from sfa.util.faults import SliverDoesNotExist
7 from sfa.rspecs.rspec import RSpec
8 from sfa.rspecs.elements.hardware_type import HardwareType
9 from sfa.rspecs.elements.node import Node
10 from sfa.rspecs.elements.link import Link
11 from sfa.rspecs.elements.sliver import Sliver
12 from sfa.rspecs.elements.login import Login
13 from sfa.rspecs.elements.location import Location
14 from sfa.rspecs.elements.interface import Interface
15 from sfa.rspecs.elements.services import Services
16 from sfa.rspecs.elements.pltag import PLTag
17 from sfa.rspecs.elements.lease import Lease
18 from sfa.rspecs.elements.granularity import Granularity
19 from sfa.rspecs.version_manager import VersionManager
20
21 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
22 from sfa.planetlab.vlink import get_tc_rate
23 from sfa.planetlab.topology import Topology
24 from sfa.storage.alchemy import dbsession
25 from sfa.storage.model import SliverAllocation
26
27
28 import time
29
30 class PlAggregate:
31
32     def __init__(self, driver):
33         self.driver = driver
34
35     def get_nodes(self, options={}):
36         filter = {'peer_id': None}
37         geni_available = options.get('geni_available')    
38         if geni_available == True:
39             filter['boot_state'] = 'boot'
40         nodes = self.driver.shell.GetNodes(filter)
41        
42         return nodes  
43  
44     def get_sites(self, filter={}):
45         sites = {}
46         for site in self.driver.shell.GetSites(filter):
47             sites[site['site_id']] = site
48         return sites
49
50     def get_interfaces(self, filter={}):
51         interfaces = {}
52         for interface in self.driver.shell.GetInterfaces(filter):
53             iface = Interface()
54             if interface['bwlimit']:
55                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
56             interfaces[interface['interface_id']] = interface
57         return interfaces
58
59     def get_links(self, sites, nodes, interfaces):
60         
61         topology = Topology() 
62         links = []
63         for (site_id1, site_id2) in topology:
64             site_id1 = int(site_id1)
65             site_id2 = int(site_id2)
66             link = Link()
67             if not site_id1 in sites or site_id2 not in sites:
68                 continue
69             site1 = sites[site_id1]
70             site2 = sites[site_id2]
71             # get hrns
72             site1_hrn = self.driver.hrn + '.' + site1['login_base']
73             site2_hrn = self.driver.hrn + '.' + site2['login_base']
74
75             for s1_node_id in site1['node_ids']:
76                 for s2_node_id in site2['node_ids']:
77                     if s1_node_id not in nodes or s2_node_id not in nodes:
78                         continue
79                     node1 = nodes[s1_node_id]
80                     node2 = nodes[s2_node_id]
81                     # set interfaces
82                     # just get first interface of the first node
83                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
84                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
85                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
86                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
87
88                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
89                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
90
91                     # set link
92                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
93                     link['interface1'] = if1
94                     link['interface2'] = if2
95                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
96                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
97                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
98                     links.append(link)
99
100         return links
101
102     def get_node_tags(self, filter={}):
103         node_tags = {}
104         for node_tag in self.driver.shell.GetNodeTags(filter):
105             node_tags[node_tag['node_tag_id']] = node_tag
106         return node_tags
107
108     def get_pl_initscripts(self, filter={}):
109         pl_initscripts = {}
110         filter.update({'enabled': True})
111         for initscript in self.driver.shell.GetInitScripts(filter):
112             pl_initscripts[initscript['initscript_id']] = initscript
113         return pl_initscripts
114
115     def get_slivers(self, urns, options={}):
116         names = set()
117         slice_ids = set()
118         node_ids = []
119         for urn in urns:
120             xrn = PlXrn(xrn=urn)
121             if xrn.type == 'sliver':
122                  # id: slice_id-node_id
123                 id_parts = xrn.leaf.split('-')
124                 slice_ids.add(id_parts[0]) 
125                 node_ids.append(id_parts[1])
126             else:  
127                 names.add(xrn.pl_slicename())
128             if xrn.id:
129                 ids.add(xrn.id)
130
131         filter = {}
132         if names:
133             filter['name'] = list(names)
134         if slice_ids:
135             filter['slice_id'] = list(slice_ids)
136         slices = self.driver.shell.GetSlices(filter)
137         if not slices:
138             return []
139         slice = slices[0]
140         if node_ids:
141             slice['node_ids'] = node_ids
142         tags_dict = self.get_slice_tags(slice)
143         nodes_dict = self.get_slice_nodes(slice, options)
144         slivers = []
145         for node in nodes_dict.values():
146             node.update(slices[0]) 
147             node['tags'] = tags_dict[node['node_id']]
148             sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
149             node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
150             node['urn'] = node['sliver_id'] 
151             slivers.append(node)
152         return slivers
153
154     def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
155         rspec_node = Node()
156         # xxx how to retrieve site['login_base']
157         site=sites[node['site_id']]
158         rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
159         rspec_node['component_name'] = node['hostname']
160         rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
161         rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
162         # do not include boot state (<available> element) in the manifest rspec
163         rspec_node['boot_state'] = node['boot_state']
164         if node['boot_state'] == 'boot': 
165             rspec_node['available'] = 'true'
166         else:
167             rspec_node['available'] = 'false'
168         rspec_node['exclusive'] = 'false'
169         rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
170                                         HardwareType({'name': 'pc'})]
171         # only doing this because protogeni rspec needs
172         # to advertise available initscripts
173         rspec_node['pl_initscripts'] = pl_initscripts.values()
174         # add site/interface info to nodes.
175         # assumes that sites, interfaces and tags have already been prepared.
176         if site['longitude'] and site['latitude']:
177             location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
178             rspec_node['location'] = location
179         # Granularity
180         granularity = Granularity({'grain': grain})
181         rspec_node['granularity'] = granularity
182         rspec_node['interfaces'] = []
183         if_count=0
184         for if_id in node['interface_ids']:
185             interface = Interface(interfaces[if_id])
186             interface['ipv4'] = interface['ip']
187             interface['component_id'] = PlXrn(auth=self.driver.hrn,
188                                               interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
189             # interfaces in the manifest need a client id
190             if slice:
191                 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
192             rspec_node['interfaces'].append(interface)
193             if_count+=1
194         tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']]
195         rspec_node['tags'] = tags
196         return rspec_node
197
198     def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, pl_initscripts):
199         # get the granularity in second for the reservation system
200         grain = self.driver.shell.GetLeaseGranularity()
201         rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
202         # xxx how to retrieve site['login_base']
203         rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
204         # remove interfaces from manifest
205         rspec_node['interfaces'] = []
206         # add sliver info
207         rspec_sliver = Sliver({'sliver_id': sliver['urn'],
208                          'name': sliver['name'],
209                          'type': 'plab-vserver',
210                          'tags': []})
211         rspec_node['sliver_id'] = rspec_sliver['sliver_id']
212         rspec_node['client_id'] = sliver['hostname']
213         rspec_node['slivers'] = [rspec_sliver]
214
215         # slivers always provide the ssh service
216         login = Login({'authentication': 'ssh-keys', 'hostname': sliver['hostname'], 'port':'22', 'username': sliver['name']})
217         service = Services({'login': login})
218         rspec_node['services'] = [service]    
219         return rspec_node      
220
221     def get_slice_tags(self, slice):
222         slice_tag_ids = []
223         slice_tag_ids.extend(slice['slice_tag_ids'])
224         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
225         # sorted by node_id
226         tags_dict = defaultdict(list)
227         for tag in tags:
228             tags_dict[tag['node_id']] = tag
229         return tags_dict
230
231     def get_slice_nodes(self, slice, options={}):
232         nodes_dict = {}
233         filter = {'peer_id': None}
234         tags_filter = {}
235         if slice and slice.get('node_ids'):
236             filter['node_id'] = slice['node_ids']
237         else:
238             # there are no nodes to look up
239             return nodes_dict
240         tags_filter=filter.copy()
241         geni_available = options.get('geni_available')
242         if geni_available == True:
243             filter['boot_state'] = 'boot'
244         nodes = self.driver.shell.GetNodes(filter)
245         for node in nodes:
246             nodes_dict[node['node_id']] = node
247         return nodes_dict
248
249     def rspec_node_to_geni_sliver(self, rspec_node):
250         op_status = "geni_unknown"
251         state = rspec_node['boot_state'].lower()
252         if state == 'boot':
253             op_status = 'geni_ready'
254         else:
255             op_status =' geni_failed'
256
257
258         # required fields
259         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
260                        'geni_expires': rspec_node['expires'],
261                        'geni_operational_status': op_status,
262                        'geni_error': None,
263                        }
264         return geni_sliver        
265
266     def get_leases(self, slice=None, options={}):
267         
268         now = int(time.time())
269         filter={}
270         filter.update({'clip':now})
271         if slice:
272            filter.update({'name':slice['name']})
273         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
274         leases = self.driver.shell.GetLeases(filter)
275         grain = self.driver.shell.GetLeaseGranularity()
276
277         site_ids = []
278         for lease in leases:
279             site_ids.append(lease['site_id'])
280
281         # get sites
282         sites_dict  = self.get_sites({'site_id': site_ids}) 
283   
284         rspec_leases = []
285         for lease in leases:
286
287             rspec_lease = Lease()
288             
289             # xxx how to retrieve site['login_base']
290             site_id=lease['site_id']
291             site=sites_dict[site_id]
292
293             rspec_lease['lease_id'] = lease['lease_id']
294             rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
295             slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
296             slice_urn = hrn_to_urn(slice_hrn, 'slice')
297             rspec_lease['slice_id'] = slice_urn
298             rspec_lease['start_time'] = lease['t_from']
299             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
300             rspec_leases.append(rspec_lease)
301         return rspec_leases
302
303     
304     def list_resources(self, version = None, options={}):
305
306         version_manager = VersionManager()
307         version = version_manager.get_version(version)
308         rspec_version = version_manager._get_version(version.type, version.version, 'ad')
309         rspec = RSpec(version=rspec_version, user_options=options)
310        
311         if not options.get('list_leases') or options['list_leases'] != 'leases':
312             # get nodes
313             nodes  = self.get_nodes(options)
314             site_ids = []
315             interface_ids = []
316             tag_ids = []
317             nodes_dict = {}
318             for node in nodes:
319                 site_ids.append(node['site_id'])
320                 interface_ids.extend(node['interface_ids'])
321                 tag_ids.extend(node['node_tag_ids'])
322                 nodes_dict[node['node_id']] = node
323             sites = self.get_sites({'site_id': site_ids})
324             interfaces = self.get_interfaces({'interface_id':interface_ids})
325             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
326             pl_initscripts = self.get_pl_initscripts()
327             # convert nodes to rspec nodes
328             rspec_nodes = []
329             for node in nodes:
330                 rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
331                 rspec_nodes.append(rspec_node)
332             rspec.version.add_nodes(rspec_nodes)
333
334             # add links
335             links = self.get_links(sites, nodes_dict, interfaces)        
336             rspec.version.add_links(links)
337         return rspec.toxml()
338
339     def describe(self, urns, version=None, options={}):
340         version_manager = VersionManager()
341         version = version_manager.get_version(version)
342         rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
343         rspec = RSpec(version=rspec_version, user_options=options)
344
345         # get slivers
346         geni_slivers = []
347         slivers = self.get_slivers(urns, options) 
348         if len(slivers) == 0:
349             raise SliverDoesNotExist("You have not allocated any slivers here for %s" % str(urns))
350         rspec.xml.set('expires',  datetime_to_string(utcparse(slivers[0]['expires'])))
351
352         # lookup the sliver allocations
353         sliver_ids = [sliver['sliver_id'] for sliver in slivers]
354         constraint = SliverAllocation.sliver_id.in_(sliver_ids)
355         sliver_allocations = dbsession.query(SliverAllocation).filter(constraint)
356         sliver_allocation_dict = {}
357         for sliver_allocation in sliver_allocations:
358             sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
359       
360         if not options.get('list_leases') or options['list_leases'] != 'leases':
361             # add slivers
362             site_ids = []
363             interface_ids = []
364             tag_ids = []
365             nodes_dict = {}
366             for sliver in slivers:
367                 site_ids.append(sliver['site_id'])
368                 interface_ids.extend(sliver['interface_ids'])
369                 tag_ids.extend(sliver['node_tag_ids'])
370                 nodes_dict[sliver['node_id']] = sliver
371             sites = self.get_sites({'site_id': site_ids})
372             interfaces = self.get_interfaces({'interface_id':interface_ids})
373             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
374             pl_initscripts = self.get_pl_initscripts()
375             rspec_nodes = []
376             for sliver in slivers:
377                 if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
378                     continue
379                 rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts)
380                 # manifest node element shouldn't contain available attribute
381                 rspec_node.pop('available')
382                 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node)
383                 sliver_allocation_record = sliver_allocation_dict.get(sliver['sliver_id'])
384                 if sliver_allocation_record:
385                     sliver_allocation = sliver_allocation_record.allocation_state
386                 else:
387                     sliver_allocation = 'geni_unallocated'
388                 geni_sliver['geni_allocation_status'] = sliver_allocation
389                 rspec_nodes.append(rspec_node) 
390                 geni_slivers.append(geni_sliver)
391             rspec.version.add_nodes(rspec_nodes)
392
393             # add sliver defaults
394             #default_sliver = slivers.get(None, [])
395             #if default_sliver:
396             #    default_sliver_attribs = default_sliver.get('tags', [])
397             #    for attrib in default_sliver_attribs:
398             #        rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
399
400             # add links 
401             links = self.get_links(sites, nodes_dict, interfaces)        
402             rspec.version.add_links(links)
403
404         if not options.get('list_leases') or options['list_leases'] != 'resources':
405             leases = self.get_leases(slivers[0])
406             rspec.version.add_leases(leases)
407
408                
409         return {'geni_urn': urns[0], 
410                 'geni_rspec': rspec.toxml(),
411                 'geni_slivers': geni_slivers}