sfa/planetlab/plaggregate.py

   1 #!/usr/bin/python
   2 from collections import defaultdict
   3 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
   4 from sfa.util.sfatime import utcparse, datetime_to_string
   5 from sfa.util.sfalogging import logger
   6 from sfa.util.faults import SliverDoesNotExist
   7 from sfa.rspecs.rspec import RSpec
   8 from sfa.rspecs.elements.hardware_type import HardwareType
   9 from sfa.rspecs.elements.node import Node
  10 from sfa.rspecs.elements.link import Link
  11 from sfa.rspecs.elements.sliver import Sliver
  12 from sfa.rspecs.elements.login import Login
  13 from sfa.rspecs.elements.location import Location
  14 from sfa.rspecs.elements.interface import Interface
  15 from sfa.rspecs.elements.services import Services
  16 from sfa.rspecs.elements.pltag import PLTag
  17 from sfa.rspecs.elements.lease import Lease
  18 from sfa.rspecs.elements.granularity import Granularity
  19 from sfa.rspecs.version_manager import VersionManager
  20
  21 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
  22 from sfa.planetlab.vlink import get_tc_rate
  23 from sfa.planetlab.topology import Topology
  24 from sfa.storage.alchemy import dbsession
  25 from sfa.storage.model import SliverAllocation
  26
  27
  28 import time
  29
  30 class PlAggregate:
  31
  32     def __init__(self, driver):
  33         self.driver = driver
  34
  35     def get_nodes(self, options={}):
  36         filter = {'peer_id': None}
  37         geni_available = options.get('geni_available')
  38         if geni_available == True:
  39             filter['boot_state'] = 'boot'
  40         nodes = self.driver.shell.GetNodes(filter)
  41
  42         return nodes
  43
  44     def get_sites(self, filter={}):
  45         sites = {}
  46         for site in self.driver.shell.GetSites(filter):
  47             sites[site['site_id']] = site
  48         return sites
  49
  50     def get_interfaces(self, filter={}):
  51         interfaces = {}
  52         for interface in self.driver.shell.GetInterfaces(filter):
  53             iface = Interface()
  54             if interface['bwlimit']:
  55                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
  56             interfaces[interface['interface_id']] = interface
  57         return interfaces
  58
  59     def get_links(self, sites, nodes, interfaces):
  60
  61         topology = Topology()
  62         links = []
  63         for (site_id1, site_id2) in topology:
  64             site_id1 = int(site_id1)
  65             site_id2 = int(site_id2)
  66             link = Link()
  67             if not site_id1 in sites or site_id2 not in sites:
  68                 continue
  69             site1 = sites[site_id1]
  70             site2 = sites[site_id2]
  71             # get hrns
  72             site1_hrn = self.driver.hrn + '.' + site1['login_base']
  73             site2_hrn = self.driver.hrn + '.' + site2['login_base']
  74
  75             for s1_node_id in site1['node_ids']:
  76                 for s2_node_id in site2['node_ids']:
  77                     if s1_node_id not in nodes or s2_node_id not in nodes:
  78                         continue
  79                     node1 = nodes[s1_node_id]
  80                     node2 = nodes[s2_node_id]
  81                     # set interfaces
  82                     # just get first interface of the first node
  83                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
  84                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
  85                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
  86                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
  87
  88                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
  89                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
  90
  91                     # set link
  92                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
  93                     link['interface1'] = if1
  94                     link['interface2'] = if2
  95                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
  96                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
  97                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
  98                     links.append(link)
  99
 100         return links
 101
 102     def get_node_tags(self, filter={}):
 103         node_tags = {}
 104         for node_tag in self.driver.shell.GetNodeTags(filter):
 105             node_tags[node_tag['node_tag_id']] = node_tag
 106         return node_tags
 107
 108     def get_pl_initscripts(self, filter={}):
 109         pl_initscripts = {}
 110         filter.update({'enabled': True})
 111         for initscript in self.driver.shell.GetInitScripts(filter):
 112             pl_initscripts[initscript['initscript_id']] = initscript
 113         return pl_initscripts
 114
 115     def get_slivers(self, urns, options={}):
 116         names = set()
 117         slice_ids = set()
 118         node_ids = []
 119         for urn in urns:
 120             xrn = PlXrn(xrn=urn)
 121             if xrn.type == 'sliver':
 122                  # id: slice_id-node_id
 123                 id_parts = xrn.leaf.split('-')
 124                 slice_ids.add(id_parts[0])
 125                 node_ids.append(id_parts[1])
 126             else:
 127                 names.add(xrn.pl_slicename())
 128             if xrn.id:
 129                 ids.add(xrn.id)
 130
 131         filter = {}
 132         if names:
 133             filter['name'] = list(names)
 134         if slice_ids:
 135             filter['slice_id'] = list(slice_ids)
 136         slices = self.driver.shell.GetSlices(filter)
 137         if not slices:
 138             return []
 139         slice = slices[0]
 140         if node_ids:
 141             slice['node_ids'] = node_ids
 142         tags_dict = self.get_slice_tags(slice)
 143         nodes_dict = self.get_slice_nodes(slice, options)
 144         slivers = []
 145         for node in nodes_dict.values():
 146             node.update(slices[0])
 147             node['tags'] = tags_dict[node['node_id']]
 148             sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
 149             node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
 150             node['urn'] = node['sliver_id']
 151             slivers.append(node)
 152         return slivers
 153
 154     def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
 155         rspec_node = Node()
 156         # xxx how to retrieve site['login_base']
 157         site=sites[node['site_id']]
 158         rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
 159         rspec_node['component_name'] = node['hostname']
 160         rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 161         rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 162         # do not include boot state (<available> element) in the manifest rspec
 163         rspec_node['boot_state'] = node['boot_state']
 164         rspec_node['exclusive'] = 'false'
 165         rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 166                                         HardwareType({'name': 'pc'})]
 167         # only doing this because protogeni rspec needs
 168         # to advertise available initscripts
 169         rspec_node['pl_initscripts'] = pl_initscripts.values()
 170         # add site/interface info to nodes.
 171         # assumes that sites, interfaces and tags have already been prepared.
 172         if site['longitude'] and site['latitude']:
 173             location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 174             rspec_node['location'] = location
 175         # Granularity
 176         granularity = Granularity({'grain': grain})
 177         rspec_node['granularity'] = granularity
 178         rspec_node['interfaces'] = []
 179         if_count=0
 180         for if_id in node['interface_ids']:
 181             interface = Interface(interfaces[if_id])
 182             interface['ipv4'] = interface['ip']
 183             interface['component_id'] = PlXrn(auth=self.driver.hrn,
 184                                               interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 185             # interfaces in the manifest need a client id
 186             if slice:
 187                 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 188             rspec_node['interfaces'].append(interface)
 189             if_count+=1
 190         tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']]
 191         rspec_node['tags'] = tags
 192         return rspec_node
 193
 194     def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, pl_initscripts):
 195         # get the granularity in second for the reservation system
 196         grain = self.driver.shell.GetLeaseGranularity()
 197         rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
 198         # xxx how to retrieve site['login_base']
 199         rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
 200         # remove interfaces from manifest
 201         rspec_node['interfaces'] = []
 202         # add sliver info
 203         rspec_sliver = Sliver({'sliver_id': sliver['urn'],
 204                          'name': sliver['name'],
 205                          'type': 'plab-vserver',
 206                          'tags': []})
 207         rspec_node['sliver_id'] = rspec_sliver['sliver_id']
 208         rspec_node['client_id'] = sliver['hostname']
 209         rspec_node['slivers'] = [rspec_sliver]
 210
 211         # slivers always provide the ssh service
 212         login = Login({'authentication': 'ssh-keys', 'hostname': sliver['hostname'], 'port':'22', 'username': sliver['name']})
 213         service = Services({'login': login})
 214         rspec_node['services'] = [service]
 215         return rspec_node
 216
 217     def get_slice_tags(self, slice):
 218         slice_tag_ids = []
 219         slice_tag_ids.extend(slice['slice_tag_ids'])
 220         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
 221         # sorted by node_id
 222         tags_dict = defaultdict(list)
 223         for tag in tags:
 224             tags_dict[tag['node_id']] = tag
 225         return tags_dict
 226
 227     def get_slice_nodes(self, slice, options={}):
 228         nodes_dict = {}
 229         filter = {'peer_id': None}
 230         tags_filter = {}
 231         if slice and slice.get('node_ids'):
 232             filter['node_id'] = slice['node_ids']
 233         else:
 234             # there are no nodes to look up
 235             return nodes_dict
 236         tags_filter=filter.copy()
 237         geni_available = options.get('geni_available')
 238         if geni_available == True:
 239             filter['boot_state'] = 'boot'
 240         nodes = self.driver.shell.GetNodes(filter)
 241         for node in nodes:
 242             nodes_dict[node['node_id']] = node
 243         return nodes_dict
 244
 245     def rspec_node_to_geni_sliver(self, rspec_node):
 246         op_status = "geni_unknown"
 247         state = rspec_node['boot_state'].lower()
 248         if state == 'boot':
 249             op_status = 'geni_ready'
 250         else:
 251             op_status =' geni_failed'
 252
 253
 254         # required fields
 255         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
 256                        'geni_expires': rspec_node['expires'],
 257                        'geni_operational_status': op_status,
 258                        'geni_error': None,
 259                        }
 260         return geni_sliver
 261
 262     def get_leases(self, slice=None, options={}):
 263
 264         now = int(time.time())
 265         filter={}
 266         filter.update({'clip':now})
 267         if slice:
 268            filter.update({'name':slice['name']})
 269         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
 270         leases = self.driver.shell.GetLeases(filter)
 271         grain = self.driver.shell.GetLeaseGranularity()
 272
 273         site_ids = []
 274         for lease in leases:
 275             site_ids.append(lease['site_id'])
 276
 277         # get sites
 278         sites_dict  = self.get_sites({'site_id': site_ids})
 279
 280         rspec_leases = []
 281         for lease in leases:
 282
 283             rspec_lease = Lease()
 284
 285             # xxx how to retrieve site['login_base']
 286             site_id=lease['site_id']
 287             site=sites_dict[site_id]
 288
 289             rspec_lease['lease_id'] = lease['lease_id']
 290             rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
 291             slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
 292             slice_urn = hrn_to_urn(slice_hrn, 'slice')
 293             rspec_lease['slice_id'] = slice_urn
 294             rspec_lease['start_time'] = lease['t_from']
 295             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
 296             rspec_leases.append(rspec_lease)
 297         return rspec_leases
 298
 299
 300     def list_resources(self, version = None, options={}):
 301
 302         version_manager = VersionManager()
 303         version = version_manager.get_version(version)
 304         rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 305         rspec = RSpec(version=rspec_version, user_options=options)
 306
 307         if not options.get('list_leases') or options['list_leases'] != 'leases':
 308             # get nodes
 309             nodes  = self.get_nodes(options)
 310             site_ids = []
 311             interface_ids = []
 312             tag_ids = []
 313             nodes_dict = {}
 314             for node in nodes:
 315                 site_ids.append(node['site_id'])
 316                 interface_ids.extend(node['interface_ids'])
 317                 tag_ids.extend(node['node_tag_ids'])
 318                 nodes_dict[node['node_id']] = node
 319             sites = self.get_sites({'site_id': site_ids})
 320             interfaces = self.get_interfaces({'interface_id':interface_ids})
 321             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 322             pl_initscripts = self.get_pl_initscripts()
 323             # convert nodes to rspec nodes
 324             rspec_nodes = []
 325             for node in nodes:
 326                 rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
 327                 rspec_nodes.append(rspec_node)
 328             rspec.version.add_nodes(rspec_nodes)
 329
 330             # add links
 331             links = self.get_links(sites, nodes_dict, interfaces)
 332             rspec.version.add_links(links)
 333         return rspec.toxml()
 334
 335     def describe(self, urns, version=None, options={}):
 336         version_manager = VersionManager()
 337         version = version_manager.get_version(version)
 338         rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 339         rspec = RSpec(version=version, user_options=options)
 340
 341         # get slivers
 342         geni_slivers = []
 343         slivers = self.get_slivers(urns, options)
 344         if len(slivers) == 0:
 345             raise SliverDoesNotExist("You have not allocated any slivers here for %s" % str(urns))
 346         rspec.xml.set('expires',  datetime_to_string(utcparse(slivers[0]['expires'])))
 347
 348         # lookup the sliver allocations
 349         sliver_ids = [sliver['sliver_id'] for sliver in slivers]
 350         constraint = SliverAllocation.sliver_id.in_(sliver_ids)
 351         sliver_allocations = dbsession.query(SliverAllocation).filter(constraint)
 352         sliver_allocation_dict = {}
 353         for sliver_allocation in sliver_allocations:
 354             sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
 355
 356         if not options.get('list_leases') or options['list_leases'] != 'leases':
 357             # add slivers
 358             site_ids = []
 359             interface_ids = []
 360             tag_ids = []
 361             nodes_dict = {}
 362             for sliver in slivers:
 363                 site_ids.append(sliver['site_id'])
 364                 interface_ids.extend(sliver['interface_ids'])
 365                 tag_ids.extend(sliver['node_tag_ids'])
 366                 nodes_dict[sliver['node_id']] = sliver
 367             sites = self.get_sites({'site_id': site_ids})
 368             interfaces = self.get_interfaces({'interface_id':interface_ids})
 369             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 370             pl_initscripts = self.get_pl_initscripts()
 371             rspec_nodes = []
 372             for sliver in slivers:
 373                 if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
 374                     continue
 375                 rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts)
 376                 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node)
 377                 sliver_allocation_record = sliver_allocation_dict.get(sliver['sliver_id'])
 378                 if sliver_allocation_record:
 379                     sliver_allocation = sliver_allocation.allocation_state
 380                 else:
 381                     sliver_allocation = 'geni_unallocated'
 382                 geni_sliver['geni_allocation_status'] = sliver_allocation
 383                 rspec_nodes.append(rspec_node)
 384                 geni_slivers.append(geni_sliver)
 385             rspec.version.add_nodes(rspec_nodes)
 386
 387             # add sliver defaults
 388             #default_sliver = slivers.get(None, [])
 389             #if default_sliver:
 390             #    default_sliver_attribs = default_sliver.get('tags', [])
 391             #    for attrib in default_sliver_attribs:
 392             #        rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
 393
 394             # add links
 395             links = self.get_links(sites, nodes_dict, interfaces)
 396             rspec.version.add_links(links)
 397
 398         if not options.get('list_leases') or options['list_leases'] != 'resources':
 399             leases = self.get_leases(slivers[0])
 400             rspec.version.add_leases(leases)
 401
 402
 403         return {'geni_urn': urns[0],
 404                 'geni_rspec': rspec.toxml(),
 405                 'geni_slivers': geni_slivers}