sfa/planetlab/plaggregate.py

   1 #!/usr/bin/python
   2 from collections import defaultdict
   3 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
   4 from sfa.util.sfatime import utcparse, datetime_to_string
   5 from sfa.util.sfalogging import logger
   6 from sfa.util.faults import SliverDoesNotExist
   7 from sfa.rspecs.rspec import RSpec
   8 from sfa.rspecs.elements.hardware_type import HardwareType
   9 from sfa.rspecs.elements.node import Node
  10 from sfa.rspecs.elements.link import Link
  11 from sfa.rspecs.elements.sliver import Sliver
  12 from sfa.rspecs.elements.login import Login
  13 from sfa.rspecs.elements.location import Location
  14 from sfa.rspecs.elements.interface import Interface
  15 from sfa.rspecs.elements.services import Services
  16 from sfa.rspecs.elements.pltag import PLTag
  17 from sfa.rspecs.elements.lease import Lease
  18 from sfa.rspecs.elements.granularity import Granularity
  19 from sfa.rspecs.version_manager import VersionManager
  20
  21 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
  22 from sfa.planetlab.vlink import get_tc_rate
  23 from sfa.planetlab.topology import Topology
  24 from sfa.storage.alchemy import dbsession
  25 from sfa.storage.model import SliverAllocation
  26
  27
  28 import time
  29
  30 class PlAggregate:
  31
  32     def __init__(self, driver):
  33         self.driver = driver
  34
  35     def get_nodes(self, options={}):
  36         filter = {'peer_id': None}
  37         geni_available = options.get('geni_available')
  38         if geni_available == True:
  39             filter['boot_state'] = 'boot'
  40         nodes = self.driver.shell.GetNodes(filter)
  41
  42         return nodes
  43
  44     def get_sites(self, filter={}):
  45         sites = {}
  46         for site in self.driver.shell.GetSites(filter):
  47             sites[site['site_id']] = site
  48         return sites
  49
  50     def get_interfaces(self, filter={}):
  51         interfaces = {}
  52         for interface in self.driver.shell.GetInterfaces(filter):
  53             iface = Interface()
  54             if interface['bwlimit']:
  55                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
  56             interfaces[interface['interface_id']] = interface
  57         return interfaces
  58
  59     def get_links(self, sites, nodes, interfaces):
  60
  61         topology = Topology()
  62         links = []
  63         for (site_id1, site_id2) in topology:
  64             site_id1 = int(site_id1)
  65             site_id2 = int(site_id2)
  66             link = Link()
  67             if not site_id1 in sites or site_id2 not in sites:
  68                 continue
  69             site1 = sites[site_id1]
  70             site2 = sites[site_id2]
  71             # get hrns
  72             site1_hrn = self.driver.hrn + '.' + site1['login_base']
  73             site2_hrn = self.driver.hrn + '.' + site2['login_base']
  74
  75             for s1_node_id in site1['node_ids']:
  76                 for s2_node_id in site2['node_ids']:
  77                     if s1_node_id not in nodes or s2_node_id not in nodes:
  78                         continue
  79                     node1 = nodes[s1_node_id]
  80                     node2 = nodes[s2_node_id]
  81                     # set interfaces
  82                     # just get first interface of the first node
  83                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
  84                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
  85                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
  86                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
  87
  88                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
  89                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
  90
  91                     # set link
  92                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
  93                     link['interface1'] = if1
  94                     link['interface2'] = if2
  95                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
  96                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
  97                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
  98                     links.append(link)
  99
 100         return links
 101
 102     def get_node_tags(self, filter={}):
 103         node_tags = {}
 104         for node_tag in self.driver.shell.GetNodeTags(filter):
 105             node_tags[node_tag['node_tag_id']] = node_tag
 106         return node_tags
 107
 108     def get_pl_initscripts(self, filter={}):
 109         pl_initscripts = {}
 110         filter.update({'enabled': True})
 111         for initscript in self.driver.shell.GetInitScripts(filter):
 112             pl_initscripts[initscript['initscript_id']] = initscript
 113         return pl_initscripts
 114
 115     def get_slivers(self, urns, options={}):
 116         names = set()
 117         slice_ids = set()
 118         node_ids = []
 119         for urn in urns:
 120             xrn = PlXrn(xrn=urn)
 121             if xrn.type == 'sliver':
 122                  # id: slice_id-node_id
 123                 try:
 124                     sliver_id_parts = xrn.get_sliver_id_parts()
 125                     slice_id = int(sliver_id_parts[0])
 126                     node_id = int(sliver_id_parts[1])
 127                     slice_ids.add(slice_id)
 128                     node_ids.append(node_id)
 129                 except ValueError:
 130                     pass
 131             else:
 132                 names.add(xrn.pl_slicename())
 133
 134         filter = {}
 135         if names:
 136             filter['name'] = list(names)
 137         if slice_ids:
 138             filter['slice_id'] = list(slice_ids)
 139         slices = self.driver.shell.GetSlices(filter)
 140         if not slices:
 141             return []
 142         slice = slices[0]
 143         if node_ids:
 144             node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']]
 145             slice['node_ids'] = node_ids
 146         tags_dict = self.get_slice_tags(slice)
 147         nodes_dict = self.get_slice_nodes(slice, options)
 148         slivers = []
 149         for node in nodes_dict.values():
 150             node.update(slices[0])
 151             node['tags'] = tags_dict[node['node_id']]
 152             sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
 153             node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
 154             node['urn'] = node['sliver_id']
 155             slivers.append(node)
 156         return slivers
 157
 158     def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
 159         rspec_node = Node()
 160         # xxx how to retrieve site['login_base']
 161         site=sites[node['site_id']]
 162         rspec_node['component_id'] = PlXrn(self.driver.hrn, hostname=node['hostname']).get_urn()
 163         rspec_node['component_name'] = node['hostname']
 164         rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 165         rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 166         # do not include boot state (<available> element) in the manifest rspec
 167         rspec_node['boot_state'] = node['boot_state']
 168         if node['boot_state'] == 'boot':
 169             rspec_node['available'] = 'true'
 170         else:
 171             rspec_node['available'] = 'false'
 172         rspec_node['exclusive'] = 'false'
 173         rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 174                                         HardwareType({'name': 'pc'})]
 175         # only doing this because protogeni rspec needs
 176         # to advertise available initscripts
 177         rspec_node['pl_initscripts'] = pl_initscripts.values()
 178         # add site/interface info to nodes.
 179         # assumes that sites, interfaces and tags have already been prepared.
 180         if site['longitude'] and site['latitude']:
 181             location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 182             rspec_node['location'] = location
 183         # Granularity
 184         granularity = Granularity({'grain': grain})
 185         rspec_node['granularity'] = granularity
 186         rspec_node['interfaces'] = []
 187         if_count=0
 188         for if_id in node['interface_ids']:
 189             interface = Interface(interfaces[if_id])
 190             interface['ipv4'] = interface['ip']
 191             interface['component_id'] = PlXrn(auth=self.driver.hrn,
 192                                               interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 193             # interfaces in the manifest need a client id
 194             if slice:
 195                 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 196             rspec_node['interfaces'].append(interface)
 197             if_count+=1
 198         tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags]
 199         rspec_node['tags'] = tags
 200         return rspec_node
 201
 202     def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, \
 203                              pl_initscripts, sliver_allocations):
 204         # get the granularity in second for the reservation system
 205         grain = self.driver.shell.GetLeaseGranularity()
 206         rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
 207         # xxx how to retrieve site['login_base']
 208         rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
 209         # remove interfaces from manifest
 210         rspec_node['interfaces'] = []
 211         # add sliver info
 212         rspec_sliver = Sliver({'sliver_id': sliver['urn'],
 213                          'name': sliver['name'],
 214                          'type': 'plab-vserver',
 215                          'tags': []})
 216         rspec_node['sliver_id'] = rspec_sliver['sliver_id']
 217         rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id
 218         if sliver_allocations[sliver['urn']].component_id:
 219             rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id
 220         rspec_node['slivers'] = [rspec_sliver]
 221
 222         # slivers always provide the ssh service
 223         login = Login({'authentication': 'ssh-keys', 'hostname': sliver['hostname'], 'port':'22', 'username': sliver['name']})
 224         service = Services({'login': login})
 225         rspec_node['services'] = [service]
 226         return rspec_node
 227
 228     def get_slice_tags(self, slice):
 229         slice_tag_ids = []
 230         slice_tag_ids.extend(slice['slice_tag_ids'])
 231         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
 232         # sorted by node_id
 233         tags_dict = defaultdict(list)
 234         for tag in tags:
 235             tags_dict[tag['node_id']] = tag
 236         return tags_dict
 237
 238     def get_slice_nodes(self, slice, options={}):
 239         nodes_dict = {}
 240         filter = {'peer_id': None}
 241         tags_filter = {}
 242         if slice and slice.get('node_ids'):
 243             filter['node_id'] = slice['node_ids']
 244         else:
 245             # there are no nodes to look up
 246             return nodes_dict
 247         tags_filter=filter.copy()
 248         geni_available = options.get('geni_available')
 249         if geni_available == True:
 250             filter['boot_state'] = 'boot'
 251         nodes = self.driver.shell.GetNodes(filter)
 252         for node in nodes:
 253             nodes_dict[node['node_id']] = node
 254         return nodes_dict
 255
 256     def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}):
 257         if rspec_node['sliver_id'] in sliver_allocations:
 258             # set sliver allocation and operational status
 259             sliver_allocation = sliver_allocations[rspec_node['sliver_id']]
 260             if sliver_allocation:
 261                 allocation_status = sliver_allocation.allocation_state
 262                 if allocation_status == 'geni_allocated':
 263                     op_status =  'geni_pending_allocation'
 264                 elif allocation_status == 'geni_provisioned':
 265                     if rspec_node['boot_state'] == 'boot':
 266                         op_status = 'geni_ready'
 267                     else:
 268                         op_status = 'geni_failed'
 269                 else:
 270                     op_status = 'geni_unknown'
 271             else:
 272                 allocation_status = 'geni_unallocated'
 273         # required fields
 274         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
 275                        'geni_expires': rspec_node['expires'],
 276                        'geni_allocation_status' : allocation_status,
 277                        'geni_operational_status': op_status,
 278                        'geni_error': '',
 279                        }
 280         return geni_sliver
 281
 282     def get_leases(self, slice=None, options={}):
 283
 284         now = int(time.time())
 285         filter={}
 286         filter.update({'clip':now})
 287         if slice:
 288            filter.update({'name':slice['name']})
 289         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
 290         leases = self.driver.shell.GetLeases(filter)
 291         grain = self.driver.shell.GetLeaseGranularity()
 292
 293         site_ids = []
 294         for lease in leases:
 295             site_ids.append(lease['site_id'])
 296
 297         # get sites
 298         sites_dict  = self.get_sites({'site_id': site_ids})
 299
 300         rspec_leases = []
 301         for lease in leases:
 302
 303             rspec_lease = Lease()
 304
 305             # xxx how to retrieve site['login_base']
 306             site_id=lease['site_id']
 307             site=sites_dict[site_id]
 308
 309             rspec_lease['lease_id'] = lease['lease_id']
 310             rspec_lease['component_id'] = PlXrn(self.driver.hrn, hostname=lease['hostname']).urn
 311             slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
 312             slice_urn = hrn_to_urn(slice_hrn, 'slice')
 313             rspec_lease['slice_id'] = slice_urn
 314             rspec_lease['start_time'] = lease['t_from']
 315             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
 316             rspec_leases.append(rspec_lease)
 317         return rspec_leases
 318
 319
 320     def list_resources(self, version = None, options={}):
 321
 322         version_manager = VersionManager()
 323         version = version_manager.get_version(version)
 324         rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 325         rspec = RSpec(version=rspec_version, user_options=options)
 326
 327         if not options.get('list_leases') or options['list_leases'] != 'leases':
 328             # get nodes
 329             nodes  = self.get_nodes(options)
 330             site_ids = []
 331             interface_ids = []
 332             tag_ids = []
 333             nodes_dict = {}
 334             for node in nodes:
 335                 site_ids.append(node['site_id'])
 336                 interface_ids.extend(node['interface_ids'])
 337                 tag_ids.extend(node['node_tag_ids'])
 338                 nodes_dict[node['node_id']] = node
 339             sites = self.get_sites({'site_id': site_ids})
 340             interfaces = self.get_interfaces({'interface_id':interface_ids})
 341             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 342             pl_initscripts = self.get_pl_initscripts()
 343             # convert nodes to rspec nodes
 344             rspec_nodes = []
 345             for node in nodes:
 346                 rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
 347                 rspec_nodes.append(rspec_node)
 348             rspec.version.add_nodes(rspec_nodes)
 349
 350             # add links
 351             links = self.get_links(sites, nodes_dict, interfaces)
 352             rspec.version.add_links(links)
 353         return rspec.toxml()
 354
 355     def describe(self, urns, version=None, options={}):
 356         version_manager = VersionManager()
 357         version = version_manager.get_version(version)
 358         rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 359         rspec = RSpec(version=rspec_version, user_options=options)
 360
 361         # get slivers
 362         geni_slivers = []
 363         slivers = self.get_slivers(urns, options)
 364         if slivers:
 365             rspec_expires = datetime_to_string(utcparse(slivers[0]['expires']))
 366         else:
 367             rspec_expires = datetime_to_string(utcparse(time.time()))
 368         rspec.xml.set('expires',  rspec_expires)
 369
 370         # lookup the sliver allocations
 371         geni_urn = urns[0]
 372         sliver_ids = [sliver['sliver_id'] for sliver in slivers]
 373         constraint = SliverAllocation.sliver_id.in_(sliver_ids)
 374         sliver_allocations = dbsession.query(SliverAllocation).filter(constraint)
 375         sliver_allocation_dict = {}
 376         for sliver_allocation in sliver_allocations:
 377             geni_urn = sliver_allocation.slice_urn
 378             sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
 379
 380         if not options.get('list_leases') or options['list_leases'] != 'leases':
 381             # add slivers
 382             site_ids = []
 383             interface_ids = []
 384             tag_ids = []
 385             nodes_dict = {}
 386             for sliver in slivers:
 387                 site_ids.append(sliver['site_id'])
 388                 interface_ids.extend(sliver['interface_ids'])
 389                 tag_ids.extend(sliver['node_tag_ids'])
 390                 nodes_dict[sliver['node_id']] = sliver
 391             sites = self.get_sites({'site_id': site_ids})
 392             interfaces = self.get_interfaces({'interface_id':interface_ids})
 393             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 394             pl_initscripts = self.get_pl_initscripts()
 395             rspec_nodes = []
 396             for sliver in slivers:
 397                 if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
 398                     continue
 399                 rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags,
 400                                                        pl_initscripts, sliver_allocation_dict)
 401                 # manifest node element shouldn't contain available attribute
 402                 rspec_node.pop('available')
 403                 rspec_nodes.append(rspec_node)
 404                 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict)
 405                 geni_slivers.append(geni_sliver)
 406             rspec.version.add_nodes(rspec_nodes)
 407
 408             # add sliver defaults
 409             #default_sliver = slivers.get(None, [])
 410             #if default_sliver:
 411             #    default_sliver_attribs = default_sliver.get('tags', [])
 412             #    for attrib in default_sliver_attribs:
 413             #        rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
 414
 415             # add links
 416             links = self.get_links(sites, nodes_dict, interfaces)
 417             rspec.version.add_links(links)
 418
 419         if not options.get('list_leases') or options['list_leases'] != 'resources':
 420             if slivers:
 421                 leases = self.get_leases(slivers[0])
 422                 rspec.version.add_leases(leases)
 423
 424
 425         return {'geni_urn': geni_urn,
 426                 'geni_rspec': rspec.toxml(),
 427                 'geni_slivers': geni_slivers}