sfa/planetlab/plaggregate.py

   1 #!/usr/bin/python
   2 from collections import defaultdict
   3 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
   4 from sfa.util.sfatime import utcparse, datetime_to_string
   5 from sfa.util.sfalogging import logger
   6 from sfa.util.faults import SliverDoesNotExist
   7 from sfa.rspecs.rspec import RSpec
   8 from sfa.rspecs.elements.hardware_type import HardwareType
   9 from sfa.rspecs.elements.node import Node
  10 from sfa.rspecs.elements.link import Link
  11 from sfa.rspecs.elements.sliver import Sliver
  12 from sfa.rspecs.elements.login import Login
  13 from sfa.rspecs.elements.location import Location
  14 from sfa.rspecs.elements.interface import Interface
  15 from sfa.rspecs.elements.services import Services
  16 from sfa.rspecs.elements.pltag import PLTag
  17 from sfa.rspecs.elements.lease import Lease
  18 from sfa.rspecs.elements.granularity import Granularity
  19 from sfa.rspecs.version_manager import VersionManager
  20
  21 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
  22 from sfa.planetlab.vlink import get_tc_rate
  23 from sfa.planetlab.topology import Topology
  24
  25 import time
  26
  27 class PlAggregate:
  28
  29     def __init__(self, driver):
  30         self.driver = driver
  31
  32     def get_nodes(self, options={}):
  33         filter = {'peer_id': None}
  34         geni_available = options.get('geni_available')
  35         if geni_available == True:
  36             filter['boot_state'] = 'boot'
  37         nodes = self.driver.shell.GetNodes(filter)
  38
  39         return nodes
  40
  41     def get_sites(self, filter={}):
  42         sites = {}
  43         for site in self.driver.shell.GetSites(filter):
  44             sites[site['site_id']] = site
  45         return sites
  46
  47     def get_interfaces(self, filter={}):
  48         interfaces = {}
  49         for interface in self.driver.shell.GetInterfaces(filter):
  50             iface = Interface()
  51             if interface['bwlimit']:
  52                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
  53             interfaces[interface['interface_id']] = interface
  54         return interfaces
  55
  56     def get_links(self, sites, nodes, interfaces):
  57
  58         topology = Topology()
  59         links = []
  60         for (site_id1, site_id2) in topology:
  61             site_id1 = int(site_id1)
  62             site_id2 = int(site_id2)
  63             link = Link()
  64             if not site_id1 in sites or site_id2 not in sites:
  65                 continue
  66             site1 = sites[site_id1]
  67             site2 = sites[site_id2]
  68             # get hrns
  69             site1_hrn = self.driver.hrn + '.' + site1['login_base']
  70             site2_hrn = self.driver.hrn + '.' + site2['login_base']
  71
  72             for s1_node_id in site1['node_ids']:
  73                 for s2_node_id in site2['node_ids']:
  74                     if s1_node_id not in nodes or s2_node_id not in nodes:
  75                         continue
  76                     node1 = nodes[s1_node_id]
  77                     node2 = nodes[s2_node_id]
  78                     # set interfaces
  79                     # just get first interface of the first node
  80                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
  81                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
  82                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
  83                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
  84
  85                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
  86                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
  87
  88                     # set link
  89                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
  90                     link['interface1'] = if1
  91                     link['interface2'] = if2
  92                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
  93                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
  94                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
  95                     links.append(link)
  96
  97         return links
  98
  99     def get_node_tags(self, filter={}):
 100         node_tags = {}
 101         for node_tag in self.driver.shell.GetNodeTags(filter):
 102             node_tags[node_tag['node_tag_id']] = node_tag
 103         return node_tags
 104
 105     def get_pl_initscripts(self, filter={}):
 106         pl_initscripts = {}
 107         filter.update({'enabled': True})
 108         for initscript in self.driver.shell.GetInitScripts(filter):
 109             pl_initscripts[initscript['initscript_id']] = initscript
 110         return pl_initscripts
 111
 112     def get_slivers(self, urns, options):
 113         names = set()
 114         slice_ids = set()
 115         node_ids = []
 116         for urn in urns:
 117             xrn = PlXrn(xrn=urn)
 118             if xrn.type == 'sliver':
 119                  # id: slice_id-node_id
 120                 id_parts = xrn.leaf.split('-')
 121                 slice_ids.add(id_parts[0])
 122                 node_ids.append(id_parts[1])
 123             else:
 124                 names.add(xrn.pl_slicename())
 125             if xrn.id:
 126                 ids.add(xrn.id)
 127
 128         filter = {}
 129         if names:
 130             filter['name'] = list(names)
 131         if slice_ids:
 132             filter['slice_id'] = list(slice_ids)
 133         slices = self.driver.shell.GetSlices(filter)
 134         if not slices:
 135             return []
 136         slice = slices[0]
 137         if node_ids:
 138             slice['node_ids'] = node_ids
 139         tags_dict = self.get_slice_tags(slice)
 140         nodes_dict = self.get_slice_nodes(slice, options)
 141         slivers = []
 142         for node in nodes_dict.values():
 143             node.update(slices[0])
 144             node['tags'] = tags_dict[node['node_id']]
 145             sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
 146             node['urn'] = PlXrn(xrn=sliver_hrn, type='sliver').get_urn()
 147             slivers.append(node)
 148         return slivers
 149
 150     def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
 151         rspec_node = Node()
 152         # xxx how to retrieve site['login_base']
 153         site=sites[node['site_id']]
 154         rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
 155         rspec_node['component_name'] = node['hostname']
 156         rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 157         rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 158         # do not include boot state (<available> element) in the manifest rspec
 159         rspec_node['boot_state'] = node['boot_state']
 160         rspec_node['exclusive'] = 'false'
 161         rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 162                                         HardwareType({'name': 'pc'})]
 163         # only doing this because protogeni rspec needs
 164         # to advertise available initscripts
 165         rspec_node['pl_initscripts'] = pl_initscripts.values()
 166         # add site/interface info to nodes.
 167         # assumes that sites, interfaces and tags have already been prepared.
 168         if site['longitude'] and site['latitude']:
 169             location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 170             rspec_node['location'] = location
 171         # Granularity
 172         granularity = Granularity({'grain': grain})
 173         rspec_node['granularity'] = granularity
 174         rspec_node['interfaces'] = []
 175         if_count=0
 176         for if_id in node['interface_ids']:
 177             interface = Interface(interfaces[if_id])
 178             interface['ipv4'] = interface['ip']
 179             interface['component_id'] = PlXrn(auth=self.driver.hrn,
 180                                               interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 181             # interfaces in the manifest need a client id
 182             if slice:
 183                 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 184             rspec_node['interfaces'].append(interface)
 185             if_count+=1
 186         tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']]
 187         rspec_node['tags'] = tags
 188         return rspec_node
 189
 190     def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, pl_initscripts):
 191         # get the granularity in second for the reservation system
 192         grain = self.driver.shell.GetLeaseGranularity()
 193         rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
 194         # xxx how to retrieve site['login_base']
 195         rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
 196         # remove interfaces from manifest
 197         rspec_node['interfaces'] = []
 198         # add sliver info
 199         rspec_sliver = Sliver({'sliver_id': sliver['urn'],
 200                          'name': sliver['name'],
 201                          'type': 'plab-vserver',
 202                          'tags': []})
 203         rspec_node['sliver_id'] = rspec_sliver['sliver_id']
 204         rspec_node['client_id'] = sliver['hostname']
 205         rspec_node['slivers'] = [rspec_sliver]
 206
 207         # slivers always provide the ssh service
 208         login = Login({'authentication': 'ssh-keys', 'hostname': sliver['hostname'], 'port':'22', 'username': sliver['name']})
 209         service = Services({'login': login})
 210         rspec_node['services'] = [service]
 211         return rspec_node
 212
 213     def get_slice_tags(self, slice):
 214         slice_tag_ids = []
 215         slice_tag_ids.extend(slice['slice_tag_ids'])
 216         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
 217         # sorted by node_id
 218         tags_dict = defaultdict(list)
 219         for tag in tags:
 220             tags_dict[tag['node_id']] = tag
 221         return tags_dict
 222
 223     def get_slice_nodes(self, slice, options={}):
 224         nodes_dict = {}
 225         filter = {'peer_id': None}
 226         tags_filter = {}
 227         if slice and slice.get('node_ids'):
 228             filter['node_id'] = slice['node_ids']
 229         else:
 230             # there are no nodes to look up
 231             return nodes_dict
 232         tags_filter=filter.copy()
 233         geni_available = options.get('geni_available')
 234         if geni_available == True:
 235             filter['boot_state'] = 'boot'
 236         nodes = self.driver.shell.GetNodes(filter)
 237         for node in nodes:
 238             nodes_dict[node['node_id']] = node
 239         return nodes_dict
 240
 241     def rspec_node_to_geni_sliver(self, rspec_node, allocation_status=None):
 242         op_status = "geni_unknown"
 243         state = rspec_node['boot_state'].lower()
 244         if state == 'boot':
 245             op_status = 'geni_ready'
 246         else:
 247             op_status =' geni_failed'
 248
 249         if not allocation_status:
 250             allocation_status = 'provisioned'
 251
 252         # required fields
 253         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
 254                        'geni_expires': rspec_node['expires'],
 255                        'geni_allocation_status': allocation_status,
 256                        'geni_operational_status': op_status,
 257                        'geni_error': None,
 258                        }
 259         return geni_sliver
 260
 261     def get_leases(self, slice=None, options={}):
 262
 263         now = int(time.time())
 264         filter={}
 265         filter.update({'clip':now})
 266         if slice:
 267            filter.update({'name':slice['name']})
 268         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
 269         leases = self.driver.shell.GetLeases(filter)
 270         grain = self.driver.shell.GetLeaseGranularity()
 271
 272         site_ids = []
 273         for lease in leases:
 274             site_ids.append(lease['site_id'])
 275
 276         # get sites
 277         sites_dict  = self.get_sites({'site_id': site_ids})
 278
 279         rspec_leases = []
 280         for lease in leases:
 281
 282             rspec_lease = Lease()
 283
 284             # xxx how to retrieve site['login_base']
 285             site_id=lease['site_id']
 286             site=sites_dict[site_id]
 287
 288             rspec_lease['lease_id'] = lease['lease_id']
 289             rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
 290             slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
 291             slice_urn = hrn_to_urn(slice_hrn, 'slice')
 292             rspec_lease['slice_id'] = slice_urn
 293             rspec_lease['start_time'] = lease['t_from']
 294             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
 295             rspec_leases.append(rspec_lease)
 296         return rspec_leases
 297
 298
 299     def list_resources(self, version = None, options={}):
 300
 301         version_manager = VersionManager()
 302         version = version_manager.get_version(version)
 303         rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 304         rspec = RSpec(version=rspec_version, user_options=options)
 305
 306         if not options.get('list_leases') or options['list_leases'] != 'leases':
 307             # get nodes
 308             nodes  = self.get_nodes(options)
 309             site_ids = []
 310             interface_ids = []
 311             tag_ids = []
 312             nodes_dict = {}
 313             for node in nodes:
 314                 site_ids.append(node['site_id'])
 315                 interface_ids.extend(node['interface_ids'])
 316                 tag_ids.extend(node['node_tag_ids'])
 317                 nodes_dict[node['node_id']] = node
 318             sites = self.get_sites({'site_id': site_ids})
 319             interfaces = self.get_interfaces({'interface_id':interface_ids})
 320             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 321             pl_initscripts = self.get_pl_initscripts()
 322             # convert nodes to rspec nodes
 323             rspec_nodes = []
 324             for node in nodes:
 325                 rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
 326                 rspec_nodes.append(rspec_node)
 327             rspec.version.add_nodes(rspec_nodes)
 328
 329             # add links
 330             links = self.get_links(sites, nodes_dict, interfaces)
 331             rspec.version.add_links(links)
 332         return rspec.toxml()
 333
 334     def describe(self, urns, version=None, options={}, allocation_status=None):
 335         version_manager = VersionManager()
 336         version = version_manager.get_version(version)
 337         rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 338         rspec = RSpec(version=version, user_options=options)
 339
 340         # get slivers
 341         geni_slivers = []
 342         slivers = self.get_slivers(urns, options)
 343         if len(slivers) == 0:
 344             raise SliverDoesNotExist("You have not allocated any slivers here for %s" % str(urns))
 345         rspec.xml.set('expires',  datetime_to_string(utcparse(slivers[0]['expires'])))
 346
 347         if not options.get('list_leases') or options['list_leases'] != 'leases':
 348             # add slivers
 349             site_ids = []
 350             interface_ids = []
 351             tag_ids = []
 352             nodes_dict = {}
 353             for sliver in slivers:
 354                 site_ids.append(sliver['site_id'])
 355                 interface_ids.extend(sliver['interface_ids'])
 356                 tag_ids.extend(sliver['node_tag_ids'])
 357                 nodes_dict[sliver['node_id']] = sliver
 358             sites = self.get_sites({'site_id': site_ids})
 359             interfaces = self.get_interfaces({'interface_id':interface_ids})
 360             node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 361             pl_initscripts = self.get_pl_initscripts()
 362             rspec_nodes = []
 363             for sliver in slivers:
 364                 if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
 365                     continue
 366                 rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts)
 367                 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, allocation_status=allocation_status)
 368                 rspec_nodes.append(rspec_node)
 369                 geni_slivers.append(geni_sliver)
 370             rspec.version.add_nodes(rspec_nodes)
 371
 372             # add sliver defaults
 373             #default_sliver = slivers.get(None, [])
 374             #if default_sliver:
 375             #    default_sliver_attribs = default_sliver.get('tags', [])
 376             #    for attrib in default_sliver_attribs:
 377             #        rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
 378
 379             # add links
 380             links = self.get_links(sites, nodes_dict, interfaces)
 381             rspec.version.add_links(links)
 382
 383         if not options.get('list_leases') or options['list_leases'] != 'resources':
 384             leases = self.get_leases(slivers[0])
 385             rspec.version.add_leases(leases)
 386
 387
 388         return {'geni_urn': urns[0],
 389                 'geni_rspec': rspec.toxml(),
 390                 'geni_slivers': geni_slivers}