sfa/planetlab/plaggregate.py

   1 #!/usr/bin/python
   2 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, get_authority, get_leaf
   3 from sfa.util.sfatime import utcparse, datetime_to_string
   4 from sfa.util.sfalogging import logger
   5
   6 from sfa.rspecs.rspec import RSpec
   7 from sfa.rspecs.elements.hardware_type import HardwareType
   8 from sfa.rspecs.elements.node import Node
   9 from sfa.rspecs.elements.link import Link
  10 from sfa.rspecs.elements.sliver import Sliver
  11 from sfa.rspecs.elements.login import Login
  12 from sfa.rspecs.elements.location import Location
  13 from sfa.rspecs.elements.interface import Interface
  14 from sfa.rspecs.elements.services import Services
  15 from sfa.rspecs.elements.pltag import PLTag
  16 from sfa.rspecs.elements.lease import Lease
  17 from sfa.rspecs.elements.granularity import Granularity
  18 from sfa.rspecs.version_manager import VersionManager
  19
  20 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn, xrn_to_ext_slicename, top_auth
  21 from sfa.planetlab.vlink import get_tc_rate
  22 from sfa.planetlab.topology import Topology
  23
  24 import time
  25
  26 class PlAggregate:
  27
  28     def __init__(self, driver):
  29         self.driver = driver
  30
  31     def get_sites(self, filter={}):
  32         sites = {}
  33         for site in self.driver.shell.GetSites(filter):
  34             sites[site['site_id']] = site
  35         return sites
  36
  37     def get_interfaces(self, filter={}):
  38         interfaces = {}
  39         for interface in self.driver.shell.GetInterfaces(filter):
  40             iface = Interface()
  41             if interface['bwlimit']:
  42                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
  43             interfaces[interface['interface_id']] = interface
  44         return interfaces
  45
  46     def get_links(self, sites, nodes, interfaces):
  47
  48         topology = Topology()
  49         links = []
  50         for (site_id1, site_id2) in topology:
  51             site_id1 = int(site_id1)
  52             site_id2 = int(site_id2)
  53             link = Link()
  54             if not site_id1 in sites or site_id2 not in sites:
  55                 continue
  56             site1 = sites[site_id1]
  57             site2 = sites[site_id2]
  58             # get hrns
  59             site1_hrn = self.driver.hrn + '.' + site1['login_base']
  60             site2_hrn = self.driver.hrn + '.' + site2['login_base']
  61
  62             for s1_node_id in site1['node_ids']:
  63                 for s2_node_id in site2['node_ids']:
  64                     if s1_node_id not in nodes or s2_node_id not in nodes:
  65                         continue
  66                     node1 = nodes[s1_node_id]
  67                     node2 = nodes[s2_node_id]
  68                     # set interfaces
  69                     # just get first interface of the first node
  70                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
  71                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
  72                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
  73                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
  74
  75                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
  76                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
  77
  78                     # set link
  79                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
  80                     link['interface1'] = if1
  81                     link['interface2'] = if2
  82                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
  83                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
  84                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
  85                     links.append(link)
  86
  87         return links
  88
  89     def get_node_tags(self, filter={}):
  90         node_tags = {}
  91         for node_tag in self.driver.shell.GetNodeTags(filter):
  92             node_tags[node_tag['node_tag_id']] = node_tag
  93         return node_tags
  94
  95     def get_pl_initscripts(self, filter={}):
  96         pl_initscripts = {}
  97         filter.update({'enabled': True})
  98         for initscript in self.driver.shell.GetInitScripts(filter):
  99             pl_initscripts[initscript['initscript_id']] = initscript
 100         return pl_initscripts
 101
 102
 103     def get_slice_and_slivers(self, slice_xrn):
 104         """
 105         Returns a dict of slivers keyed on the sliver's node_id
 106         """
 107         slivers = {}
 108         slice = None
 109         if not slice_xrn:
 110             return (slice, slivers)
 111
 112         slice_urn = hrn_to_urn(slice_xrn, 'slice')
 113         slice_hrn, _ = urn_to_hrn(slice_xrn)
 114         top_auth_hrn = top_auth(slice_hrn)
 115         if top_auth_hrn == self.driver.hrn:
 116             slice_name = hrn_to_pl_slicename(slice_hrn)
 117         else:
 118             slice_name = xrn_to_ext_slicename(slice_hrn)
 119
 120         slices = self.driver.shell.GetSlices(slice_name)
 121         if not slices:
 122             return (slice, slivers)
 123         slice = slices[0]
 124
 125         # sort slivers by node id
 126         for node_id in slice['node_ids']:
 127             sliver_xrn = Xrn(slice_urn, type='sliver', id=node_id)
 128             sliver_xrn.set_authority(self.driver.hrn)
 129             sliver = Sliver({'sliver_id': sliver_xrn.urn,
 130                              'name': slice['name'],
 131                              'type': 'plab-vserver',
 132                              'tags': []})
 133             slivers[node_id]= sliver
 134
 135         # sort sliver attributes by node id
 136         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']})
 137         for tag in tags:
 138             # most likely a default/global sliver attribute (node_id == None)
 139             if tag['node_id'] not in slivers:
 140                 sliver_xrn = Xrn(slice_urn, type='sliver', id=tag['node_id'])
 141                 sliver_xrn.set_authority(self.driver.hrn)
 142                 sliver = Sliver({'sliver_id': sliver_xrn.urn,
 143                                  'name': slice['name'],
 144                                  'type': 'plab-vserver',
 145                                  'tags': []})
 146                 slivers[tag['node_id']] = sliver
 147             slivers[tag['node_id']]['tags'].append(tag)
 148
 149         return (slice, slivers)
 150
 151     def get_nodes_and_links(self, slice_xrn, slice=None,slivers=[], options={}):
 152         # if we are dealing with a slice that has no node just return
 153         # and empty list
 154         if slice_xrn:
 155             if not slice or not slice['node_ids']:
 156                 return ([],[])
 157
 158         filter = {}
 159         tags_filter = {}
 160         if slice and 'node_ids' in slice and slice['node_ids']:
 161             filter['node_id'] = slice['node_ids']
 162             tags_filter=filter.copy()
 163
 164         geni_available = options.get('geni_available')
 165         if geni_available == True:
 166             filter['boot_state'] = 'boot'
 167
 168         filter.update({'peer_id': None})
 169         nodes = self.driver.shell.GetNodes(filter)
 170
 171         # get the granularity in second for the reservation system
 172         grain = self.driver.shell.GetLeaseGranularity()
 173
 174         site_ids = []
 175         interface_ids = []
 176         tag_ids = []
 177         nodes_dict = {}
 178         for node in nodes:
 179             site_ids.append(node['site_id'])
 180             interface_ids.extend(node['interface_ids'])
 181             tag_ids.extend(node['node_tag_ids'])
 182             nodes_dict[node['node_id']] = node
 183
 184         # get sites
 185         sites_dict  = self.get_sites({'site_id': site_ids})
 186         # get interfaces
 187         interfaces = self.get_interfaces({'interface_id':interface_ids})
 188         # get tags
 189         node_tags = self.get_node_tags(tags_filter)
 190         # get initscripts
 191         pl_initscripts = self.get_pl_initscripts()
 192
 193         links = self.get_links(sites_dict, nodes_dict, interfaces)
 194
 195         rspec_nodes = []
 196         for node in nodes:
 197             # skip whitelisted nodes
 198             if node['slice_ids_whitelist']:
 199                 if not slice or slice['slice_id'] not in node['slice_ids_whitelist']:
 200                     continue
 201             rspec_node = Node()
 202             # xxx how to retrieve site['login_base']
 203             site_id=node['site_id']
 204             site=sites_dict[site_id]
 205             rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
 206             rspec_node['component_name'] = node['hostname']
 207             rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 208             rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 209             # do not include boot state (<available> element) in the manifest rspec
 210             if not slice:
 211                 rspec_node['boot_state'] = node['boot_state']
 212
 213             #add the exclusive tag to distinguish between Shared and Reservable nodes
 214             if node['node_type'] == 'reservable':
 215                 rspec_node['exclusive'] = 'true'
 216             else:
 217                 rspec_node['exclusive'] = 'false'
 218
 219             rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 220                                             HardwareType({'name': 'pc'})]
 221             # only doing this because protogeni rspec needs
 222             # to advertise available initscripts
 223             rspec_node['pl_initscripts'] = pl_initscripts.values()
 224              # add site/interface info to nodes.
 225             # assumes that sites, interfaces and tags have already been prepared.
 226             site = sites_dict[node['site_id']]
 227             if site['longitude'] and site['latitude']:
 228                 location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 229                 rspec_node['location'] = location
 230             # Granularity
 231             granularity = Granularity({'grain': grain})
 232             rspec_node['granularity'] = granularity
 233
 234             rspec_node['interfaces'] = []
 235             if_count=0
 236             for if_id in node['interface_ids']:
 237                 interface = Interface(interfaces[if_id])
 238                 interface['ipv4'] = interface['ip']
 239                 interface['component_id'] = PlXrn(auth=self.driver.hrn,
 240                                                   interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 241                 # interfaces in the manifest need a client id
 242                 if slice:
 243                     interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 244                 rspec_node['interfaces'].append(interface)
 245                 if_count+=1
 246
 247             tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']\
 248                     if tag_id in node_tags]
 249             rspec_node['tags'] = tags
 250             if node['node_id'] in slivers:
 251                 # add sliver info
 252                 sliver = slivers[node['node_id']]
 253                 rspec_node['sliver_id'] = sliver['sliver_id']
 254                 rspec_node['slivers'] = [sliver]
 255                 for tag in sliver['tags']:
 256                     if tag['tagname'] == 'client_id':
 257                          rspec_node['client_id'] = tag['value']
 258
 259                 # slivers always provide the ssh service
 260                 login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']})
 261                 service = Services({'login': login})
 262                 rspec_node['services'] = [service]
 263             rspec_nodes.append(rspec_node)
 264         return (rspec_nodes, links)
 265
 266
 267     def get_leases(self, slice_xrn=None, slice=None, options={}):
 268
 269         if slice_xrn and not slice:
 270             return []
 271
 272         now = int(time.time())
 273         filter={}
 274         filter.update({'clip':now})
 275         if slice:
 276            filter.update({'name':slice['name']})
 277         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
 278         leases = self.driver.shell.GetLeases(filter)
 279         grain = self.driver.shell.GetLeaseGranularity()
 280
 281         site_ids = []
 282         for lease in leases:
 283             site_ids.append(lease['site_id'])
 284
 285         # get sites
 286         sites_dict  = self.get_sites({'site_id': site_ids})
 287
 288         rspec_leases = []
 289         for lease in leases:
 290
 291             rspec_lease = Lease()
 292
 293             # xxx how to retrieve site['login_base']
 294             site_id=lease['site_id']
 295             site=sites_dict[site_id]
 296
 297             #rspec_lease['lease_id'] = lease['lease_id']
 298             rspec_lease['component_id'] = hrn_to_urn(self.driver.shell.GetNodeHrn(lease['hostname']), 'node')
 299             #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
 300             if slice_xrn:
 301                 slice_urn = slice_xrn
 302                 slice_hrn, _ = urn_to_hrn(slice_urn)
 303                 # Check slice HRN
 304                 if slice_hrn != self.driver.shell.GetSliceHrn(lease['slice_id']):
 305                     self.driver.shell.SetSliceHrn(lease['slice_id'], slice_hrn)
 306             else:
 307                 slice_hrn = self.driver.shell.GetSliceHrn(lease['slice_id'])
 308                 slice_urn = hrn_to_urn(slice_hrn, 'slice')
 309             rspec_lease['slice_id'] = slice_urn
 310             rspec_lease['start_time'] = lease['t_from']
 311             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
 312             rspec_leases.append(rspec_lease)
 313         return rspec_leases
 314
 315
 316     def get_rspec(self, slice_xrn=None, version = None, options={}):
 317
 318         version_manager = VersionManager()
 319         version = version_manager.get_version(version)
 320         if not slice_xrn:
 321             rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 322         else:
 323             rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 324
 325         slice, slivers = self.get_slice_and_slivers(slice_xrn)
 326         rspec = RSpec(version=rspec_version, user_options=options)
 327         if slice and 'expires' in slice:
 328             rspec.xml.set('expires',  datetime_to_string(utcparse(slice['expires'])))
 329
 330         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'leases':
 331             if slice_xrn and not slivers:
 332                 nodes, links = [], []
 333             else:
 334                 nodes, links = self.get_nodes_and_links(slice_xrn, slice, slivers, options)
 335             rspec.version.add_nodes(nodes)
 336             rspec.version.add_links(links)
 337             # add sliver defaults
 338             default_sliver = slivers.get(None, [])
 339             if default_sliver:
 340                 default_sliver_attribs = default_sliver.get('tags', [])
 341                 for attrib in default_sliver_attribs:
 342                     logger.info("adding default sliver attribute xrn=%s attrib=%s"%(slice_xrn,attrib))
 343                     rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'], self.driver.hrn)
 344
 345         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
 346            leases = self.get_leases(slice_xrn, slice)
 347            rspec.version.add_leases(leases)
 348
 349         return rspec.toxml()
 350
 351