sfa/planetlab/plaggregate.py

   1 #!/usr/bin/python
   2 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, get_authority, get_leaf
   3 from sfa.util.sfatime import utcparse, datetime_to_string
   4 from sfa.util.sfalogging import logger
   5
   6 from sfa.rspecs.rspec import RSpec
   7 from sfa.rspecs.elements.hardware_type import HardwareType
   8 from sfa.rspecs.elements.node import Node
   9 from sfa.rspecs.elements.link import Link
  10 from sfa.rspecs.elements.sliver import Sliver
  11 from sfa.rspecs.elements.login import Login
  12 from sfa.rspecs.elements.location import Location
  13 from sfa.rspecs.elements.interface import Interface
  14 from sfa.rspecs.elements.services import Services
  15 from sfa.rspecs.elements.pltag import PLTag
  16 from sfa.rspecs.elements.lease import Lease
  17 from sfa.rspecs.elements.granularity import Granularity
  18 from sfa.rspecs.version_manager import VersionManager
  19
  20 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
  21 from sfa.planetlab.vlink import get_tc_rate
  22 from sfa.planetlab.topology import Topology
  23
  24 import time
  25
  26 class PlAggregate:
  27
  28     def __init__(self, driver):
  29         self.driver = driver
  30
  31     def get_sites(self, filter={}):
  32         sites = {}
  33         for site in self.driver.shell.GetSites(filter):
  34             sites[site['site_id']] = site
  35         return sites
  36
  37     def get_interfaces(self, filter={}):
  38         interfaces = {}
  39         for interface in self.driver.shell.GetInterfaces(filter):
  40             iface = Interface()
  41             if interface['bwlimit']:
  42                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
  43             interfaces[interface['interface_id']] = interface
  44         return interfaces
  45
  46     def get_links(self, sites, nodes, interfaces):
  47
  48         topology = Topology()
  49         links = []
  50         for (site_id1, site_id2) in topology:
  51             site_id1 = int(site_id1)
  52             site_id2 = int(site_id2)
  53             link = Link()
  54             if not site_id1 in sites or site_id2 not in sites:
  55                 continue
  56             site1 = sites[site_id1]
  57             site2 = sites[site_id2]
  58             # get hrns
  59             site1_hrn = self.driver.hrn + '.' + site1['login_base']
  60             site2_hrn = self.driver.hrn + '.' + site2['login_base']
  61
  62             for s1_node_id in site1['node_ids']:
  63                 for s2_node_id in site2['node_ids']:
  64                     if s1_node_id not in nodes or s2_node_id not in nodes:
  65                         continue
  66                     node1 = nodes[s1_node_id]
  67                     node2 = nodes[s2_node_id]
  68                     # set interfaces
  69                     # just get first interface of the first node
  70                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
  71                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
  72                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
  73                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
  74
  75                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
  76                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
  77
  78                     # set link
  79                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
  80                     link['interface1'] = if1
  81                     link['interface2'] = if2
  82                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
  83                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
  84                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
  85                     links.append(link)
  86
  87         return links
  88
  89     def get_node_tags(self, filter={}):
  90         node_tags = {}
  91         for node_tag in self.driver.shell.GetNodeTags(filter):
  92             node_tags[node_tag['node_tag_id']] = node_tag
  93         return node_tags
  94
  95     def get_pl_initscripts(self, filter={}):
  96         pl_initscripts = {}
  97         filter.update({'enabled': True})
  98         for initscript in self.driver.shell.GetInitScripts(filter):
  99             pl_initscripts[initscript['initscript_id']] = initscript
 100         return pl_initscripts
 101
 102
 103     def get_slice_and_slivers(self, slice_xrn):
 104         """
 105         Returns a dict of slivers keyed on the sliver's node_id
 106         """
 107         slivers = {}
 108         slice = None
 109         if not slice_xrn:
 110             return (slice, slivers)
 111
 112         slice_urn = hrn_to_urn(slice_xrn, 'slice')
 113         slice_hrn, _ = urn_to_hrn(slice_xrn)
 114         site_hrn = get_authority(slice_hrn)
 115         top_auth_hrn = site_hrn.split('.')[0]
 116         if top_auth_hrn == self.driver.hrn:
 117             slice_name = hrn_to_pl_slicename(slice_hrn)
 118         else:
 119             login_base = '8'.join(site_hrn.split('.'))
 120             slice_name = '_'.join([login_base, get_leaf(slice_hrn)])
 121
 122         slices = self.driver.shell.GetSlices(slice_name)
 123         if not slices:
 124             return (slice, slivers)
 125         slice = slices[0]
 126
 127         # sort slivers by node id
 128         for node_id in slice['node_ids']:
 129             sliver_xrn = Xrn(slice_urn, type='sliver', id=node_id)
 130             sliver_xrn.set_authority(self.driver.hrn)
 131             sliver = Sliver({'sliver_id': sliver_xrn.urn,
 132                              'name': slice['name'],
 133                              'type': 'plab-vserver',
 134                              'tags': []})
 135             slivers[node_id]= sliver
 136
 137         # sort sliver attributes by node id
 138         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']})
 139         for tag in tags:
 140             # most likely a default/global sliver attribute (node_id == None)
 141             if tag['node_id'] not in slivers:
 142                 sliver_xrn = Xrn(slice_urn, type='sliver', id=tag['node_id'])
 143                 sliver_xrn.set_authority(self.driver.hrn)
 144                 sliver = Sliver({'sliver_id': sliver_xrn.urn,
 145                                  'name': slice['name'],
 146                                  'type': 'plab-vserver',
 147                                  'tags': []})
 148                 slivers[tag['node_id']] = sliver
 149             slivers[tag['node_id']]['tags'].append(tag)
 150
 151         return (slice, slivers)
 152
 153     def get_nodes_and_links(self, slice_xrn, slice=None,slivers=[], options={}):
 154         # if we are dealing with a slice that has no node just return
 155         # and empty list
 156         if slice_xrn:
 157             if not slice or not slice['node_ids']:
 158                 return ([],[])
 159
 160         filter = {}
 161         tags_filter = {}
 162         if slice and 'node_ids' in slice and slice['node_ids']:
 163             filter['node_id'] = slice['node_ids']
 164             tags_filter=filter.copy()
 165
 166         geni_available = options.get('geni_available')
 167         if geni_available == True:
 168             filter['boot_state'] = 'boot'
 169
 170         filter.update({'peer_id': None})
 171         nodes = self.driver.shell.GetNodes(filter)
 172
 173         # get the granularity in second for the reservation system
 174         grain = self.driver.shell.GetLeaseGranularity()
 175
 176         site_ids = []
 177         interface_ids = []
 178         tag_ids = []
 179         nodes_dict = {}
 180         for node in nodes:
 181             site_ids.append(node['site_id'])
 182             interface_ids.extend(node['interface_ids'])
 183             tag_ids.extend(node['node_tag_ids'])
 184             nodes_dict[node['node_id']] = node
 185
 186         # get sites
 187         sites_dict  = self.get_sites({'site_id': site_ids})
 188         # get interfaces
 189         interfaces = self.get_interfaces({'interface_id':interface_ids})
 190         # get tags
 191         node_tags = self.get_node_tags(tags_filter)
 192         # get initscripts
 193         pl_initscripts = self.get_pl_initscripts()
 194
 195         links = self.get_links(sites_dict, nodes_dict, interfaces)
 196
 197         rspec_nodes = []
 198         for node in nodes:
 199             # skip whitelisted nodes
 200             if node['slice_ids_whitelist']:
 201                 if not slice or slice['slice_id'] not in node['slice_ids_whitelist']:
 202                     continue
 203             rspec_node = Node()
 204             # xxx how to retrieve site['login_base']
 205             site_id=node['site_id']
 206             site=sites_dict[site_id]
 207             rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
 208             rspec_node['component_name'] = node['hostname']
 209             rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 210             rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 211             # do not include boot state (<available> element) in the manifest rspec
 212             if not slice:
 213                 rspec_node['boot_state'] = node['boot_state']
 214
 215             #add the exclusive tag to distinguish between Shared and Reservable nodes
 216             if node['node_type'] == 'reservable':
 217                 rspec_node['exclusive'] = 'true'
 218             else:
 219                 rspec_node['exclusive'] = 'false'
 220
 221             rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 222                                             HardwareType({'name': 'pc'})]
 223             # only doing this because protogeni rspec needs
 224             # to advertise available initscripts
 225             rspec_node['pl_initscripts'] = pl_initscripts.values()
 226              # add site/interface info to nodes.
 227             # assumes that sites, interfaces and tags have already been prepared.
 228             site = sites_dict[node['site_id']]
 229             if site['longitude'] and site['latitude']:
 230                 location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 231                 rspec_node['location'] = location
 232             # Granularity
 233             granularity = Granularity({'grain': grain})
 234             rspec_node['granularity'] = granularity
 235
 236             rspec_node['interfaces'] = []
 237             if_count=0
 238             for if_id in node['interface_ids']:
 239                 interface = Interface(interfaces[if_id])
 240                 interface['ipv4'] = interface['ip']
 241                 interface['component_id'] = PlXrn(auth=self.driver.hrn,
 242                                                   interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 243                 # interfaces in the manifest need a client id
 244                 if slice:
 245                     interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 246                 rspec_node['interfaces'].append(interface)
 247                 if_count+=1
 248
 249             tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']\
 250                     if tag_id in node_tags]
 251             rspec_node['tags'] = tags
 252             if node['node_id'] in slivers:
 253                 # add sliver info
 254                 sliver = slivers[node['node_id']]
 255                 rspec_node['sliver_id'] = sliver['sliver_id']
 256                 rspec_node['slivers'] = [sliver]
 257                 for tag in sliver['tags']:
 258                     if tag['tagname'] == 'client_id':
 259                          rspec_node['client_id'] = tag['value']
 260
 261                 # slivers always provide the ssh service
 262                 login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']})
 263                 service = Services({'login': login})
 264                 rspec_node['services'] = [service]
 265             rspec_nodes.append(rspec_node)
 266         return (rspec_nodes, links)
 267
 268
 269     def get_leases(self, slice_xrn=None, slice=None, options={}):
 270
 271         if slice_xrn and not slice:
 272             return []
 273
 274         now = int(time.time())
 275         filter={}
 276         filter.update({'clip':now})
 277         if slice:
 278            filter.update({'name':slice['name']})
 279         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
 280         leases = self.driver.shell.GetLeases(filter)
 281         grain = self.driver.shell.GetLeaseGranularity()
 282
 283         site_ids = []
 284         for lease in leases:
 285             site_ids.append(lease['site_id'])
 286
 287         # get sites
 288         sites_dict  = self.get_sites({'site_id': site_ids})
 289
 290         rspec_leases = []
 291         for lease in leases:
 292
 293             rspec_lease = Lease()
 294
 295             # xxx how to retrieve site['login_base']
 296             site_id=lease['site_id']
 297             site=sites_dict[site_id]
 298
 299             #rspec_lease['lease_id'] = lease['lease_id']
 300             rspec_lease['component_id'] = hrn_to_urn(self.driver.shell.GetNodeHrn(lease['hostname']), 'node')
 301             #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
 302             if slice_xrn:
 303                 slice_urn = slice_xrn
 304                 slice_hrn, _ = urn_to_hrn(slice_urn)
 305                 # Check slice HRN
 306                 if slice_hrn != self.driver.shell.GetSliceHrn(lease['slice_id']):
 307                     self.driver.shell.SetSliceHrn(lease['slice_id'], slice_hrn)
 308             else:
 309                 slice_hrn = self.driver.shell.GetSliceHrn(lease['slice_id'])
 310                 slice_urn = hrn_to_urn(slice_hrn, 'slice')
 311             rspec_lease['slice_id'] = slice_urn
 312             rspec_lease['start_time'] = lease['t_from']
 313             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
 314             rspec_leases.append(rspec_lease)
 315         return rspec_leases
 316
 317
 318     def get_rspec(self, slice_xrn=None, version = None, options={}):
 319
 320         version_manager = VersionManager()
 321         version = version_manager.get_version(version)
 322         if not slice_xrn:
 323             rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 324         else:
 325             rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 326
 327         slice, slivers = self.get_slice_and_slivers(slice_xrn)
 328         rspec = RSpec(version=rspec_version, user_options=options)
 329         if slice and 'expires' in slice:
 330             rspec.xml.set('expires',  datetime_to_string(utcparse(slice['expires'])))
 331
 332         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'leases':
 333             if slice_xrn and not slivers:
 334                 nodes, links = [], []
 335             else:
 336                 nodes, links = self.get_nodes_and_links(slice_xrn, slice, slivers, options)
 337             rspec.version.add_nodes(nodes)
 338             rspec.version.add_links(links)
 339             # add sliver defaults
 340             default_sliver = slivers.get(None, [])
 341             if default_sliver:
 342                 default_sliver_attribs = default_sliver.get('tags', [])
 343                 for attrib in default_sliver_attribs:
 344                     logger.info("adding default sliver attribute xrn=%s attrib=%s"%(slice_xrn,attrib))
 345                     rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'], self.driver.hrn)
 346
 347         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
 348            leases = self.get_leases(slice_xrn, slice)
 349            rspec.version.add_leases(leases)
 350
 351         return rspec.toxml()
 352
 353