sfa/planetlab/plaggregate.py

   1 #!/usr/bin/python
   2 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, get_authority, get_leaf
   3 from sfa.util.sfatime import utcparse, datetime_to_string
   4 from sfa.util.sfalogging import logger
   5
   6 from sfa.rspecs.rspec import RSpec
   7 from sfa.rspecs.elements.hardware_type import HardwareType
   8 from sfa.rspecs.elements.node import Node
   9 from sfa.rspecs.elements.link import Link
  10 from sfa.rspecs.elements.sliver import Sliver
  11 from sfa.rspecs.elements.login import Login
  12 from sfa.rspecs.elements.location import Location
  13 from sfa.rspecs.elements.interface import Interface
  14 from sfa.rspecs.elements.services import Services
  15 from sfa.rspecs.elements.pltag import PLTag
  16 from sfa.rspecs.elements.lease import Lease
  17 from sfa.rspecs.elements.granularity import Granularity
  18 from sfa.rspecs.version_manager import VersionManager
  19
  20 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn, top_auth, hash_loginbase
  21 from sfa.planetlab.vlink import get_tc_rate
  22 from sfa.planetlab.topology import Topology
  23
  24 import time
  25
  26 class PlAggregate:
  27
  28     def __init__(self, driver):
  29         self.driver = driver
  30
  31     def get_sites(self, filter={}):
  32         sites = {}
  33         for site in self.driver.shell.GetSites(filter):
  34             sites[site['site_id']] = site
  35         return sites
  36
  37     def get_interfaces(self, filter={}):
  38         interfaces = {}
  39         for interface in self.driver.shell.GetInterfaces(filter):
  40             iface = Interface()
  41             if interface['bwlimit']:
  42                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
  43             interfaces[interface['interface_id']] = interface
  44         return interfaces
  45
  46     def get_links(self, sites, nodes, interfaces):
  47
  48         topology = Topology()
  49         links = []
  50         for (site_id1, site_id2) in topology:
  51             site_id1 = int(site_id1)
  52             site_id2 = int(site_id2)
  53             link = Link()
  54             if not site_id1 in sites or site_id2 not in sites:
  55                 continue
  56             site1 = sites[site_id1]
  57             site2 = sites[site_id2]
  58             # get hrns
  59             site1_hrn = self.driver.hrn + '.' + site1['login_base']
  60             site2_hrn = self.driver.hrn + '.' + site2['login_base']
  61
  62             for s1_node_id in site1['node_ids']:
  63                 for s2_node_id in site2['node_ids']:
  64                     if s1_node_id not in nodes or s2_node_id not in nodes:
  65                         continue
  66                     node1 = nodes[s1_node_id]
  67                     node2 = nodes[s2_node_id]
  68                     # set interfaces
  69                     # just get first interface of the first node
  70                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
  71                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
  72                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
  73                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
  74
  75                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
  76                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
  77
  78                     # set link
  79                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
  80                     link['interface1'] = if1
  81                     link['interface2'] = if2
  82                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
  83                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
  84                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
  85                     links.append(link)
  86
  87         return links
  88
  89     def get_node_tags(self, filter={}):
  90         node_tags = {}
  91         for node_tag in self.driver.shell.GetNodeTags(filter):
  92             node_tags[node_tag['node_tag_id']] = node_tag
  93         return node_tags
  94
  95     def get_pl_initscripts(self, filter={}):
  96         pl_initscripts = {}
  97         filter.update({'enabled': True})
  98         for initscript in self.driver.shell.GetInitScripts(filter):
  99             pl_initscripts[initscript['initscript_id']] = initscript
 100         return pl_initscripts
 101
 102
 103     def get_slice_and_slivers(self, slice_xrn):
 104         """
 105         Returns a dict of slivers keyed on the sliver's node_id
 106         """
 107         slivers = {}
 108         slice = None
 109         if not slice_xrn:
 110             return (slice, slivers)
 111
 112         slice_urn = hrn_to_urn(slice_xrn, 'slice')
 113         slice_hrn, _ = urn_to_hrn(slice_xrn)
 114
 115         top_auth_hrn = top_auth(slice_hrn)
 116         site_hrn = '.'.join(slice_hrn.split('.')[:-1])
 117         slice_part = slice_hrn.split('.')[-1]
 118         if top_auth_hrn == self.driver.hrn:
 119             login_base = slice_hrn.split('.')[-2][:12]
 120         else:
 121             login_base = hash_loginbase(site_hrn)
 122
 123         slice_name = '_'.join([login_base, slice_part])
 124
 125         slices = self.driver.shell.GetSlices(slice_name)
 126         if not slices:
 127             return (slice, slivers)
 128         slice = slices[0]
 129
 130         # sort slivers by node id
 131         for node_id in slice['node_ids']:
 132             sliver_xrn = Xrn(slice_urn, type='sliver', id=node_id)
 133             sliver_xrn.set_authority(self.driver.hrn)
 134             sliver = Sliver({'sliver_id': sliver_xrn.urn,
 135                              'name': slice['name'],
 136                              'type': 'plab-vserver',
 137                              'tags': []})
 138             slivers[node_id]= sliver
 139
 140         # sort sliver attributes by node id
 141         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']})
 142         for tag in tags:
 143             # most likely a default/global sliver attribute (node_id == None)
 144             if tag['node_id'] not in slivers:
 145                 sliver_xrn = Xrn(slice_urn, type='sliver', id=tag['node_id'])
 146                 sliver_xrn.set_authority(self.driver.hrn)
 147                 sliver = Sliver({'sliver_id': sliver_xrn.urn,
 148                                  'name': slice['name'],
 149                                  'type': 'plab-vserver',
 150                                  'tags': []})
 151                 slivers[tag['node_id']] = sliver
 152             slivers[tag['node_id']]['tags'].append(tag)
 153
 154         return (slice, slivers)
 155
 156     def get_nodes_and_links(self, slice_xrn, slice=None,slivers=[], options={}):
 157         # if we are dealing with a slice that has no node just return
 158         # and empty list
 159         if slice_xrn:
 160             if not slice or not slice['node_ids']:
 161                 return ([],[])
 162
 163         filter = {}
 164         tags_filter = {}
 165         if slice and 'node_ids' in slice and slice['node_ids']:
 166             filter['node_id'] = slice['node_ids']
 167             tags_filter=filter.copy()
 168
 169         geni_available = options.get('geni_available')
 170         if geni_available == True:
 171             filter['boot_state'] = 'boot'
 172
 173         filter.update({'peer_id': None})
 174         nodes = self.driver.shell.GetNodes(filter)
 175
 176         # get the granularity in second for the reservation system
 177         grain = self.driver.shell.GetLeaseGranularity()
 178
 179         site_ids = []
 180         interface_ids = []
 181         tag_ids = []
 182         nodes_dict = {}
 183         for node in nodes:
 184             site_ids.append(node['site_id'])
 185             interface_ids.extend(node['interface_ids'])
 186             tag_ids.extend(node['node_tag_ids'])
 187             nodes_dict[node['node_id']] = node
 188
 189         # get sites
 190         sites_dict  = self.get_sites({'site_id': site_ids})
 191         # get interfaces
 192         interfaces = self.get_interfaces({'interface_id':interface_ids})
 193         # get tags
 194         node_tags = self.get_node_tags(tags_filter)
 195         # get initscripts
 196         pl_initscripts = self.get_pl_initscripts()
 197
 198         links = self.get_links(sites_dict, nodes_dict, interfaces)
 199
 200         rspec_nodes = []
 201         for node in nodes:
 202             # skip whitelisted nodes
 203             if node['slice_ids_whitelist']:
 204                 if not slice or slice['slice_id'] not in node['slice_ids_whitelist']:
 205                     continue
 206             rspec_node = Node()
 207             # xxx how to retrieve site['login_base']
 208             site_id=node['site_id']
 209             site=sites_dict[site_id]
 210             rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
 211             rspec_node['component_name'] = node['hostname']
 212             rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 213             rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 214             # do not include boot state (<available> element) in the manifest rspec
 215             if not slice:
 216                 rspec_node['boot_state'] = node['boot_state']
 217
 218             #add the exclusive tag to distinguish between Shared and Reservable nodes
 219             if node['node_type'] == 'reservable':
 220                 rspec_node['exclusive'] = 'true'
 221             else:
 222                 rspec_node['exclusive'] = 'false'
 223
 224             rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 225                                             HardwareType({'name': 'pc'})]
 226             # only doing this because protogeni rspec needs
 227             # to advertise available initscripts
 228             rspec_node['pl_initscripts'] = pl_initscripts.values()
 229              # add site/interface info to nodes.
 230             # assumes that sites, interfaces and tags have already been prepared.
 231             site = sites_dict[node['site_id']]
 232             if site['longitude'] and site['latitude']:
 233                 location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 234                 rspec_node['location'] = location
 235             # Granularity
 236             granularity = Granularity({'grain': grain})
 237             rspec_node['granularity'] = granularity
 238
 239             rspec_node['interfaces'] = []
 240             if_count=0
 241             for if_id in node['interface_ids']:
 242                 interface = Interface(interfaces[if_id])
 243                 interface['ipv4'] = interface['ip']
 244                 interface['component_id'] = PlXrn(auth=self.driver.hrn,
 245                                                   interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 246                 # interfaces in the manifest need a client id
 247                 if slice:
 248                     interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 249                 rspec_node['interfaces'].append(interface)
 250                 if_count+=1
 251
 252             tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']\
 253                     if tag_id in node_tags]
 254             rspec_node['tags'] = tags
 255             if node['node_id'] in slivers:
 256                 # add sliver info
 257                 sliver = slivers[node['node_id']]
 258                 rspec_node['sliver_id'] = sliver['sliver_id']
 259                 rspec_node['slivers'] = [sliver]
 260                 for tag in sliver['tags']:
 261                     if tag['tagname'] == 'client_id':
 262                          rspec_node['client_id'] = tag['value']
 263
 264                 # slivers always provide the ssh service
 265                 login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']})
 266                 service = Services({'login': login})
 267                 rspec_node['services'] = [service]
 268             rspec_nodes.append(rspec_node)
 269         return (rspec_nodes, links)
 270
 271
 272     def get_leases(self, slice_xrn=None, slice=None, options={}):
 273
 274         if slice_xrn and not slice:
 275             return []
 276
 277         now = int(time.time())
 278         filter={}
 279         filter.update({'clip':now})
 280         if slice:
 281            filter.update({'name':slice['name']})
 282         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
 283         leases = self.driver.shell.GetLeases(filter)
 284         grain = self.driver.shell.GetLeaseGranularity()
 285
 286         site_ids = []
 287         for lease in leases:
 288             site_ids.append(lease['site_id'])
 289
 290         # get sites
 291         sites_dict  = self.get_sites({'site_id': site_ids})
 292
 293         rspec_leases = []
 294         for lease in leases:
 295
 296             rspec_lease = Lease()
 297
 298             # xxx how to retrieve site['login_base']
 299             site_id=lease['site_id']
 300             site=sites_dict[site_id]
 301
 302             #rspec_lease['lease_id'] = lease['lease_id']
 303             rspec_lease['component_id'] = hrn_to_urn(self.driver.shell.GetNodeHrn(lease['hostname']), 'node')
 304             #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
 305             if slice_xrn:
 306                 slice_urn = slice_xrn
 307                 slice_hrn, _ = urn_to_hrn(slice_urn)
 308                 # Check slice HRN
 309                 if slice_hrn != self.driver.shell.GetSliceHrn(lease['slice_id']):
 310                     self.driver.shell.SetSliceHrn(lease['slice_id'], slice_hrn)
 311             else:
 312                 slice_hrn = self.driver.shell.GetSliceHrn(lease['slice_id'])
 313                 slice_urn = hrn_to_urn(slice_hrn, 'slice')
 314             rspec_lease['slice_id'] = slice_urn
 315             rspec_lease['start_time'] = lease['t_from']
 316             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
 317             rspec_leases.append(rspec_lease)
 318         return rspec_leases
 319
 320
 321     def get_rspec(self, slice_xrn=None, version = None, options={}):
 322
 323         version_manager = VersionManager()
 324         version = version_manager.get_version(version)
 325         if not slice_xrn:
 326             rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 327         else:
 328             rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 329
 330         slice, slivers = self.get_slice_and_slivers(slice_xrn)
 331         rspec = RSpec(version=rspec_version, user_options=options)
 332         if slice and 'expires' in slice:
 333             rspec.xml.set('expires',  datetime_to_string(utcparse(slice['expires'])))
 334
 335         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'leases':
 336             if slice_xrn and not slivers:
 337                 nodes, links = [], []
 338             else:
 339                 nodes, links = self.get_nodes_and_links(slice_xrn, slice, slivers, options)
 340             rspec.version.add_nodes(nodes)
 341             rspec.version.add_links(links)
 342             # add sliver defaults
 343             default_sliver = slivers.get(None, [])
 344             if default_sliver:
 345                 default_sliver_attribs = default_sliver.get('tags', [])
 346                 for attrib in default_sliver_attribs:
 347                     logger.info("adding default sliver attribute xrn=%s attrib=%s"%(slice_xrn,attrib))
 348                     rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'], self.driver.hrn)
 349
 350         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
 351            leases = self.get_leases(slice_xrn, slice)
 352            rspec.version.add_leases(leases)
 353
 354         return rspec.toxml()
 355
 356