sfa/planetlab/plaggregate.py

   1 #!/usr/bin/python
   2 from collections import defaultdict
   3 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
   4 from sfa.util.sfatime import utcparse, datetime_to_string
   5 from sfa.util.sfalogging import logger
   6
   7 from sfa.rspecs.rspec import RSpec
   8 from sfa.rspecs.elements.hardware_type import HardwareType
   9 from sfa.rspecs.elements.node import Node
  10 from sfa.rspecs.elements.link import Link
  11 from sfa.rspecs.elements.sliver import Sliver
  12 from sfa.rspecs.elements.login import Login
  13 from sfa.rspecs.elements.location import Location
  14 from sfa.rspecs.elements.interface import Interface
  15 from sfa.rspecs.elements.services import Services
  16 from sfa.rspecs.elements.pltag import PLTag
  17 from sfa.rspecs.elements.lease import Lease
  18 from sfa.rspecs.elements.granularity import Granularity
  19 from sfa.rspecs.version_manager import VersionManager
  20
  21 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
  22 from sfa.planetlab.vlink import get_tc_rate
  23 from sfa.planetlab.topology import Topology
  24
  25 import time
  26
  27 class PlAggregate:
  28
  29     def __init__(self, driver):
  30         self.driver = driver
  31
  32     def get_nodes(self, options={}):
  33         filter = {'peer_id': None}
  34         geni_available = options.get('geni_available')
  35         if geni_available == True:
  36             filter['boot_state'] = 'boot'
  37         nodes = self.driver.shell.GetNodes(filter)
  38
  39         return nodes
  40
  41     def get_sites(self, filter={}):
  42         sites = {}
  43         for site in self.driver.shell.GetSites(filter):
  44             sites[site['site_id']] = site
  45         return sites
  46
  47     def get_interfaces(self, filter={}):
  48         interfaces = {}
  49         for interface in self.driver.shell.GetInterfaces(filter):
  50             iface = Interface()
  51             if interface['bwlimit']:
  52                 interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
  53             interfaces[interface['interface_id']] = interface
  54         return interfaces
  55
  56     def get_links(self, sites, nodes, interfaces):
  57
  58         topology = Topology()
  59         links = []
  60         for (site_id1, site_id2) in topology:
  61             site_id1 = int(site_id1)
  62             site_id2 = int(site_id2)
  63             link = Link()
  64             if not site_id1 in sites or site_id2 not in sites:
  65                 continue
  66             site1 = sites[site_id1]
  67             site2 = sites[site_id2]
  68             # get hrns
  69             site1_hrn = self.driver.hrn + '.' + site1['login_base']
  70             site2_hrn = self.driver.hrn + '.' + site2['login_base']
  71
  72             for s1_node_id in site1['node_ids']:
  73                 for s2_node_id in site2['node_ids']:
  74                     if s1_node_id not in nodes or s2_node_id not in nodes:
  75                         continue
  76                     node1 = nodes[s1_node_id]
  77                     node2 = nodes[s2_node_id]
  78                     # set interfaces
  79                     # just get first interface of the first node
  80                     if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
  81                     if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
  82                     if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
  83                     if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']
  84
  85                     if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
  86                     if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )
  87
  88                     # set link
  89                     link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
  90                     link['interface1'] = if1
  91                     link['interface2'] = if2
  92                     link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
  93                     link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
  94                     link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
  95                     links.append(link)
  96
  97         return links
  98
  99     def get_node_tags(self, filter={}):
 100         node_tags = {}
 101         for node_tag in self.driver.shell.GetNodeTags(filter):
 102             node_tags[node_tag['node_tag_id']] = node_tag
 103         return node_tags
 104
 105     def get_pl_initscripts(self, filter={}):
 106         pl_initscripts = {}
 107         filter.update({'enabled': True})
 108         for initscript in self.driver.shell.GetInitScripts(filter):
 109             pl_initscripts[initscript['initscript_id']] = initscript
 110         return pl_initscripts
 111
 112     def get_slivers(self, urns, options):
 113         names = set()
 114         ids = set()
 115         for urn in urns:
 116             xrn = PlXrn(xrn=urn)
 117             names.add(xrn.get_slice_name())
 118             if xrn.id:
 119                 ids.add(xrn.id)
 120
 121         slices = self.driver.shell.GetSlices(names)
 122         # filter on id
 123         if ids:
 124             slices = [slice for slice in slices if slice['slice_id'] in ids]
 125
 126         tags_dict = self.get_slice_tags(slices)
 127         nodes_dict = self.get_slice_nodes(slices, options)
 128         slivers = []
 129         for node in nodes_dict.values():
 130             sliver = node.update(slices[0])
 131             sliver['tags'] = tags_dict[node['node_id']]
 132         return slivers
 133
 134     def node_to_rspec_node(self, sites, interfaces, tags, pl_initscripts=[], grain=None, options={}):
 135         rspec_node = Node()
 136         # xxx how to retrieve site['login_base']
 137         site=sites_dict[node['site_id']]
 138         rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
 139         rspec_node['component_name'] = node['hostname']
 140         rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 141         rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 142         # do not include boot state (<available> element) in the manifest rspec
 143         rspec_node['boot_state'] = node['boot_state']
 144         rspec_node['exclusive'] = 'false'
 145         rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 146                                         HardwareType({'name': 'pc'})]
 147         # only doing this because protogeni rspec needs
 148         # to advertise available initscripts
 149         rspec_node['pl_initscripts'] = pl_initscripts.values()
 150         # add site/interface info to nodes.
 151         # assumes that sites, interfaces and tags have already been prepared.
 152         if site['longitude'] and site['latitude']:
 153             location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 154             rspec_node['location'] = location
 155         # Granularity
 156         granularity = Granularity({'grain': grain})
 157         rspec_node['granularity'] = granularity
 158         rspec_node['interfaces'] = []
 159         if_count=0
 160         for if_id in node['interface_ids']:
 161             interface = Interface(interfaces[if_id])
 162             interface['ipv4'] = interface['ip']
 163             interface['component_id'] = PlXrn(auth=self.driver.hrn,
 164                                               interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 165             # interfaces in the manifest need a client id
 166             if slice:
 167                 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 168             rspec_node['interfaces'].append(interface)
 169             if_count+=1
 170         tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']]
 171         rspec_node['tags'] = tags
 172         return rspec_node
 173
 174     def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, pl_initscripts):
 175         # get the granularity in second for the reservation system
 176         grain = self.driver.shell.GetLeaseGranularity()
 177         rspec_node = self.get_rspec_node(node, sites, interfaces, node_tags, pl_initscripts, grain)
 178         # xxx how to retrieve site['login_base']
 179         rspec_node['expires'] = datetime_to_string(utcparse(slice[0]['expires']))
 180         # remove interfaces from manifest
 181         rspec_node['interfaces'] = []
 182         # add sliver info
 183         id = ":".join(map(str, [slices[0]['slice_id'], node['node_id']]))
 184         sliver_xrn = Xrn(slice_urn, id=id).get_urn()
 185         sliver_xrn.set_authority(self.driver.hrn)
 186         sliver = Sliver({'sliver_id': sliver_xrn.get_urn(),
 187                          'name': slice[0]['name'],
 188                          'type': 'plab-vserver',
 189                          'tags': []})
 190         rspec_node['sliver_id'] = sliver['sliver_id']
 191         rspec_node['client_id'] = node['hostname']
 192         rspec_node['slivers'] = [sliver]
 193
 194         # slivers always provide the ssh service
 195         login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']})
 196         service = Services({'login': login})
 197         rspec_node['services'] = [service]
 198         return rspec_node
 199
 200     def get_slice_tags(self, slices):
 201         slice_tag_ids = []
 202         for slice in slices:
 203             slice_tag_ids.extend(slice['slice_tag_ids'])
 204         tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
 205         # sorted by node_id
 206         tags_dict = defaultdict([])
 207         for tag in tags:
 208             tags_dict[tag['node_id']] = tag
 209         return tags_dict
 210
 211     def get_slice_nodes(self, slices, options={}):
 212         filter = {'peer_id': None}
 213         tags_filter = {}
 214         if slice and 'node_ids' in slice and slice['node_ids']:
 215             filter['node_id'] = slice['node_ids']
 216             tags_filter=filter.copy()
 217
 218         geni_available = options.get('geni_available')
 219         if geni_available == True:
 220             filter['boot_state'] = 'boot'
 221         nodes = self.driver.shell.GetNodes(filter)
 222         nodes_dict = {}
 223         for node in nodes:
 224             nodes_dict[node['node_id']] = node
 225         return nodes_dict
 226
 227     def rspec_node_to_geni_sliver(self, rspec_node):
 228         op_status = "geni_unknown"
 229         state = sliver['boot_stat'].lower()
 230         if state == 'boot':
 231             op_status = 'geni_ready'
 232         else:
 233             op_status =' geni_failed'
 234
 235         # required fields
 236         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
 237                        'geni_expires': rspec_node['expires'],
 238                        'geni_allocation_status': 'geni_provisioned',
 239                        'geni_operational_status': op_status,
 240                        'geni_error': None,
 241                        }
 242         return geni_sliver
 243
 244     def get_leases(self, slice=None, options={}):
 245
 246         now = int(time.time())
 247         filter={}
 248         filter.update({'clip':now})
 249         if slice:
 250            filter.update({'name':slice['name']})
 251         return_fields = ['lease_id', 'hostname', 'site_id', 'name', 't_from', 't_until']
 252         leases = self.driver.shell.GetLeases(filter)
 253         grain = self.driver.shell.GetLeaseGranularity()
 254
 255         site_ids = []
 256         for lease in leases:
 257             site_ids.append(lease['site_id'])
 258
 259         # get sites
 260         sites_dict  = self.get_sites({'site_id': site_ids})
 261
 262         rspec_leases = []
 263         for lease in leases:
 264
 265             rspec_lease = Lease()
 266
 267             # xxx how to retrieve site['login_base']
 268             site_id=lease['site_id']
 269             site=sites_dict[site_id]
 270
 271             rspec_lease['lease_id'] = lease['lease_id']
 272             rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
 273             slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
 274             slice_urn = hrn_to_urn(slice_hrn, 'slice')
 275             rspec_lease['slice_id'] = slice_urn
 276             rspec_lease['start_time'] = lease['t_from']
 277             rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
 278             rspec_leases.append(rspec_lease)
 279         return rspec_leases
 280
 281
 282     def list_resources(self, version = None, options={}):
 283
 284         version_manager = VersionManager()
 285         version = version_manager.get_version(version)
 286         rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 287         rspec = RSpec(version=rspec_version, user_options=options)
 288
 289         if not options.get('list_leases') or options['list_leases'] != 'leases':
 290             # get nodes
 291             nodes  = self.get_nodes(options)
 292             site_ids = []
 293             interface_ids = []
 294             tag_ids = []
 295             nodes_dict = {}
 296             for sliver in slivers:
 297                 site_ids.append(sliver['site_id'])
 298                 interface_ids.extend(sliver['interface_ids'])
 299                 tag_ids.extend(sliver['node_tag_ids'])
 300                 nodes_dict[sliver['node_id']] = sliver
 301             sites = self.get_sites({'site_id': site_ids})
 302             interfaces = self.get_interfaces({'interface_id':interface_ids})
 303             node_tags = self.get_node_tags(tags_filter)
 304             pl_initscripts = self.get_pl_initscripts()
 305             # convert nodes to rspec nodes
 306             rspec_nodes = []
 307             for node in nodes:
 308                 rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
 309                 rspec_nodes.append(rspec_node)
 310             rspec.version.add_nodes(rspec_nodes)
 311
 312             # add links
 313             links = self.get_links(sites_dict, nodes_dict, interfaces)
 314             rspec.version.add_links(links)
 315         return rspec.toxml()
 316
 317     def describe(self, urns, version=None, options={}):
 318         # update nova connection
 319         tenant_name = OSXrn(xrn=urns[0], type='slice').get_tenant_name()
 320         self.driver.shell.nova_manager.connect(tenant=tenant_name)
 321
 322         version_manager = VersionManager()
 323         version = version_manager.get_version(version)
 324         rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 325         rspec = RSpec(version=version, user_options=options)
 326
 327         # get slivers
 328         geni_slivers = []
 329         slivers = self.get_slivers(urns, options)
 330         if len(slivers) == 0:
 331             raise SliverDoesNotExist("You have not allocated any slivers here")
 332         rspec.xml.set('expires',  datetime_to_string(utcparse(slivers[0]['expires'])))
 333
 334         if not options.get('list_leases') or options['list_leases'] != 'leases':
 335             # add slivers
 336             site_ids = []
 337             interface_ids = []
 338             tag_ids = []
 339             nodes_dict = {}
 340             for sliver in slivers:
 341                 site_ids.append(sliver['site_id'])
 342                 interface_ids.extend(sliver['interface_ids'])
 343                 tag_ids.extend(sliver['node_tag_ids'])
 344                 nodes_dict[sliver['node_id']] = sliver
 345             sites = self.get_sites({'site_id': site_ids})
 346             interfaces = self.get_interfaces({'interface_id':interface_ids})
 347             node_tags = self.get_node_tags(tags_filter)
 348             pl_initscripts = self.get_pl_initscripts()
 349             rspec_nodes = []
 350             for sliver in slivers:
 351                 if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
 352                     continue
 353                 rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts)
 354                 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node)
 355                 rspec_nodes.append(rspec_node)
 356                 geni_slivers.append(geni_sliver)
 357             rspec.version.add_nodes(rspec_nodes)
 358
 359             # add sliver defaults
 360             default_sliver = slivers.get(None, [])
 361             if default_sliver:
 362                 default_sliver_attribs = default_sliver.get('tags', [])
 363                 for attrib in default_sliver_attribs:
 364                     rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
 365
 366             # add links
 367             links = self.get_links(sites_dict, nodes_dict, interfaces)
 368             rspec.version.add_links(links)
 369
 370         if not options.get('list_leases') or options['list_leases'] != 'resources':
 371             leases = self.get_leases(slivers[0])
 372             rspec.version.add_leases(leases)
 373
 374
 375 <<<<<<< HEAD:sfa/planetlab/plaggregate.py
 376         return {'geni_urn': urns[0],
 377                 'geni_rspec': rspec.toxml(),
 378                 'geni_slivers': geni_slivers}
 379 =======
 380         return  {'geni_urn': urns[0],
 381                  'geni_rspec': rspec.toxml(),
 382                  'geni_slivers': geni_slivers}
 383 >>>>>>> 3acb26e0ac5bc5de68effe62d0878ba674e40369:sfa/planetlab/plaggregate.py
 384