X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=blobdiff_plain;f=sfa%2Fplanetlab%2Fplaggregate.py;fp=sfa%2Fplanetlab%2Fplaggregate.py;h=c6006fbbc5f8c936ca1d88e9e1daae501aa821f9;hp=47c637bc5573dc84c2b127aaf771fb747c96ee36;hb=1cc8e9613cab8b5b22478de369f259e591c54e6d;hpb=f357d5c677573e29f260f82318c9450119474dce

diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py
index 47c637bc..c6006fbb 100644
--- a/sfa/planetlab/plaggregate.py
+++ b/sfa/planetlab/plaggregate.py
@@ -1,17 +1,18 @@
 #!/usr/bin/python
+from collections import defaultdict
 from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
 from sfa.util.sfatime import utcparse, datetime_to_string
 from sfa.util.sfalogging import logger
-
+from sfa.util.faults import SliverDoesNotExist
 from sfa.rspecs.rspec import RSpec
 from sfa.rspecs.elements.hardware_type import HardwareType
-from sfa.rspecs.elements.node import Node
+from sfa.rspecs.elements.node import NodeElement
 from sfa.rspecs.elements.link import Link
 from sfa.rspecs.elements.sliver import Sliver
 from sfa.rspecs.elements.login import Login
 from sfa.rspecs.elements.location import Location
 from sfa.rspecs.elements.interface import Interface
-from sfa.rspecs.elements.services import Services
+from sfa.rspecs.elements.services import ServicesElement
 from sfa.rspecs.elements.pltag import PLTag
 from sfa.rspecs.elements.lease import Lease
 from sfa.rspecs.elements.granularity import Granularity
@@ -20,6 +21,9 @@ from sfa.rspecs.version_manager import VersionManager
 from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
 from sfa.planetlab.vlink import get_tc_rate
 from sfa.planetlab.topology import Topology
+from sfa.storage.alchemy import dbsession
+from sfa.storage.model import SliverAllocation
+
 
 import time
 
@@ -27,6 +31,15 @@ class PlAggregate:
 
     def __init__(self, driver):
         self.driver = driver
+
+    def get_nodes(self, options={}):
+        filter = {'peer_id': None}
+        geni_available = options.get('geni_available')    
+        if geni_available == True:
+            filter['boot_state'] = 'boot'
+        nodes = self.driver.shell.GetNodes(filter)
+       
+        return nodes  
  
     def get_sites(self, filter={}):
         sites = {}
@@ -99,164 +112,212 @@ class PlAggregate:
             pl_initscripts[initscript['initscript_id']] = initscript
         return pl_initscripts
 
+    def get_slivers(self, urns, options={}):
+        names = set()
+        slice_ids = set()
+        node_ids = []
+        for urn in urns:
+            xrn = PlXrn(xrn=urn)
+            if xrn.type == 'sliver':
+                 # id: slice_id-node_id
+                try:
+                    sliver_id_parts = xrn.get_sliver_id_parts()
+                    slice_id = int(sliver_id_parts[0]) 
+                    node_id = int(sliver_id_parts[1])
+                    slice_ids.add(slice_id) 
+                    node_ids.append(node_id)
+                except ValueError:
+                    pass 
+            else:  
+                names.add(xrn.pl_slicename())
 
-    def get_slice_and_slivers(self, slice_xrn):
-        """
-        Returns a dict of slivers keyed on the sliver's node_id
-        """
-        slivers = {}
-        slice = None
-        if not slice_xrn:
-            return (slice, slivers)
-        slice_urn = hrn_to_urn(slice_xrn, 'slice')
-        slice_hrn, _ = urn_to_hrn(slice_xrn)
-        slice_name = hrn_to_pl_slicename(slice_hrn)
-        slices = self.driver.shell.GetSlices(slice_name)
+        filter = {}
+        if names:
+            filter['name'] = list(names)
+        if slice_ids:
+            filter['slice_id'] = list(slice_ids)
+        # get slices
+        slices = self.driver.shell.GetSlices(filter)
         if not slices:
-            return (slice, slivers)
-        slice = slices[0]
-
-        # sort slivers by node id    
-        for node_id in slice['node_ids']:
-            sliver_xrn = Xrn(slice_urn, type='sliver', id=node_id)
-            sliver_xrn.set_authority(self.driver.hrn)
-            sliver = Sliver({'sliver_id': sliver_xrn.urn,
-                             'name': slice['name'],
-                             'type': 'plab-vserver', 
-                             'tags': []})
-            slivers[node_id]= sliver
-
-        # sort sliver attributes by node id    
-        tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']})
-        for tag in tags:
-            # most likely a default/global sliver attribute (node_id == None)
-            if tag['node_id'] not in slivers:
-                sliver_xrn = Xrn(slice_urn, type='sliver', id=tag['node_id'])
-                sliver_xrn.set_authority(self.driver.hrn)
-                sliver = Sliver({'sliver_id': sliver_xrn.urn,
-                                 'name': slice['name'],
-                                 'type': 'plab-vserver',
-                                 'tags': []})
-                slivers[tag['node_id']] = sliver
-            slivers[tag['node_id']]['tags'].append(tag)
+            return []
+        slice = slices[0]     
+        slice['hrn'] = PlXrn(auth=self.driver.hrn, slicename=slice['name']).hrn   
+
+        # get sliver users
+        persons = []
+        person_ids = []
+        for slice in slices:
+            person_ids.extend(slice['person_ids'])
+        if person_ids:
+            persons = self.driver.shell.GetPersons(person_ids)
+                 
+        # get user keys
+        keys = {}
+        key_ids = []
+        for person in persons:
+            key_ids.extend(person['key_ids'])
         
-        return (slice, slivers)
-
-    def get_nodes_and_links(self, slice_xrn, slice=None,slivers=[], options={}):
-        # if we are dealing with a slice that has no node just return 
-        # and empty list    
-        if slice_xrn:
-            if not slice or not slice['node_ids']:
-                return ([],[])
+        if key_ids:
+            key_list = self.driver.shell.GetKeys(key_ids)
+            for key in key_list:
+                keys[key['key_id']] = key  
+
+        # construct user key info
+        users = []
+        for person in persons:
+            name = person['email'][0:person['email'].index('@')]
+            user = {
+                'login': slice['name'], 
+                'user_urn': Xrn('%s.%s' % (self.driver.hrn, name), type='user').urn,
+                'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys]
+            }
+            users.append(user)
+
+        if node_ids:
+            node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']]
+            slice['node_ids'] = node_ids
+        tags_dict = self.get_slice_tags(slice)
+        nodes_dict = self.get_slice_nodes(slice, options)
+        slivers = []
+        for node in nodes_dict.values():
+            node.update(slice) 
+            node['tags'] = tags_dict[node['node_id']]
+            sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
+            node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
+            node['urn'] = node['sliver_id'] 
+            node['services_user'] = users
+            slivers.append(node)
+        return slivers
+
+    def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
+        rspec_node = NodeElement()
+        # xxx how to retrieve site['login_base']
+        site=sites[node['site_id']]
+        rspec_node['component_id'] = PlXrn(self.driver.hrn, hostname=node['hostname']).get_urn()
+        rspec_node['component_name'] = node['hostname']
+        rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
+        rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
+        # do not include boot state (<available> element) in the manifest rspec
+        rspec_node['boot_state'] = node['boot_state']
+        if node['boot_state'] == 'boot': 
+            rspec_node['available'] = 'true'
+        else:
+            rspec_node['available'] = 'false'
+        rspec_node['exclusive'] = 'false'
+        rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
+                                        HardwareType({'name': 'pc'})]
+        # only doing this because protogeni rspec needs
+        # to advertise available initscripts
+        rspec_node['pl_initscripts'] = pl_initscripts.values()
+        # add site/interface info to nodes.
+        # assumes that sites, interfaces and tags have already been prepared.
+        if site['longitude'] and site['latitude']:
+            location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
+            rspec_node['location'] = location
+        # Granularity
+        granularity = Granularity({'grain': grain})
+        rspec_node['granularity'] = granularity
+        rspec_node['interfaces'] = []
+        if_count=0
+        for if_id in node['interface_ids']:
+            interface = Interface(interfaces[if_id])
+            interface['ipv4'] = interface['ip']
+            interface['component_id'] = PlXrn(auth=self.driver.hrn,
+                                              interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
+            # interfaces in the manifest need a client id
+            if slice:
+                interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
+            rspec_node['interfaces'].append(interface)
+            if_count+=1
+        tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags]
+        rspec_node['tags'] = tags
+        return rspec_node
+
+    def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, \
+                             pl_initscripts, sliver_allocations):
+        # get the granularity in second for the reservation system
+        grain = self.driver.shell.GetLeaseGranularity()
+        rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
+        # xxx how to retrieve site['login_base']
+        rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
+        # remove interfaces from manifest
+        rspec_node['interfaces'] = []
+        # add sliver info
+        rspec_sliver = Sliver({'sliver_id': sliver['urn'],
+                         'name': sliver['name'],
+                         'type': 'plab-vserver',
+                         'tags': []})
+        rspec_node['sliver_id'] = rspec_sliver['sliver_id']
+        rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id
+        if sliver_allocations[sliver['urn']].component_id:
+            rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id
+        rspec_node['slivers'] = [rspec_sliver]
+
+        # slivers always provide the ssh service
+        login = Login({'authentication': 'ssh-keys', 
+                       'hostname': sliver['hostname'], 
+                       'port':'22', 
+                       'username': sliver['name'],
+                       'login': sliver['name']
+                      })
+        service = ServicesElement({'login': login,
+                            'services_user': sliver['services_user']})
+        rspec_node['services'] = [service]    
+        return rspec_node      
+
+    def get_slice_tags(self, slice):
+        slice_tag_ids = []
+        slice_tag_ids.extend(slice['slice_tag_ids'])
+        tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
+        # sorted by node_id
+        tags_dict = defaultdict(list)
+        for tag in tags:
+            tags_dict[tag['node_id']] = tag
+        return tags_dict
 
-        filter = {}
+    def get_slice_nodes(self, slice, options={}):
+        nodes_dict = {}
+        filter = {'peer_id': None}
         tags_filter = {}
-        if slice and 'node_ids' in slice and slice['node_ids']:
+        if slice and slice.get('node_ids'):
             filter['node_id'] = slice['node_ids']
-            tags_filter=filter.copy()
-
-        geni_available = options.get('geni_available')    
+        else:
+            # there are no nodes to look up
+            return nodes_dict
+        tags_filter=filter.copy()
+        geni_available = options.get('geni_available')
         if geni_available == True:
-            filter['boot_state'] = 'boot'     
-        
-        filter.update({'peer_id': None})
+            filter['boot_state'] = 'boot'
         nodes = self.driver.shell.GetNodes(filter)
-        
-        # get the granularity in second for the reservation system
-        grain = self.driver.shell.GetLeaseGranularity()
-       
-        site_ids = []
-        interface_ids = []
-        tag_ids = []
-        nodes_dict = {}
         for node in nodes:
-            site_ids.append(node['site_id'])
-            interface_ids.extend(node['interface_ids'])
-            tag_ids.extend(node['node_tag_ids'])
             nodes_dict[node['node_id']] = node
- 
-        # get sites
-        sites_dict  = self.get_sites({'site_id': site_ids}) 
-        # get interfaces
-        interfaces = self.get_interfaces({'interface_id':interface_ids}) 
-        # get tags
-        node_tags = self.get_node_tags(tags_filter)
-        # get initscripts
-        pl_initscripts = self.get_pl_initscripts()
-        
-        links = self.get_links(sites_dict, nodes_dict, interfaces)
-
-        rspec_nodes = []
-        for node in nodes:
-            # skip whitelisted nodes
-            if node['slice_ids_whitelist']:
-                if not slice or slice['slice_id'] not in node['slice_ids_whitelist']:
-                    continue
-            rspec_node = Node()
-            # xxx how to retrieve site['login_base']
-            site_id=node['site_id']
-            site=sites_dict[site_id]
-            rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
-            rspec_node['component_name'] = node['hostname']
-            rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
-            rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
-            # do not include boot state (<available> element) in the manifest rspec
-            if not slice:     
-                rspec_node['boot_state'] = node['boot_state']
-
-            #add the exclusive tag to distinguish between Shared and Reservable nodes
-            if node['node_type'] == 'reservable':
-                rspec_node['exclusive'] = 'true'
+        return nodes_dict
+
+    def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}):
+        if rspec_node['sliver_id'] in sliver_allocations:
+            # set sliver allocation and operational status
+            sliver_allocation = sliver_allocations[rspec_node['sliver_id']]
+            if sliver_allocation:
+                allocation_status = sliver_allocation.allocation_state
+                if allocation_status == 'geni_allocated':
+                    op_status =  'geni_pending_allocation'
+                elif allocation_status == 'geni_provisioned':
+                    if rspec_node['boot_state'] == 'boot':
+                        op_status = 'geni_ready'
+                    else:
+                        op_status = 'geni_failed'
+                else:
+                    op_status = 'geni_unknown'
             else:
-                rspec_node['exclusive'] = 'false'
-
-            rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
-                                            HardwareType({'name': 'pc'})]
-            # only doing this because protogeni rspec needs
-            # to advertise available initscripts 
-            rspec_node['pl_initscripts'] = pl_initscripts.values()
-             # add site/interface info to nodes.
-            # assumes that sites, interfaces and tags have already been prepared.
-            site = sites_dict[node['site_id']]
-            if site['longitude'] and site['latitude']:  
-                location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
-                rspec_node['location'] = location
-            # Granularity
-            granularity = Granularity({'grain': grain})
-            rspec_node['granularity'] = granularity
-
-            rspec_node['interfaces'] = []
-            if_count=0
-            for if_id in node['interface_ids']:
-                interface = Interface(interfaces[if_id]) 
-                interface['ipv4'] = interface['ip']
-                interface['component_id'] = PlXrn(auth=self.driver.hrn, 
-                                                  interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
-                # interfaces in the manifest need a client id
-                if slice:
-                    interface['client_id'] = "%s:%s" % (node['node_id'], if_id)            
-                rspec_node['interfaces'].append(interface)
-                if_count+=1
-
-            tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']\
-                    if tag_id in node_tags]
-            rspec_node['tags'] = tags
-            if node['node_id'] in slivers:
-                # add sliver info
-                sliver = slivers[node['node_id']]
-                rspec_node['sliver_id'] = sliver['sliver_id']
-                rspec_node['slivers'] = [sliver]
-                for tag in sliver['tags']:
-                    if tag['tagname'] == 'client_id':
-                         rspec_node['client_id'] = tag['value']
-                
-                # slivers always provide the ssh service
-                login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']})
-                service = Services({'login': login})
-                rspec_node['services'] = [service]
-            rspec_nodes.append(rspec_node)
-        return (rspec_nodes, links)
-             
+                allocation_status = 'geni_unallocated'    
+        # required fields
+        geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
+                       'geni_expires': rspec_node['expires'],
+                       'geni_allocation_status' : allocation_status,
+                       'geni_operational_status': op_status,
+                       'geni_error': '',
+                       }
+        return geni_sliver        
 
     def get_leases(self, slice_xrn=None, slice=None, options={}):
         
@@ -288,7 +349,6 @@ class PlAggregate:
             site_id=lease['site_id']
             site=sites_dict[site_id]
 
-            #rspec_lease['lease_id'] = lease['lease_id']
             rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
             if slice_xrn:
                 slice_urn = slice_xrn
@@ -303,39 +363,116 @@ class PlAggregate:
         return rspec_leases
 
     
-    def get_rspec(self, slice_xrn=None, version = None, options={}):
+    def list_resources(self, version = None, options={}):
 
         version_manager = VersionManager()
         version = version_manager.get_version(version)
-        if not slice_xrn:
-            rspec_version = version_manager._get_version(version.type, version.version, 'ad')
-        else:
-            rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
-
-        slice, slivers = self.get_slice_and_slivers(slice_xrn)
+        rspec_version = version_manager._get_version(version.type, version.version, 'ad')
         rspec = RSpec(version=rspec_version, user_options=options)
-        if slice and 'expires' in slice:
-            rspec.xml.set('expires',  datetime_to_string(utcparse(slice['expires'])))
-
-        if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'leases':
-            if slice_xrn and not slivers:
-                nodes, links = [], []
-            else:
-                nodes, links = self.get_nodes_and_links(slice_xrn, slice, slivers, options)
-            rspec.version.add_nodes(nodes)
+       
+        if not options.get('list_leases') or options['list_leases'] != 'leases':
+            # get nodes
+            nodes  = self.get_nodes(options)
+            site_ids = []
+            interface_ids = []
+            tag_ids = []
+            nodes_dict = {}
+            for node in nodes:
+                site_ids.append(node['site_id'])
+                interface_ids.extend(node['interface_ids'])
+                tag_ids.extend(node['node_tag_ids'])
+                nodes_dict[node['node_id']] = node
+            sites = self.get_sites({'site_id': site_ids})
+            interfaces = self.get_interfaces({'interface_id':interface_ids})
+            node_tags = self.get_node_tags({'node_tag_id': tag_ids})
+            pl_initscripts = self.get_pl_initscripts()
+            # convert nodes to rspec nodes
+            rspec_nodes = []
+            for node in nodes:
+                rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
+                rspec_nodes.append(rspec_node)
+            rspec.version.add_nodes(rspec_nodes)
+
+            # add links
+            links = self.get_links(sites, nodes_dict, interfaces)        
             rspec.version.add_links(links)
-            # add sliver defaults
-            default_sliver = slivers.get(None, [])
-            if default_sliver:
-                default_sliver_attribs = default_sliver.get('tags', [])
-                for attrib in default_sliver_attribs:
-                    logger.info(attrib)
-                    rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
-        
+
         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
            leases = self.get_leases(slice_xrn, slice)
            rspec.version.add_leases(leases)
 
         return rspec.toxml()
 
+    def describe(self, urns, version=None, options={}):
+        version_manager = VersionManager()
+        version = version_manager.get_version(version)
+        rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
+        rspec = RSpec(version=rspec_version, user_options=options)
+
+        # get slivers
+        geni_slivers = []
+        slivers = self.get_slivers(urns, options)
+        if slivers:
+            rspec_expires = datetime_to_string(utcparse(slivers[0]['expires']))
+        else:
+            rspec_expires = datetime_to_string(utcparse(time.time()))      
+        rspec.xml.set('expires',  rspec_expires)
+
+        # lookup the sliver allocations
+        geni_urn = urns[0]
+        sliver_ids = [sliver['sliver_id'] for sliver in slivers]
+        constraint = SliverAllocation.sliver_id.in_(sliver_ids)
+        sliver_allocations = dbsession.query(SliverAllocation).filter(constraint)
+        sliver_allocation_dict = {}
+        for sliver_allocation in sliver_allocations:
+            geni_urn = sliver_allocation.slice_urn
+            sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
+      
+        if not options.get('list_leases') or options['list_leases'] != 'leases':
+            # add slivers
+            site_ids = []
+            interface_ids = []
+            tag_ids = []
+            nodes_dict = {}
+            for sliver in slivers:
+                site_ids.append(sliver['site_id'])
+                interface_ids.extend(sliver['interface_ids'])
+                tag_ids.extend(sliver['node_tag_ids'])
+                nodes_dict[sliver['node_id']] = sliver
+            sites = self.get_sites({'site_id': site_ids})
+            interfaces = self.get_interfaces({'interface_id':interface_ids})
+            node_tags = self.get_node_tags({'node_tag_id': tag_ids})
+            pl_initscripts = self.get_pl_initscripts()
+            rspec_nodes = []
+            for sliver in slivers:
+                if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
+                    continue
+                rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, 
+                                                       pl_initscripts, sliver_allocation_dict)
+                # manifest node element shouldn't contain available attribute
+                rspec_node.pop('available')
+                rspec_nodes.append(rspec_node) 
+                geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict)
+                geni_slivers.append(geni_sliver)
+            rspec.version.add_nodes(rspec_nodes)
+
+            # add sliver defaults
+            #default_sliver = slivers.get(None, [])
+            #if default_sliver:
+            #    default_sliver_attribs = default_sliver.get('tags', [])
+            #    for attrib in default_sliver_attribs:
+            #        rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
+
+            # add links 
+            links = self.get_links(sites, nodes_dict, interfaces)        
+            rspec.version.add_links(links)
+
+        if not options.get('list_leases') or options['list_leases'] != 'resources':
+            if slivers:
+                leases = self.get_leases(slivers[0])
+                rspec.version.add_leases(leases)
 
+               
+        return {'geni_urn': geni_urn, 
+                'geni_rspec': rspec.toxml(),
+                'geni_slivers': geni_slivers}