Merge branch 'geni-v3' of ssh://git.onelab.eu/git/sfa into geni-v3

[sfa.git] / sfa / planetlab / plaggregate.py

diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py
index 98232fd..9ab8c6d 100644 (file)
--- a/sfa/planetlab/plaggregate.py
+++ b/sfa/planetlab/plaggregate.py
@@ -1,27 +1,26 @@
 #!/usr/bin/python
 from collections import defaultdict
 #!/usr/bin/python
 from collections import defaultdict

-from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
+from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, get_authority, get_leaf

 from sfa.util.sfatime import utcparse, datetime_to_string
 from sfa.util.sfalogging import logger
 from sfa.util.faults import SliverDoesNotExist
 from sfa.rspecs.rspec import RSpec
 from sfa.rspecs.elements.hardware_type import HardwareType
 from sfa.util.sfatime import utcparse, datetime_to_string
 from sfa.util.sfalogging import logger
 from sfa.util.faults import SliverDoesNotExist
 from sfa.rspecs.rspec import RSpec
 from sfa.rspecs.elements.hardware_type import HardwareType

-from sfa.rspecs.elements.node import Node
+from sfa.rspecs.elements.node import NodeElement

 from sfa.rspecs.elements.link import Link
 from sfa.rspecs.elements.sliver import Sliver
 from sfa.rspecs.elements.login import Login
 from sfa.rspecs.elements.location import Location
 from sfa.rspecs.elements.interface import Interface
 from sfa.rspecs.elements.link import Link
 from sfa.rspecs.elements.sliver import Sliver
 from sfa.rspecs.elements.login import Login
 from sfa.rspecs.elements.location import Location
 from sfa.rspecs.elements.interface import Interface

-from sfa.rspecs.elements.services import Services
+from sfa.rspecs.elements.services import ServicesElement

 from sfa.rspecs.elements.pltag import PLTag
 from sfa.rspecs.elements.lease import Lease
 from sfa.rspecs.elements.granularity import Granularity
 from sfa.rspecs.version_manager import VersionManager
 
 from sfa.rspecs.elements.pltag import PLTag
 from sfa.rspecs.elements.lease import Lease
 from sfa.rspecs.elements.granularity import Granularity
 from sfa.rspecs.version_manager import VersionManager
 

-from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
+from sfa.planetlab.plxrn import PlXrn, hostname_to_urn

 from sfa.planetlab.vlink import get_tc_rate
 from sfa.planetlab.topology import Topology
 from sfa.planetlab.vlink import get_tc_rate
 from sfa.planetlab.topology import Topology

-from sfa.storage.alchemy import dbsession

 from sfa.storage.model import SliverAllocation
 
 
 from sfa.storage.model import SliverAllocation
 
 


@@ -32,7 +31,8 @@ class PlAggregate:
     def __init__(self, driver):
         self.driver = driver
 
     def __init__(self, driver):
         self.driver = driver
 

-    def get_nodes(self, options={}):
+    def get_nodes(self, options=None):
+        if options is None: options={}

         filter = {'peer_id': None}
         geni_available = options.get('geni_available')    
         if geni_available == True:
         filter = {'peer_id': None}
         geni_available = options.get('geni_available')    
         if geni_available == True:


@@ -41,13 +41,15 @@ class PlAggregate:
        
         return nodes  
  
        
         return nodes  
  

-    def get_sites(self, filter={}):
+    def get_sites(self, filter=None):
+        if filter is None: filter={}

         sites = {}
         for site in self.driver.shell.GetSites(filter):
             sites[site['site_id']] = site
         return sites
 
         sites = {}
         for site in self.driver.shell.GetSites(filter):
             sites[site['site_id']] = site
         return sites
 

-    def get_interfaces(self, filter={}):
+    def get_interfaces(self, filter=None):
+        if filter is None: filter={}

         interfaces = {}
         for interface in self.driver.shell.GetInterfaces(filter):
             iface = Interface()
         interfaces = {}
         for interface in self.driver.shell.GetInterfaces(filter):
             iface = Interface()


@@ -99,63 +101,111 @@ class PlAggregate:
 
         return links
 
 
         return links
 

-    def get_node_tags(self, filter={}):
+    def get_node_tags(self, filter=None):
+        if filter is None: filter={}

         node_tags = {}
         for node_tag in self.driver.shell.GetNodeTags(filter):
             node_tags[node_tag['node_tag_id']] = node_tag
         return node_tags
 
         node_tags = {}
         for node_tag in self.driver.shell.GetNodeTags(filter):
             node_tags[node_tag['node_tag_id']] = node_tag
         return node_tags
 

-    def get_pl_initscripts(self, filter={}):
+    def get_pl_initscripts(self, filter=None):
+        if filter is None: filter={}

         pl_initscripts = {}
         filter.update({'enabled': True})
         for initscript in self.driver.shell.GetInitScripts(filter):
             pl_initscripts[initscript['initscript_id']] = initscript
         return pl_initscripts
 
         pl_initscripts = {}
         filter.update({'enabled': True})
         for initscript in self.driver.shell.GetInitScripts(filter):
             pl_initscripts[initscript['initscript_id']] = initscript
         return pl_initscripts
 

-    def get_slivers(self, urns, options={}):
+    def get_slivers(self, urns, options=None):
+        if options is None: options={}

         names = set()
         slice_ids = set()
         node_ids = []
         names = set()
         slice_ids = set()
         node_ids = []

+        slice_hrn = None

         for urn in urns:
             xrn = PlXrn(xrn=urn)
             if xrn.type == 'sliver':
                  # id: slice_id-node_id
         for urn in urns:
             xrn = PlXrn(xrn=urn)
             if xrn.type == 'sliver':
                  # id: slice_id-node_id

-                id_parts = xrn.leaf.split('-')
-                slice_ids.add(id_parts[0]) 
-                node_ids.append(id_parts[1])
+                try:
+                    sliver_id_parts = xrn.get_sliver_id_parts()
+                    slice_id = int(sliver_id_parts[0]) 
+                    node_id = int(sliver_id_parts[1])
+                    slice_ids.add(slice_id) 
+                    node_ids.append(node_id)
+                except ValueError:
+                    pass 

             else:  
             else:  

-                names.add(xrn.pl_slicename())
-            if xrn.id:
-                ids.add(xrn.id)
+                slice_hrn = xrn.get_hrn()

 
         filter = {}
 
         filter = {}

-        if names:
-            filter['name'] = list(names)
+        filter['peer_id'] = None

         if slice_ids:
             filter['slice_id'] = list(slice_ids)
         if slice_ids:
             filter['slice_id'] = list(slice_ids)

-        slices = self.driver.shell.GetSlices(filter)
+        # get all slices
+        all_slices = self.driver.shell.GetSlices(filter, ['slice_id', 'name', 'hrn', 'person_ids', 'node_ids', 'slice_tag_ids', 'expires'])
+        if slice_hrn:
+            slices = [slice for slice in all_slices if slice['hrn'] == slice_hrn]
+        else:
+            slices = all_slices
+      

         if not slices:
             return []
         if not slices:
             return []

-        slice = slices[0]
+        slice = slices[0]     
+        slice['hrn'] = slice_hrn   
+
+        # get sliver users
+        persons = []
+        person_ids = []
+        for slice in slices:
+            person_ids.extend(slice['person_ids'])
+        if person_ids:
+            persons = self.driver.shell.GetPersons(person_ids)
+                 
+        # get user keys
+        keys = {}
+        key_ids = []
+        for person in persons:
+            key_ids.extend(person['key_ids'])
+        
+        if key_ids:
+            key_list = self.driver.shell.GetKeys(key_ids)
+            for key in key_list:
+                keys[key['key_id']] = key  
+
+        # construct user key info
+        users = []
+        for person in persons:
+            person_urn = hrn_to_urn(self.driver.shell.GetPersonHrn(int(person['person_id'])), 'user')
+            user = {
+                'login': slice['name'], 
+                'user_urn': person_urn,
+                'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys]
+            }
+            users.append(user)
+

         if node_ids:
         if node_ids:

+            node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']]

             slice['node_ids'] = node_ids
         tags_dict = self.get_slice_tags(slice)
         nodes_dict = self.get_slice_nodes(slice, options)
         slivers = []
         for node in nodes_dict.values():
             slice['node_ids'] = node_ids
         tags_dict = self.get_slice_tags(slice)
         nodes_dict = self.get_slice_nodes(slice, options)
         slivers = []
         for node in nodes_dict.values():

-            node.update(slices[0]) 
+            node.update(slice) 

             node['tags'] = tags_dict[node['node_id']]
             sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
             node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
             node['urn'] = node['sliver_id'] 
             node['tags'] = tags_dict[node['node_id']]
             sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
             node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
             node['urn'] = node['sliver_id'] 

+            node['services_user'] = users

             slivers.append(node)
         return slivers
 
             slivers.append(node)
         return slivers
 

-    def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
-        rspec_node = Node()
+    def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=None, grain=None, options=None):
+        if pl_initscripts is None: pl_initscripts=[]
+        if options is None: options={}
+        rspec_node = NodeElement()

         # xxx how to retrieve site['login_base']
         site=sites[node['site_id']]
         # xxx how to retrieve site['login_base']
         site=sites[node['site_id']]

-        rspec_node['component_id'] = PlXrn(self.driver.hrn, hostname=node['hostname']).get_urn()
+        rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])

         rspec_node['component_name'] = node['hostname']
         rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
         rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
         rspec_node['component_name'] = node['hostname']
         rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
         rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')


@@ -165,7 +215,13 @@ class PlAggregate:
             rspec_node['available'] = 'true'
         else:
             rspec_node['available'] = 'false'
             rspec_node['available'] = 'true'
         else:
             rspec_node['available'] = 'false'

-        rspec_node['exclusive'] = 'false'
+
+        #distinguish between Shared and Reservable nodes
+        if node['node_type'] == 'reservable':
+            rspec_node['exclusive'] = 'true'
+        else:
+            rspec_node['exclusive'] = 'false'
+

         rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
                                         HardwareType({'name': 'pc'})]
         # only doing this because protogeni rspec needs
         rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
                                         HardwareType({'name': 'pc'})]
         # only doing this because protogeni rspec needs


@@ -191,7 +247,7 @@ class PlAggregate:
                 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
             rspec_node['interfaces'].append(interface)
             if_count+=1
                 interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
             rspec_node['interfaces'].append(interface)
             if_count+=1

-        tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']]
+        tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags]

         rspec_node['tags'] = tags
         return rspec_node
 
         rspec_node['tags'] = tags
         return rspec_node
 


@@ -210,12 +266,21 @@ class PlAggregate:
                          'type': 'plab-vserver',
                          'tags': []})
         rspec_node['sliver_id'] = rspec_sliver['sliver_id']
                          'type': 'plab-vserver',
                          'tags': []})
         rspec_node['sliver_id'] = rspec_sliver['sliver_id']

-        rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id
+        if sliver['urn'] in sliver_allocations:
+            rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id
+            if sliver_allocations[sliver['urn']].component_id:
+                rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id

         rspec_node['slivers'] = [rspec_sliver]
 
         # slivers always provide the ssh service
         rspec_node['slivers'] = [rspec_sliver]
 
         # slivers always provide the ssh service

-        login = Login({'authentication': 'ssh-keys', 'hostname': sliver['hostname'], 'port':'22', 'username': sliver['name']})
-        service = Services({'login': login})
+        login = Login({'authentication': 'ssh-keys', 
+                       'hostname': sliver['hostname'], 
+                       'port':'22', 
+                       'username': sliver['name'],
+                       'login': sliver['name']
+                      })
+        service = ServicesElement({'login': login,
+                            'services_user': sliver['services_user']})

         rspec_node['services'] = [service]    
         return rspec_node      
 
         rspec_node['services'] = [service]    
         return rspec_node      
 


@@ -229,7 +294,8 @@ class PlAggregate:
             tags_dict[tag['node_id']] = tag
         return tags_dict
 
             tags_dict[tag['node_id']] = tag
         return tags_dict
 

-    def get_slice_nodes(self, slice, options={}):
+    def get_slice_nodes(self, slice, options=None):
+        if options is None: options={}

         nodes_dict = {}
         filter = {'peer_id': None}
         tags_filter = {}
         nodes_dict = {}
         filter = {'peer_id': None}
         tags_filter = {}


@@ -247,7 +313,8 @@ class PlAggregate:
             nodes_dict[node['node_id']] = node
         return nodes_dict
 
             nodes_dict[node['node_id']] = node
         return nodes_dict
 

-    def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}):
+    def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations=None):
+        if sliver_allocations is None: sliver_allocations={}

         if rspec_node['sliver_id'] in sliver_allocations:
             # set sliver allocation and operational status
             sliver_allocation = sliver_allocations[rspec_node['sliver_id']]
         if rspec_node['sliver_id'] in sliver_allocations:
             # set sliver allocation and operational status
             sliver_allocation = sliver_allocations[rspec_node['sliver_id']]


@@ -264,16 +331,20 @@ class PlAggregate:
                     op_status = 'geni_unknown'
             else:
                 allocation_status = 'geni_unallocated'    
                     op_status = 'geni_unknown'
             else:
                 allocation_status = 'geni_unallocated'    

+        else:
+            allocation_status = 'geni_unallocated'
+            op_status = 'geni_failed'

         # required fields
         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
                        'geni_expires': rspec_node['expires'],
                        'geni_allocation_status' : allocation_status,
                        'geni_operational_status': op_status,
         # required fields
         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
                        'geni_expires': rspec_node['expires'],
                        'geni_allocation_status' : allocation_status,
                        'geni_operational_status': op_status,

-                       'geni_error': None,
+                       'geni_error': '',

                        }
         return geni_sliver        
 
                        }
         return geni_sliver        
 

-    def get_leases(self, slice=None, options={}):
+    def get_leases(self, slice=None, options=None):
+        if options is None: options={}

         
         now = int(time.time())
         filter={}
         
         now = int(time.time())
         filter={}


@@ -300,9 +371,8 @@ class PlAggregate:
             site_id=lease['site_id']
             site=sites_dict[site_id]
 
             site_id=lease['site_id']
             site=sites_dict[site_id]
 

-            rspec_lease['lease_id'] = lease['lease_id']
-            rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
-            slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
+            rspec_lease['component_id'] = hrn_to_urn(self.driver.shell.GetNodeHrn(lease['hostname']), 'node')
+            slice_hrn = self.driver.shell.GetSliceHrn(lease['slice_id'])

             slice_urn = hrn_to_urn(slice_hrn, 'slice')
             rspec_lease['slice_id'] = slice_urn
             rspec_lease['start_time'] = lease['t_from']
             slice_urn = hrn_to_urn(slice_hrn, 'slice')
             rspec_lease['slice_id'] = slice_urn
             rspec_lease['start_time'] = lease['t_from']


@@ -311,7 +381,8 @@ class PlAggregate:
         return rspec_leases
 
     
         return rspec_leases
 
     

-    def list_resources(self, version = None, options={}):
+    def list_resources(self, version = None, options=None):
+        if options is None: options={}

 
         version_manager = VersionManager()
         version = version_manager.get_version(version)
 
         version_manager = VersionManager()
         version = version_manager.get_version(version)


@@ -344,9 +415,15 @@ class PlAggregate:
             # add links
             links = self.get_links(sites, nodes_dict, interfaces)        
             rspec.version.add_links(links)
             # add links
             links = self.get_links(sites, nodes_dict, interfaces)        
             rspec.version.add_links(links)

+
+        if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
+           leases = self.get_leases()
+           rspec.version.add_leases(leases)
+

         return rspec.toxml()
 
         return rspec.toxml()
 

-    def describe(self, urns, version=None, options={}):
+    def describe(self, urns, version=None, options=None):
+        if options is None: options={}

         version_manager = VersionManager()
         version = version_manager.get_version(version)
         rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
         version_manager = VersionManager()
         version = version_manager.get_version(version)
         rspec_version = version_manager._get_version(version.type, version.version, 'manifest')


@@ -362,11 +439,13 @@ class PlAggregate:
         rspec.xml.set('expires',  rspec_expires)
 
         # lookup the sliver allocations
         rspec.xml.set('expires',  rspec_expires)
 
         # lookup the sliver allocations

+        geni_urn = urns[0]

         sliver_ids = [sliver['sliver_id'] for sliver in slivers]
         constraint = SliverAllocation.sliver_id.in_(sliver_ids)
         sliver_ids = [sliver['sliver_id'] for sliver in slivers]
         constraint = SliverAllocation.sliver_id.in_(sliver_ids)

-        sliver_allocations = dbsession.query(SliverAllocation).filter(constraint)
+        sliver_allocations = self.driver.api.dbsession().query(SliverAllocation).filter(constraint)

         sliver_allocation_dict = {}
         for sliver_allocation in sliver_allocations:
         sliver_allocation_dict = {}
         for sliver_allocation in sliver_allocations:

+            geni_urn = sliver_allocation.slice_urn

             sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
       
         if not options.get('list_leases') or options['list_leases'] != 'leases':
             sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
       
         if not options.get('list_leases') or options['list_leases'] != 'leases':


@@ -414,6 +493,6 @@ class PlAggregate:
                 rspec.version.add_leases(leases)
 
                
                 rspec.version.add_leases(leases)
 
                

-        return {'geni_urn': urns[0], 
+        return {'geni_urn': geni_urn, 

                 'geni_rspec': rspec.toxml(),
                 'geni_slivers': geni_slivers}
                 'geni_rspec': rspec.toxml(),
                 'geni_slivers': geni_slivers}