renamed sfaticket from util/ to trust/

[sfa.git] / sfa / plc / api.py

diff --git a/sfa/plc/api.py b/sfa/plc/api.py
index 636a000..8a03a87 100644 (file)
--- a/sfa/plc/api.py
+++ b/sfa/plc/api.py
@@ -1,41 +1,89 @@
 #
 #

-# Geniwrapper XML-RPC and SOAP interfaces
-#
-### $Id$
-### $URL$
+# SFA XML-RPC and SOAP interfaces

 #
 
 import sys
 import os
 import traceback
 import string
 #
 
 import sys
 import os
 import traceback
 import string

+import datetime

 import xmlrpclib
 
 import xmlrpclib
 

-from sfa.trust.auth import Auth
-from sfa.util.config import *

 from sfa.util.faults import *
 from sfa.util.faults import *

-from sfa.util.debug import *
-from sfa.trust.rights import *
-from sfa.trust.credential import *
-from sfa.trust.certificate import *
-from sfa.util.misc import *
-from sfa.util.sfalogging import *
-from sfa.util.genitable import *

 from sfa.util.api import *
 from sfa.util.api import *

-
-class GeniAPI(BaseAPI):
+from sfa.util.config import *
+from sfa.util.sfalogging import logger
+import sfa.util.xmlrpcprotocol as xmlrpcprotocol
+from sfa.trust.auth import Auth
+from sfa.trust.rights import Right, Rights, determine_rights
+from sfa.trust.credential import Credential,Keypair
+from sfa.trust.certificate import Certificate
+from sfa.util.xrn import get_authority, hrn_to_urn
+from sfa.util.plxrn import hostname_to_hrn, hrn_to_pl_slicename, hrn_to_pl_slicename, slicename_to_hrn
+from sfa.util.nodemanager import NodeManager
+try:
+    from collections import defaultdict
+except:
+    class defaultdict(dict):
+        def __init__(self, default_factory=None, *a, **kw):
+            if (default_factory is not None and
+                not hasattr(default_factory, '__call__')):
+                raise TypeError('first argument must be callable')
+            dict.__init__(self, *a, **kw)
+            self.default_factory = default_factory
+        def __getitem__(self, key):
+            try:
+                return dict.__getitem__(self, key)
+            except KeyError:
+                return self.__missing__(key)
+        def __missing__(self, key):
+            if self.default_factory is None:
+                raise KeyError(key)
+            self[key] = value = self.default_factory()
+            return value
+        def __reduce__(self):
+            if self.default_factory is None:
+                args = tuple()
+            else:
+                args = self.default_factory,
+            return type(self), args, None, None, self.items()
+        def copy(self):
+            return self.__copy__()
+        def __copy__(self):
+            return type(self)(self.default_factory, self)
+        def __deepcopy__(self, memo):
+            import copy
+            return type(self)(self.default_factory,
+                              copy.deepcopy(self.items()))
+        def __repr__(self):
+            return 'defaultdict(%s, %s)' % (self.default_factory,
+                                            dict.__repr__(self))
+## end of http://code.activestate.com/recipes/523034/ }}}
+
+def list_to_dict(recs, key):
+    """
+    convert a list of dictionaries into a dictionary keyed on the 
+    specified dictionary key 
+    """
+    keys = [rec[key] for rec in recs]
+    return dict(zip(keys, recs))
+
+class SfaAPI(BaseAPI):

 
     # flat list of method names
     import sfa.methods
     methods = sfa.methods.all
     
 
     # flat list of method names
     import sfa.methods
     methods = sfa.methods.all
     

-    def __init__(self, config = "/etc/sfa/sfa_config", encoding = "utf-8", methods='sfa.methods', 
-                 peer_cert = None, interface = None, key_file = None, cert_file = None):
-        BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert,
-                         interface=interface, key_file=key_file, cert_file=cert_file)
+    def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", 
+                 methods='sfa.methods', peer_cert = None, interface = None, 
+                key_file = None, cert_file = None, cache = None):
+        BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, \
+                         peer_cert=peer_cert, interface=interface, key_file=key_file, \
+                         cert_file=cert_file, cache=cache)

  
         self.encoding = encoding
  
         self.encoding = encoding

-
+        from sfa.util.table import SfaTable
+        self.SfaTable = SfaTable

         # Better just be documenting the API
         if config is None:
             return
         # Better just be documenting the API
         if config is None:
             return


@@ -49,86 +97,115 @@ class GeniAPI(BaseAPI):
         self.cert_file = cert_file
         self.cert = Certificate(filename=self.cert_file)
         self.credential = None
         self.cert_file = cert_file
         self.cert = Certificate(filename=self.cert_file)
         self.credential = None

-        

         # Initialize the PLC shell only if SFA wraps a myPLC
         # Initialize the PLC shell only if SFA wraps a myPLC

-        rspec_type = self.config.get_aggregate_rspec_type()
-        if (rspec_type == 'pl' or rspec_type == 'vini'):
+        rspec_type = self.config.get_aggregate_type()
+        if (rspec_type == 'pl' or rspec_type == 'vini' or \
+            rspec_type == 'eucalyptus' or rspec_type == 'max'):

             self.plshell = self.getPLCShell()
             self.plshell = self.getPLCShell()

-            self.plshell_version = self.getPLCShellVersion()
+            self.plshell_version = "4.3"

 
         self.hrn = self.config.SFA_INTERFACE_HRN
         self.time_format = "%Y-%m-%d %H:%M:%S"
 
         self.hrn = self.config.SFA_INTERFACE_HRN
         self.time_format = "%Y-%m-%d %H:%M:%S"

-        self.logger=get_sfa_logger()

 
 

+    

     def getPLCShell(self):
         self.plauth = {'Username': self.config.SFA_PLC_USER,
                        'AuthMethod': 'password',
                        'AuthString': self.config.SFA_PLC_PASSWORD}
     def getPLCShell(self):
         self.plauth = {'Username': self.config.SFA_PLC_USER,
                        'AuthMethod': 'password',
                        'AuthString': self.config.SFA_PLC_PASSWORD}

-        try:
-            self.plshell_type = 'direct'
-            import PLC.Shell
-            shell = PLC.Shell.Shell(globals = globals())
-            shell.AuthCheck(self.plauth)
-            return shell
-        except ImportError:
-            self.plshell_type = 'xmlrpc' 
-            # connect via xmlrpc
-            url = self.config.SFA_PLC_URL
-            shell = xmlrpclib.Server(url, verbose = 0, allow_none = True)
-            shell.AuthCheck(self.plauth)
-            return shell
-
-    def getPLCShellVersion(self):
-        # We need to figure out what version of PLCAPI we are talking to.
-        # Some calls we need to make later will be different depending on
-        # the api version. 
-        try:
-            # This is probably a bad way to determine api versions
-            # but its easy and will work for now. Lets try to make 
-            # a call that only exists is PLCAPI.4.3. If it fails, we
-            # can assume the api version is 4.2
-            self.plshell.GetTagTypes(self.plauth)
-            return '4.3'
-        except:
-            return '4.2'
-            

 
 

-    def getCredential(self):
-        if self.interface in ['registry']:
-            return self.getCredentialFromLocalRegistry()
+        # The native shell (PLC.Shell.Shell) is more efficient than xmlrpc,
+        # but it leaves idle db connections open. use xmlrpc until we can figure
+        # out why PLC.Shell.Shell doesn't close db connection properly     
+        #try:
+        #    sys.path.append(os.path.dirname(os.path.realpath("/usr/bin/plcsh")))
+        #    self.plshell_type = 'direct'
+        #    import PLC.Shell
+        #    shell = PLC.Shell.Shell(globals = globals())
+        #except:
+        
+        self.plshell_type = 'xmlrpc' 
+        url = self.config.SFA_PLC_URL
+        shell = xmlrpclib.Server(url, verbose = 0, allow_none = True)
+        return shell
+
+    def get_server(self, interface, cred, timeout=30):
+        """
+        Returns a connection to the specified interface. Use the specified
+        credential to determine the caller and look for the caller's key/cert 
+        in the registry hierarchy cache. 
+        """       
+        from sfa.trust.hierarchy import Hierarchy
+        if not isinstance(cred, Credential):
+            cred_obj = Credential(string=cred)

         else:
         else:

-            return self.getCredentialFromRegistry()
-    
-    def getCredentialFromRegistry(self):
-        """ 
-        Get our credential from a remote registry using a geniclient connection
+            cred_obj = cred
+        caller_gid = cred_obj.get_gid_caller()
+        hierarchy = Hierarchy()
+        auth_info = hierarchy.get_auth_info(caller_gid.get_hrn())
+        key_file = auth_info.get_privkey_filename()
+        cert_file = auth_info.get_gid_filename()
+        server = interface.get_server(key_file, cert_file, timeout)
+        return server
+               
+        
+    def getCredential(self):
+        """
+        Return a valid credential for this interface. 

         """
         type = 'authority'
         path = self.config.SFA_DATA_DIR
         filename = ".".join([self.interface, self.hrn, type, "cred"])
         cred_filename = path + os.sep + filename
         """
         type = 'authority'
         path = self.config.SFA_DATA_DIR
         filename = ".".join([self.interface, self.hrn, type, "cred"])
         cred_filename = path + os.sep + filename

-        try:
-            credential = Credential(filename = cred_filename)
-            return credential.save_to_string(save_parents=True)
-        except IOError:
-            from sfa.server.registry import Registries
-            registries = Registries(self)
-            registry = registries[self.hrn]
-           cert_string=self.cert.save_to_string(save_parents=True)
-            # get self credential
-            arg_list = [cert_string,type,self.hrn]
-            request_hash=self.key.compute_hash(arg_list)
-            self_cred = registry.get_self_credential(cert_string, type, self.hrn, request_hash)
-            # get credential
-            arg_list = [self_cred,type,self.hrn]
-            request_hash=self.key.compute_hash(arg_list)
-            cred = registry.get_credential(self_cred, type, self.hrn, request_hash)
-            
-            # save cred to file
-            Credential(string=cred).save_to_file(cred_filename, save_parents=True)
-            return cred
+        cred = None
+        if os.path.isfile(cred_filename):
+            cred = Credential(filename = cred_filename)
+            # make sure cred isnt expired
+            if not cred.get_expiration or \
+               datetime.datetime.utcnow() < cred.get_expiration():    
+                return cred.save_to_string(save_parents=True)
+
+        # get a new credential
+        if self.interface in ['registry']:
+            cred =  self.__getCredentialRaw()
+        else:
+            cred =  self.__getCredential()
+        cred.save_to_file(cred_filename, save_parents=True)

 
 

-    def getCredentialFromLocalRegistry(self):
+        return cred.save_to_string(save_parents=True)
+
+
+    def getDelegatedCredential(self, creds):
+        """
+        Attempt to find a credential delegated to us in
+        the specified list of creds.
+        """
+        from sfa.trust.hierarchy import Hierarchy
+        if creds and not isinstance(creds, list): 
+            creds = [creds]
+        hierarchy = Hierarchy()
+                
+        delegated_cred = None
+        for cred in creds:
+            if hierarchy.auth_exists(Credential(string=cred).get_gid_caller().get_hrn()):
+                delegated_cred = cred
+                break
+        return delegated_cred
+ 
+    def __getCredential(self):
+        """ 
+        Get our credential from a remote registry 
+        """
+        from sfa.server.registry import Registries
+        registries = Registries()
+        registry = registries.get_server(self.hrn, self.key_file, self.cert_file)
+        cert_string=self.cert.save_to_string(save_parents=True)
+        # get self credential
+        self_cred = registry.GetSelfCredential(cert_string, self.hrn, 'authority')
+        # get credential
+        cred = registry.GetCredential(self_cred, self.hrn, 'authority')
+        return Credential(string=cred)
+
+    def __getCredentialRaw(self):

         """
         Get our current credential directly from the local registry.
         """
         """
         Get our current credential directly from the local registry.
         """


@@ -140,8 +217,8 @@ class GeniAPI(BaseAPI):
         if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
             auth_hrn = hrn
         auth_info = self.auth.get_auth_info(auth_hrn)
         if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
             auth_hrn = hrn
         auth_info = self.auth.get_auth_info(auth_hrn)

-        table = GeniTable()
-        records = table.findObjects(hrn)
+        table = self.SfaTable()
+        records = table.findObjects({'hrn': hrn, 'type': 'authority+sa'})

         if not records:
             raise RecordNotFound
         record = records[0]
         if not records:
             raise RecordNotFound
         record = records[0]


@@ -150,19 +227,14 @@ class GeniAPI(BaseAPI):
         new_cred = Credential(subject = object_gid.get_subject())
         new_cred.set_gid_caller(object_gid)
         new_cred.set_gid_object(object_gid)
         new_cred = Credential(subject = object_gid.get_subject())
         new_cred.set_gid_caller(object_gid)
         new_cred.set_gid_object(object_gid)

-        new_cred.set_issuer(key=auth_info.get_pkey_object(), subject=auth_hrn)
-        new_cred.set_pubkey(object_gid.get_pubkey())
+        new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename())
+        

         r1 = determine_rights(type, hrn)
         new_cred.set_privileges(r1)
         r1 = determine_rights(type, hrn)
         new_cred.set_privileges(r1)

-
-        auth_kind = "authority,ma,sa"
-
-        new_cred.set_parent(self.auth.hierarchy.get_auth_cred(auth_hrn, kind=auth_kind))
-

         new_cred.encode()
         new_cred.sign()
 
         new_cred.encode()
         new_cred.sign()
 

-        return new_cred.save_to_string(save_parents=True)
+        return new_cred

    
 
     def loadCredential (self):
    
 
     def loadCredential (self):


@@ -180,16 +252,18 @@ class GeniAPI(BaseAPI):
         except IOError:
             self.credential = self.getCredentialFromRegistry()
 
         except IOError:
             self.credential = self.getCredentialFromRegistry()
 

+
+

     ##
     ##

-    # Convert geni fields to PLC fields for use when registering up updating
+    # Convert SFA fields to PLC fields for use when registering up updating

     # registry record in the PLC database
     #
     # @param type type of record (user, slice, ...)
     # @param hrn human readable name
     # registry record in the PLC database
     #
     # @param type type of record (user, slice, ...)
     # @param hrn human readable name

-    # @param geni_fields dictionary of geni fields
+    # @param sfa_fields dictionary of SFA fields

     # @param pl_fields dictionary of PLC fields (output)
 
     # @param pl_fields dictionary of PLC fields (output)
 

-    def geni_fields_to_pl_fields(self, type, hrn, record):
+    def sfa_fields_to_pl_fields(self, type, hrn, record):

 
         def convert_ints(tmpdict, int_fields):
             for field in int_fields:
 
         def convert_ints(tmpdict, int_fields):
             for field in int_fields:


@@ -208,11 +282,13 @@ class GeniAPI(BaseAPI):
                pl_record["url"] = record["url"]
            if "description" in record:
                pl_record["description"] = record["description"]
                pl_record["url"] = record["url"]
            if "description" in record:
                pl_record["description"] = record["description"]

+           if "expires" in record:
+               pl_record["expires"] = int(record["expires"])

 
         elif type == "node":
             if not "hostname" in pl_record:
                 if not "hostname" in record:
 
         elif type == "node":
             if not "hostname" in pl_record:
                 if not "hostname" in record:

-                    raise MissingGeniInfo("hostname")
+                    raise MissingSfaInfo("hostname")

                 pl_record["hostname"] = record["hostname"]
             if not "model" in pl_record:
                 pl_record["model"] = "geni"
                 pl_record["hostname"] = record["hostname"]
             if not "model" in pl_record:
                 pl_record["model"] = "geni"


@@ -234,9 +310,9 @@ class GeniAPI(BaseAPI):
 
         return pl_record
 
 
         return pl_record
 

-    def fill_record_pl_info(self, record):
+    def fill_record_pl_info(self, records):

         """
         """

-        Fill in the planetlab specific fields of a Geni record. This
+        Fill in the planetlab specific fields of a SFA record. This

         involves calling the appropriate PLC method to retrieve the 
         database record for the object.
         
         involves calling the appropriate PLC method to retrieve the 
         database record for the object.
         


@@ -244,132 +320,278 @@ class GeniAPI(BaseAPI):
     
         @param record: record to fill in field (in/out param)     
         """
     
         @param record: record to fill in field (in/out param)     
         """

-        type = record['type']
-        pointer = record['pointer']
-        auth_hrn = self.hrn
-        login_base = ''
-        # records with pointer==-1 do not have plc info associated with them.
-        # for example, the top level authority records which are
-        # authorities, but not PL "sites"
-        if pointer == -1:
-            record.update({})
-            return
+        # get ids by type
+        node_ids, site_ids, slice_ids = [], [], [] 
+        person_ids, key_ids = [], []
+        type_map = {'node': node_ids, 'authority': site_ids,
+                    'slice': slice_ids, 'user': person_ids}
+                  
+        for record in records:
+            for type in type_map:
+                if type == record['type']:
+                    type_map[type].append(record['pointer'])
+
+        # get pl records
+        nodes, sites, slices, persons, keys = {}, {}, {}, {}, {}
+        if node_ids:
+            node_list = self.plshell.GetNodes(self.plauth, node_ids)
+            nodes = list_to_dict(node_list, 'node_id')
+        if site_ids:
+            site_list = self.plshell.GetSites(self.plauth, site_ids)
+            sites = list_to_dict(site_list, 'site_id')
+        if slice_ids:
+            slice_list = self.plshell.GetSlices(self.plauth, slice_ids)
+            slices = list_to_dict(slice_list, 'slice_id')
+        if person_ids:
+            person_list = self.plshell.GetPersons(self.plauth, person_ids)
+            persons = list_to_dict(person_list, 'person_id')
+            for person in persons:
+                key_ids.extend(persons[person]['key_ids'])
+
+        pl_records = {'node': nodes, 'authority': sites,
+                      'slice': slices, 'user': persons}
+
+        if key_ids:
+            key_list = self.plshell.GetKeys(self.plauth, key_ids)
+            keys = list_to_dict(key_list, 'key_id')
+
+        # fill record info
+        for record in records:
+            # records with pointer==-1 do not have plc info.
+            # for example, the top level authority records which are
+            # authorities, but not PL "sites"
+            if record['pointer'] == -1:
+                continue
+           
+            for type in pl_records:
+                if record['type'] == type:
+                    if record['pointer'] in pl_records[type]:
+                        record.update(pl_records[type][record['pointer']])
+                        break
+            # fill in key info
+            if record['type'] == 'user':
+                if 'key_ids' not in record:
+                    logger.info("user record has no 'key_ids' - need to import from myplc ?")
+                else:
+                    pubkeys = [keys[key_id]['key'] for key_id in record['key_ids'] if key_id in keys] 
+                    record['keys'] = pubkeys
+
+        # fill in record hrns
+        records = self.fill_record_hrns(records)   
+ 
+        return records

 
 

-        if (type in ["authority"]):
-            pl_res = self.plshell.GetSites(self.plauth, [pointer])
-        elif (type == "slice"):
-            pl_res = self.plshell.GetSlices(self.plauth, [pointer])
-        elif (type == "user"):
-            pl_res = self.plshell.GetPersons(self.plauth, [pointer])
-        elif (type == "node"):
-            pl_res = self.plshell.GetNodes(self.plauth, [pointer])
-        else:
-            raise UnknownGeniType(type)
-        
-        if not pl_res:
-            raise PlanetLabRecordDoesNotExist(record['hrn'])
+    def fill_record_hrns(self, records):
+        """
+        convert pl ids to hrns
+        """

 
 

+        # get ids
+        slice_ids, person_ids, site_ids, node_ids = [], [], [], []
+        for record in records:
+            if 'site_id' in record:
+                site_ids.append(record['site_id'])
+            if 'site_ids' in records:
+                site_ids.extend(record['site_ids'])
+            if 'person_ids' in record:
+                person_ids.extend(record['person_ids'])
+            if 'slice_ids' in record:
+                slice_ids.extend(record['slice_ids'])
+            if 'node_ids' in record:
+                node_ids.extend(record['node_ids'])
+
+        # get pl records
+        slices, persons, sites, nodes = {}, {}, {}, {}
+        if site_ids:
+            site_list = self.plshell.GetSites(self.plauth, site_ids, ['site_id', 'login_base'])
+            sites = list_to_dict(site_list, 'site_id')
+        if person_ids:
+            person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'email'])
+            persons = list_to_dict(person_list, 'person_id')
+        if slice_ids:
+            slice_list = self.plshell.GetSlices(self.plauth, slice_ids, ['slice_id', 'name'])
+            slices = list_to_dict(slice_list, 'slice_id')       
+        if node_ids:
+            node_list = self.plshell.GetNodes(self.plauth, node_ids, ['node_id', 'hostname'])
+            nodes = list_to_dict(node_list, 'node_id')
+       

         # convert ids to hrns
         # convert ids to hrns

-        pl_record = pl_res[0]
-        if 'site_id' in pl_record:
-            sites = self.plshell.GetSites(self.plauth, pl_record['site_id'], ['login_base'])
-            site = sites[0]
-            login_base = site['login_base']
-            pl_record['site'] = ".".join([auth_hrn, login_base])
-        if 'person_ids' in pl_record:
-            persons =  self.plshell.GetPersons(self.plauth, pl_record['person_ids'], ['email'])
-            emails = [person['email'] for person in persons]
-            usernames = [email.split('@')[0] for email in emails]
-            person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames]
-            pl_record['persons'] = person_hrns 
-        if 'slice_ids' in pl_record:
-            slices = self.plshell.GetSlices(self.plauth, pl_record['slice_ids'], ['name'])
-            slicenames = [slice['name'] for slice in slices]
-            slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
-            pl_record['slices'] = slice_hrns
-        if 'node_ids' in pl_record:
-            nodes = self.plshell.GetNodes(self.plauth, pl_record['node_ids'], ['hostname'])
-            hostnames = [node['hostname'] for node in nodes]
-            node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
-            pl_record['nodes'] = node_hrns
-        if 'site_ids' in pl_record:
-            sites = self.plshell.GetSites(self.plauth, pl_record['site_ids'], ['login_base'])
-            login_bases = [site['login_base'] for site in sites]
-            site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases]
-            pl_record['sites'] = site_hrns
-        if 'key_ids' in pl_record:
-            keys = self.plshell.GetKeys(self.plauth, pl_record['key_ids'])
-            pubkeys = []
-            if keys:
-                pubkeys = [key['key'] for key in keys]
-            pl_record['keys'] = pubkeys     
-
-        record.update(pl_record)
-
-
-
-    def fill_record_geni_info(self, record):
-        geni_info = {}
-        type = record['type']
-        table = GeniTable()
-        if (type == "slice"):
-            person_ids = record.get("person_ids", [])
-            persons = table.find({'type': 'user', 'pointer': person_ids})
-            researchers = [person['hrn'] for person in persons]
-            geni_info['researcher'] = researchers
-
-        elif (type == "authority"):
-            person_ids = record.get("person_ids", [])
-            persons = table.find({'type': 'user', 'pointer': person_ids})
-            persons_dict = {}
-            for person in persons:
-                persons_dict[person['pointer']] = person 
-            pl_persons = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'roles'])
-            pis, techs, admins = [], [], []
-            for person in pl_persons:
-                pointer = person['person_id']
-                
-                if pointer not in persons_dict:
-                    # this means there is not sfa record for this user
-                    continue    
-                hrn = persons_dict[pointer]['hrn']    
-                if 'pi' in person['roles']:
-                    pis.append(hrn)
-                if 'tech' in person['roles']:
-                    techs.append(hrn)
-                if 'admin' in person['roles']:
-                    admins.append(hrn)
+        for record in records:
+            # get all relevant data
+            type = record['type']
+            pointer = record['pointer']
+            auth_hrn = self.hrn
+            login_base = ''
+            if pointer == -1:
+                continue
+
+            if 'site_id' in record:
+                site = sites[record['site_id']]
+                login_base = site['login_base']
+                record['site'] = ".".join([auth_hrn, login_base])
+            if 'person_ids' in record:
+                emails = [persons[person_id]['email'] for person_id in record['person_ids'] \
+                          if person_id in  persons]
+                usernames = [email.split('@')[0] for email in emails]
+                person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames]
+                record['persons'] = person_hrns 
+            if 'slice_ids' in record:
+                slicenames = [slices[slice_id]['name'] for slice_id in record['slice_ids'] \
+                              if slice_id in slices]
+                slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
+                record['slices'] = slice_hrns
+            if 'node_ids' in record:
+                hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
+                             if node_id in nodes]
+                node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
+                record['nodes'] = node_hrns
+            if 'site_ids' in record:
+                login_bases = [sites[site_id]['login_base'] for site_id in record['site_ids'] \
+                               if site_id in sites]
+                site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases]
+                record['sites'] = site_hrns

             
             

-            geni_info['PI'] = pis
-            geni_info['operator'] = techs
-            geni_info['owner'] = admins
-            # xxx TODO: OrganizationName
-
-        elif (type == "node"):
-            geni_info['dns'] = record.get("hostname", "")
-            # xxx TODO: URI, LatLong, IP, DNS
-    
-        elif (type == "user"):
-            geni_info['email'] = record.get("email", "")
-            # xxx TODO: PostalAddress, Phone
+        return records   
+
+    def fill_record_sfa_info(self, records):

 
 

-        record.update(geni_info)
+        def startswith(prefix, values):
+            return [value for value in values if value.startswith(prefix)]

 
 

-    def fill_record_info(self, record):
+        # get person ids
+        person_ids = []
+        site_ids = []
+        for record in records:
+            person_ids.extend(record.get("person_ids", []))
+            site_ids.extend(record.get("site_ids", [])) 
+            if 'site_id' in record:
+                site_ids.append(record['site_id']) 
+        
+        # get all pis from the sites we've encountered
+        # and store them in a dictionary keyed on site_id 
+        site_pis = {}
+        if site_ids:
+            pi_filter = {'|roles': ['pi'], '|site_ids': site_ids} 
+            pi_list = self.plshell.GetPersons(self.plauth, pi_filter, ['person_id', 'site_ids'])
+            for pi in pi_list:
+                # we will need the pi's hrns also
+                person_ids.append(pi['person_id'])
+                
+                # we also need to keep track of the sites these pis
+                # belong to
+                for site_id in pi['site_ids']:
+                    if site_id in site_pis:
+                        site_pis[site_id].append(pi)
+                    else:
+                        site_pis[site_id] = [pi]
+                 
+        # get sfa records for all records associated with these records.   
+        # we'll replace pl ids (person_ids) with hrns from the sfa records
+        # we obtain
+        
+        # get the sfa records
+        table = self.SfaTable()
+        person_list, persons = [], {}
+        person_list = table.find({'type': 'user', 'pointer': person_ids})
+        # create a hrns keyed on the sfa record's pointer.
+        # Its possible for  multiple records to have the same pointer so
+        # the dict's value will be a list of hrns.
+        persons = defaultdict(list)
+        for person in person_list:
+            persons[person['pointer']].append(person)
+
+        # get the pl records
+        pl_person_list, pl_persons = [], {}
+        pl_person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'roles'])
+        pl_persons = list_to_dict(pl_person_list, 'person_id')
+
+        # fill sfa info
+        for record in records:
+            # skip records with no pl info (top level authorities)
+            #if record['pointer'] == -1:
+            #    continue 
+            sfa_info = {}
+            type = record['type']
+            if (type == "slice"):
+                # all slice users are researchers
+                record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
+                record['PI'] = []
+                record['researcher'] = []
+                for person_id in record.get('person_ids', []):
+                    hrns = [person['hrn'] for person in persons[person_id]]
+                    record['researcher'].extend(hrns)                
+
+                # pis at the slice's site
+                if 'site_id' in record and record['site_id'] in site_pis:
+                    pl_pis = site_pis[record['site_id']]
+                    pi_ids = [pi['person_id'] for pi in pl_pis]
+                    for person_id in pi_ids:
+                        hrns = [person['hrn'] for person in persons[person_id]]
+                        record['PI'].extend(hrns)
+                        record['geni_creator'] = record['PI'] 
+                
+            elif (type.startswith("authority")):
+                record['url'] = None
+                if record['hrn'] in self.aggregates:
+                    
+                    record['url'] = self.aggregates[record['hrn']].get_url()
+
+                if record['pointer'] != -1:
+                    record['PI'] = []
+                    record['operator'] = []
+                    record['owner'] = []
+                    for pointer in record.get('person_ids', []):
+                        if pointer not in persons or pointer not in pl_persons:
+                            # this means there is not sfa or pl record for this user
+                            continue   
+                        hrns = [person['hrn'] for person in persons[pointer]] 
+                        roles = pl_persons[pointer]['roles']   
+                        if 'pi' in roles:
+                            record['PI'].extend(hrns)
+                        if 'tech' in roles:
+                            record['operator'].extend(hrns)
+                        if 'admin' in roles:
+                            record['owner'].extend(hrns)
+                        # xxx TODO: OrganizationName
+            elif (type == "node"):
+                sfa_info['dns'] = record.get("hostname", "")
+                # xxx TODO: URI, LatLong, IP, DNS
+    
+            elif (type == "user"):
+                sfa_info['email'] = record.get("email", "")
+                sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
+                sfa_info['geni_certificate'] = record['gid'] 
+                # xxx TODO: PostalAddress, Phone
+            record.update(sfa_info)
+
+    def fill_record_info(self, records):

         """
         """

-        Given a geni record, fill in the PLC specific and Geni specific
+        Given a SFA record, fill in the PLC specific and SFA specific

         fields in the record. 
         """
         fields in the record. 
         """

-        self.fill_record_pl_info(record)
-        self.fill_record_geni_info(record)
+        if not isinstance(records, list):
+            records = [records]
+
+        self.fill_record_pl_info(records)
+        self.fill_record_sfa_info(records)

 
     def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc):
 
     def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc):

-        # get a list of the HRNs tht are members of the old and new records
+        # get a list of the HRNs that are members of the old and new records

         if oldRecord:
             oldList = oldRecord.get(listName, [])
         else:
             oldList = []     
         newList = record.get(listName, [])
         if oldRecord:
             oldList = oldRecord.get(listName, [])
         else:
             oldList = []     
         newList = record.get(listName, [])

+        # ugly hack to see what's next
+        def normalize (value):
+            from types import StringTypes
+            from sfa.util.sfalogging import logger
+            if isinstance(value,StringTypes): return value
+            elif isinstance(value,dict): 
+                newvalue=value['text']
+                logger.info("Normalizing %s=>%s"%(value,newvalue))
+                return newvalue
+        newList=[normalize(v) for v in newList]

 
         # if the lists are the same, then we don't have to update anything
         if (oldList == newList):
 
         # if the lists are the same, then we don't have to update anything
         if (oldList == newList):


@@ -378,7 +600,7 @@ class GeniAPI(BaseAPI):
         # build a list of the new person ids, by looking up each person to get
         # their pointer
         newIdList = []
         # build a list of the new person ids, by looking up each person to get
         # their pointer
         newIdList = []

-        table = GeniTable()
+        table = self.SfaTable()

         records = table.find({'type': 'user', 'hrn': newList})
         for rec in records:
             newIdList.append(rec['pointer'])
         records = table.find({'type': 'user', 'hrn': newList})
         for rec in records:
             newIdList.append(rec['pointer'])


@@ -416,7 +638,7 @@ class GeniAPI(BaseAPI):
 
 class ComponentAPI(BaseAPI):
 
 
 class ComponentAPI(BaseAPI):
 

-    def __init__(self, config = "/etc/sfa/sfa_config", encoding = "utf-8", methods='sfa.methods',
+    def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", methods='sfa.methods',

                  peer_cert = None, interface = None, key_file = None, cert_file = None):
 
         BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert,
                  peer_cert = None, interface = None, key_file = None, cert_file = None):
 
         BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert,


@@ -427,17 +649,7 @@ class ComponentAPI(BaseAPI):
         if config is None:
             return
 
         if config is None:
             return
 

-        self.nodemanager = self.getNodeManagerShell()
-
-    def getNodeManagerShell(self):
-        # do we need an auth ?
-        auth = {}
-        try:
-            nodemanager = xmlrpclib.ServerProxy('http://127.0.0.1:812')
-        except:
-            raise
-
-        return nodemanager
+        self.nodemanager = NodeManager(self.config)

 
     def sliver_exists(self):
         sliver_dict = self.nodemanager.GetXIDs()
 
     def sliver_exists(self):
         sliver_dict = self.nodemanager.GetXIDs()


@@ -445,3 +657,73 @@ class ComponentAPI(BaseAPI):
             return True
         else:
             return False
             return True
         else:
             return False

+
+    def get_registry(self):
+        addr, port = self.config.SFA_REGISTRY_HOST, self.config.SFA_REGISTRY_PORT
+        url = "http://%(addr)s:%(port)s" % locals()
+        server = xmlrpcprotocol.get_server(url, self.key_file, self.cert_file)
+        return server
+
+    def get_node_key(self):
+        # this call requires no authentication,
+        # so we can generate a random keypair here
+        subject="component"
+        (kfd, keyfile) = tempfile.mkstemp()
+        (cfd, certfile) = tempfile.mkstemp()
+        key = Keypair(create=True)
+        key.save_to_file(keyfile)
+        cert = Certificate(subject=subject)
+        cert.set_issuer(key=key, subject=subject)
+        cert.set_pubkey(key)
+        cert.sign()
+        cert.save_to_file(certfile)
+        registry = self.get_registry()
+        # the registry will scp the key onto the node
+        registry.get_key()        
+
+    def getCredential(self):
+        """
+        Get our credential from a remote registry
+        """
+        path = self.config.SFA_DATA_DIR
+        config_dir = self.config.config_path
+        cred_filename = path + os.sep + 'node.cred'
+        try:
+            credential = Credential(filename = cred_filename)
+            return credential.save_to_string(save_parents=True)
+        except IOError:
+            node_pkey_file = config_dir + os.sep + "node.key"
+            node_gid_file = config_dir + os.sep + "node.gid"
+            cert_filename = path + os.sep + 'server.cert'
+            if not os.path.exists(node_pkey_file) or \
+               not os.path.exists(node_gid_file):
+                self.get_node_key()
+
+            # get node's hrn
+            gid = GID(filename=node_gid_file)
+            hrn = gid.get_hrn()
+            # get credential from registry
+            cert_str = Certificate(filename=cert_filename).save_to_string(save_parents=True)
+            registry = self.get_registry()
+            cred = registry.GetSelfCredential(cert_str, hrn, 'node')
+            Credential(string=cred).save_to_file(credfile, save_parents=True)            
+
+            return cred
+
+    def clean_key_cred(self):
+        """
+        remove the existing keypair and cred  and generate new ones
+        """
+        files = ["server.key", "server.cert", "node.cred"]
+        for f in files:
+            filepath = KEYDIR + os.sep + f
+            if os.path.isfile(filepath):
+                os.unlink(f)
+
+        # install the new key pair
+        # GetCredential will take care of generating the new keypair
+        # and credential
+        self.get_node_key()
+        self.getCredential()
+
+