Merge branch 'master' of ssh://git.onelab.eu/git/sfa

diff --git a/sfa.spec b/sfa.spec
index 6a59992..ffc8d7e 100644 (file)
--- a/sfa.spec
+++ b/sfa.spec
@@ -1,6 +1,6 @@
 %define name sfa
 %define version 2.0
-%define taglevel 6
+%define taglevel 8
 
 %define release %{taglevel}%{?pldistro:.%{pldistro}}%{?date:.%{date}}
 %global python_sitearch        %( python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)" )
@@ -224,6 +224,18 @@ fi
 [ "$1" -ge "1" ] && service sfa-cm restart || :
 
 %changelog
+* Wed Jan 4 2012 Tony Mack <tmack@cs.princeton.edu> - sfa-2.0-8
+- bugfix: Fixed a bug in the sfa-import-plc.py script that caused the script to 
+  exit when it encountered a user with an invalid public key.
+- server: imporved logging in sfa-import-plc.py
+ 
+* Tue Jan 3 2012 Tony Mack <tmack@cs.princeton.edu> - sfa-2.0-7
+- bugfix: Fixed appending public keys in CreateSliver
+- bugfix: Fixed various bugs in the PGv2/GENIv3 request, advertisement and manifest rspecs.
+- client: -c --current option allows users to request the current/uncached rspec.
+- server: Added 'geni_api_versions' field to GetVersion() output.
+- server: Moved PLC specific code from sfa.importer.sfaImport to sfa.importer.sfa-import-plc.
+   
 * Fri Dec 16 2011 Thierry Parmentelat <thierry.parmentelat@sophia.inria.fr> - sfa-2.0-6
 - bugfix: sfi was not sending call_id with ListResources to v2 servers
 - SFA_API_DEBUG replaced with SFA_API_LOGLEVEL

diff --git a/sfa/importer/sfa-import-plc.py b/sfa/importer/sfa-import-plc.py
index ea5d2cd..723f473 100755 (executable)
--- a/sfa/importer/sfa-import-plc.py
+++ b/sfa/importer/sfa-import-plc.py
@@ -26,7 +26,6 @@ from sfa.storage.record import SfaRecord
 from sfa.trust.gid import create_uuid    
 from sfa.trust.certificate import convert_public_key, Keypair
 from sfa.importer.sfaImport import sfaImport, _cleanup_string
-from sfa.util.sfalogging import logger
 from sfa.plc.plshell import PlShell    
 
 def process_options():
@@ -78,13 +77,6 @@ def main():
     logger.setLevelFromOptVerbose(config.SFA_API_LOGLEVEL)
     shell = PlShell (config)
     
-    # special case for vini
-    if ".vini" in interface_hrn and interface_hrn.endswith('vini'):
-        # create a fake internet2 site first
-        i2site = {'name': 'Internet2', 'abbreviated_name': 'I2',
-                    'login_base': 'internet2', 'site_id': -1}
-        sfaImporter.import_site(interface_hrn, i2site)
-   
     # create dict of all existing sfa records
     existing_records = {}
     existing_hrns = []
@@ -135,11 +127,14 @@ def main():
     slices_dict = {}
     for slice in slices:
         slices_dict[slice['slice_id']] = slice
-    # start importing 
-    for site in sites:
-        site_hrn = _get_site_hrn(interface_hrn, site)
-        logger.info("Importing site: %s" % site_hrn)
 
+    # special case for vini
+    if ".vini" in interface_hrn and interface_hrn.endswith('vini'):
+        # create a fake internet2 site first
+        i2site = {'name': 'Internet2', 'abbreviated_name': 'I2',
+                    'login_base': 'internet2', 'site_id': -1}
+        site_hrn = _get_site_hrn(interface_hrn, i2site)
+        logger.info("Importing site: %s" % site_hrn)
         # import if hrn is not in list of existing hrns or if the hrn exists
         # but its not a site record
         if site_hrn not in existing_hrns or \
@@ -149,8 +144,36 @@ def main():
             if not sfaImporter.AuthHierarchy.auth_exists(urn):
                 sfaImporter.AuthHierarchy.create_auth(urn)
             auth_info = sfaImporter.AuthHierarchy.get_auth_info(urn)
-            auth_record = SfaRecord(hrn=site_hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id'])
-            auth_record.sync(verbose=True) 
+            auth_record = SfaRecord(hrn=site_hrn, gid=auth_info.get_gid_object(), \
+                                    type="authority", pointer=site['site_id'], 
+                                    authority=get_authority(site_hrn))
+            auth_record.sync(verbose=True)
+
+    # start importing 
+    for site in sites:
+        site_hrn = _get_site_hrn(interface_hrn, site)
+        logger.info("Importing site: %s" % site_hrn)
+
+        # import if hrn is not in list of existing hrns or if the hrn exists
+        # but its not a site record
+        if site_hrn not in existing_hrns or \
+           (site_hrn, 'authority') not in existing_records:
+            try:
+                logger.info("Import: site %s " % site_hrn)
+                urn = hrn_to_urn(site_hrn, 'authority')
+                if not sfaImporter.AuthHierarchy.auth_exists(urn):
+                    sfaImporter.AuthHierarchy.create_auth(urn)
+                auth_info = sfaImporter.AuthHierarchy.get_auth_info(urn)
+                auth_record = SfaRecord(hrn=site_hrn, gid=auth_info.get_gid_object(), \
+                                        type="authority", pointer=site['site_id'], 
+                                        authority=get_authority(site_hrn))
+                logger.info("Import: importing site: %s" % auth_record.summary_string())  
+                auth_record.sync()
+            except:
+                # if the site import fails then there is no point in trying to import the
+                # site's child records (node, slices, persons), so skip them.
+                logger.log_exc("Import: failed to import site. Skipping child records") 
+                continue 
              
         # import node records
         for node_id in site['node_ids']:
@@ -164,11 +187,16 @@ def main():
                 hrn = hrn[:64]
             if hrn not in existing_hrns or \
                (hrn, 'node') not in existing_records:
-                pkey = Keypair(create=True)
-                urn = hrn_to_urn(hrn, 'node')
-                node_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
-                node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id'], authority=get_authority(hrn))    
-                node_record.sync(verbose=True)
+                try:
+                    pkey = Keypair(create=True)
+                    urn = hrn_to_urn(hrn, 'node')
+                    node_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
+                    node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id'], authority=get_authority(hrn))    
+                    logger.info("Import: importing node: %s" % node_record.summary_string())  
+                    node_record.sync()
+                except:
+                    logger.log_exc("Import: failed to import node") 
+                    
 
         # import slices
         for slice_id in site['slice_ids']:
@@ -180,12 +208,16 @@ def main():
             #slicename = _cleanup_string(slicename)
             if hrn not in existing_hrns or \
                (hrn, 'slice') not in existing_records:
-                pkey = Keypair(create=True)
-                urn = hrn_to_urn(hrn, 'slice')
-                slice_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
-                slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'],
-                                         authority=get_authority(hrn))
-                slice_record.sync(verbose=True)
+                try:
+                    pkey = Keypair(create=True)
+                    urn = hrn_to_urn(hrn, 'slice')
+                    slice_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
+                    slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'],
+                                             authority=get_authority(hrn))
+                    logger.info("Import: importing slice: %s" % slice_record.summary_string())  
+                    slice_record.sync()
+                except:
+                    logger.log_exc("Import: failed to  import slice")
 
         # import persons
         for person_id in site['person_ids']:
@@ -211,22 +243,26 @@ def main():
 
             if hrn not in existing_hrns or \
                (hrn, 'user') not in existing_records or update_record:
-                if 'key_ids' in person and person['key_ids']:
-                    key = new_keys[0]
-                    try:
-                        pkey = convert_public_key(key)
-                    except:
-                        logger.warn('unable to convert public key for %s' % hrn)
-                        pkey = Keypair(create=True)
-                else:
-                    # the user has no keys. Creating a random keypair for the user's gid
-                    logger.warn("Import: person %s does not have a PL public key"%hrn)
-                    pkey = Keypair(create=True) 
-                urn = hrn_to_urn(hrn, 'user')
-                person_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
-                person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", \
-                                          pointer=person['person_id'], authority=get_authority(hrn))
-                person_record.sync(verbose=True)
+                try:
+                    if 'key_ids' in person and person['key_ids']:
+                        key = new_keys[0]
+                        try:
+                            pkey = convert_public_key(key)
+                        except:
+                            logger.warn('unable to convert public key for %s' % hrn)
+                            pkey = Keypair(create=True)
+                    else:
+                        # the user has no keys. Creating a random keypair for the user's gid
+                        logger.warn("Import: person %s does not have a PL public key"%hrn)
+                        pkey = Keypair(create=True) 
+                    urn = hrn_to_urn(hrn, 'user')
+                    person_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
+                    person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", \
+                                              pointer=person['person_id'], authority=get_authority(hrn))
+                    logger.info("Import: importing person: %s" % person_record.summary_string())  
+                    person_record.sync()
+                except:
+                    logger.log_exc("Import: failed to import person.") 
     
     # remove stale records    
     system_records = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
@@ -291,10 +327,13 @@ def main():
             continue 
         
         if not found:
-            record_object = existing_records[(record_hrn, type)]
-            record = SfaRecord(dict=record_object)
-            record.delete()
-                                   
+            try:
+                record_object = existing_records[(record_hrn, type)]
+                record = SfaRecord(dict=record_object)
+                logger.info("Import: deleting record: %s" % record.summary_string())
+                record.delete()
+            except:
+                logger.log_exc("Import: failded to delete record")                    
     # save pub keys
     logger.info('Import: saving current pub keys')
     save_keys(keys_filename, person_keys)                

diff --git a/sfa/importer/sfaImport.py b/sfa/importer/sfaImport.py
index 6cb47f5..74f1042 100644 (file)
--- a/sfa/importer/sfaImport.py
+++ b/sfa/importer/sfaImport.py
@@ -91,7 +91,8 @@ class sfaImport:
         # create the db record if it doesnt already exist    
         auth_info = self.AuthHierarchy.get_auth_info(hrn)
         auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1, authority=get_authority(hrn))
-        auth_record.sync(verbose=True)
+        self.logger.info("Import: importing %s " % auth_record.summary_string())
+        auth_record.sync()
 
     def create_sm_client_record(self):
         """
@@ -106,7 +107,8 @@ class sfaImport:
         auth_info = self.AuthHierarchy.get_auth_info(hrn)
         record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), \
                            type="user", pointer=-1, authority=get_authority(hrn))
-        record.sync(verbose=True)
+        self.logger.info("Import: importing %s " % record.summary_string())
+        record.sync()
 
     def create_interface_records(self):
         """
@@ -124,7 +126,8 @@ class sfaImport:
             gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
             interface_record = SfaRecord(hrn=hrn, type=interface, pointer=-1,
                                          gid = gid, authority=get_authority(hrn))
-            interface_record.sync(verbose=True)
+            self.logger.info("Import: importing %s " % interface_record.summary_string())
+            interface_record.sync()
              
     def delete_record(self, hrn, type):
         # delete the record

diff --git a/sfa/storage/record.py b/sfa/storage/record.py
index e164dda..87d4126 100644 (file)
--- a/sfa/storage/record.py
+++ b/sfa/storage/record.py
@@ -365,14 +365,14 @@ class SfaRecord(Row):
                     print "     %s: %s" % (key, self[key])
     
     def summary_string(self):
-        return "Record(record_id=%s, hrn=%s, type=%s, auth=%s, pointer=%s)" % \
-                (self.get('record_id'), self.get('hrn'), self.get('type'), self.get('auth'), \
+        return "Record(record_id=%s, hrn=%s, type=%s, authority=%s, pointer=%s)" % \
+                (self.get('record_id'), self.get('hrn'), self.get('type'), self.get('authority'), \
                  self.get('pointer'))
 
     def getdict(self):
         return dict(self)
    
-    def sync(self, verbose=False):
+    def sync(self):
         """ 
         Sync this record with the database.
         """ 
@@ -389,16 +389,12 @@ class SfaRecord(Row):
         existing_records = table.find(filter)
         if not existing_records:
             table.insert(self)
-            if verbose:
-                logger.info("Inserted record: %s" %self.summary_string()) 
         else:
             existing_record = existing_records[0]
             self['record_id'] = existing_record['record_id']
             table.update(self) 
-            if verbose:
-                logger.info("Updated record: %s" % self.summary_string()) 
 
-    def delete(self, verbose=False):
+    def delete(self):
         """
         Remove record from the database.
         """
@@ -414,8 +410,6 @@ class SfaRecord(Row):
         existing_records = table.find(filter)
         for record in existing_records:
             table.remove(record)
-            if verbose:
-                logger.info("Removed record: %s" % self.summary_string())    
 
 class UserRecord(SfaRecord):
 

diff --git a/sfa/trust/certificate.py b/sfa/trust/certificate.py
index f0a2d71..595812b 100644 (file)
--- a/sfa/trust/certificate.py
+++ b/sfa/trust/certificate.py
@@ -89,7 +89,7 @@ def convert_public_key(key):
 
     # we can only convert rsa keys
     if "ssh-dss" in key:
-        return None
+        raise Exception, "keyconvert: dss keys are not supported"  
 
     (ssh_f, ssh_fn) = tempfile.mkstemp()
     ssl_fn = tempfile.mktemp()
@@ -103,20 +103,22 @@ def convert_public_key(key):
     # that it can be expected to see why it failed.
     # TODO: for production, cleanup the temporary files
     if not os.path.exists(ssl_fn):
-        return None
+        raise Exception, "keyconvert: generated certificate not found. keyconvert may have failed." 
 
     k = Keypair()
     try:
         k.load_pubkey_from_file(ssl_fn)
+        return k
     except:
         logger.log_exc("convert_public_key caught exception")
-        k = None
+        raise
+    finally:
+        # remove the temporary files
+        if os.path.exists(ssh_fn):
+            os.remove(ssh_fn)
+        if os.path.exists(ssl_fn):
+            os.remove(ssl_fn)
 
-    # remove the temporary files
-    os.remove(ssh_fn)
-    os.remove(ssl_fn)
-
-    return k
 
 ##
 # Public-private key pairs are implemented by the Keypair class.