Adding more error check in OARrestapi.

[sfa.git] / sfa / importer / iotlabimporter.py

diff --git a/sfa/importer/iotlabimporter.py b/sfa/importer/iotlabimporter.py
index 6200435..8687437 100644 (file)
--- a/sfa/importer/iotlabimporter.py
+++ b/sfa/importer/iotlabimporter.py
@@ -1,40 +1,37 @@
+""" File defining the importer class and all the methods needed to import
+the nodes, users and slices from OAR and LDAP to the SFA database.
+Also creates the iotlab specific databse and table to keep track
+of which slice hrn contains which job.
+"""

 from sfa.util.config import Config
 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
 
 from sfa.iotlab.iotlabdriver import IotlabDriver
 from sfa.util.config import Config
 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
 
 from sfa.iotlab.iotlabdriver import IotlabDriver

-
+from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB

 from sfa.trust.certificate import Keypair, convert_public_key
 from sfa.trust.gid import create_uuid
 
 from sfa.storage.alchemy import dbsession
 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
 from sfa.trust.certificate import Keypair, convert_public_key
 from sfa.trust.gid import create_uuid
 
 from sfa.storage.alchemy import dbsession
 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \

-                                                    RegUser, RegKey
+    RegUser, RegKey

 
 
 from sqlalchemy.exc import SQLAlchemyError
 
 
 
 
 from sqlalchemy.exc import SQLAlchemyError
 
 

-
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

 class IotlabImporter:
     """
     IotlabImporter class, generic importer_class. Used to populate the SFA DB
     with iotlab resources' records.
 class IotlabImporter:
     """
     IotlabImporter class, generic importer_class. Used to populate the SFA DB
     with iotlab resources' records.

-=======
-class SlabImporter:
-    """
-    SlabImporter class, generic importer_class. Used to populate the SFA DB
-    with senslab resources' records.
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

     Used to update records when new resources, users or nodes, are added
     or deleted.
     """
 
     Used to update records when new resources, users or nodes, are added
     or deleted.
     """
 

-    def __init__ (self, auth_hierarchy, loc_logger):
+    def __init__(self, auth_hierarchy, loc_logger):

         """
         Sets and defines import logger and the authority name. Gathers all the
         """
         Sets and defines import logger and the authority name. Gathers all the

-        records already registerd in the SFA DB, broke them into 3 dicts,
-        by type and hrn, by email and by type and pointer.
+        records already registerd in the SFA DB, broke them into 3 dicts, by
+        type and hrn, by email and by type and pointer.

 
         :param auth_hierarchy: authority name
         :type auth_hierarchy: string
 
         :param auth_hierarchy: authority name
         :type auth_hierarchy: string


@@ -55,26 +52,25 @@ class SlabImporter:
         #create hash by (type,hrn)
         #used  to know if a given record is already known to SFA
         self.records_by_type_hrn = \
         #create hash by (type,hrn)
         #used  to know if a given record is already known to SFA
         self.records_by_type_hrn = \

-            dict([( (record.type,record.hrn), record) \
-                                        for record in self.all_records])
+            dict([((record.type, record.hrn), record)
+                 for record in self.all_records])

 
         self.users_rec_by_email = \
 
         self.users_rec_by_email = \

-            dict([ (record.email, record) \
-                for record in self.all_records if record.type == 'user'])
+            dict([(record.email, record)
+                 for record in self.all_records if record.type == 'user'])

 
         # create hash by (type,pointer)
         self.records_by_type_pointer = \
 
         # create hash by (type,pointer)
         self.records_by_type_pointer = \

-            dict([ ( (str(record.type), record.pointer) , record) \
-                for record in self.all_records  if record.pointer != -1])
-
+            dict([((str(record.type), record.pointer), record)
+                  for record in self.all_records if record.pointer != -1])

 
 
     @staticmethod
     def hostname_to_hrn_escaped(root_auth, hostname):
         """
 
 
 
     @staticmethod
     def hostname_to_hrn_escaped(root_auth, hostname):
         """
 

-        Returns a node's hrn based on its hostname and the root
-        authority and by removing special caracters from the hostname.
+        Returns a node's hrn based on its hostname and the root authority and by
+        removing special caracters from the hostname.

 
         :param root_auth: root authority name
         :param hostname: nodes's hostname
 
         :param root_auth: root authority name
         :param hostname: nodes's hostname


@@ -82,7 +78,7 @@ class SlabImporter:
         :type hostname: string
         :rtype: string
         """
         :type hostname: string
         :rtype: string
         """

-        return '.'.join( [root_auth, Xrn.escape(hostname)] )
+        return '.'.join([root_auth, Xrn.escape(hostname)])

 
 
     @staticmethod
 
 
     @staticmethod


@@ -95,29 +91,33 @@ class SlabImporter:
         :type person_hrn: string
         :rtype: string
         """
         :type person_hrn: string
         :rtype: string
         """

-        return  (person_hrn +'_slice')
+        return (person_hrn + '_slice')

 
 

-    def add_options (self, parser):
+    def add_options(self, parser):
+        """
+        .. warning:: not used
+        """

         # we don't have any options for now
         pass
 
     def find_record_by_type_hrn(self, record_type, hrn):
         """
         # we don't have any options for now
         pass
 
     def find_record_by_type_hrn(self, record_type, hrn):
         """

-
-        Returns the record associated with a given hrn and hrn type.
-        Returns None if the key tuple is not in dictionary.
+        Finds the record associated with the hrn and its type given in parameter
+        if the tuple (hrn, type hrn) is an existing key in the dictionary.

 
         :param record_type: the record's type (slice, node, authority...)
         :type  record_type: string
         :param hrn: Human readable name of the object's record
         :type hrn: string
 
         :param record_type: the record's type (slice, node, authority...)
         :type  record_type: string
         :param hrn: Human readable name of the object's record
         :type hrn: string

-        :rtype: RegUser if user, RegSlice if slice, RegNode if node...
-                or None if record does not exist.
+        :returns: Returns the record associated with a given hrn and hrn type.
+            Returns None if the key tuple is not in the dictionary.
+        :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
+            record does not exist.

 
         """
 
         """

-        return self.records_by_type_hrn.get ( (record_type, hrn), None)
+        return self.records_by_type_hrn.get((record_type, hrn), None)

 
 

-    def locate_by_type_pointer (self, record_type, pointer):
+    def locate_by_type_pointer(self, record_type, pointer):

         """
 
         Returns the record corresponding to the key pointer and record
         """
 
         Returns the record corresponding to the key pointer and record


@@ -127,15 +127,15 @@ class SlabImporter:
         :param record_type: the record's type (slice, node, authority...)
         :type  record_type: string
         :param pointer:Pointer to where the record is in the origin db,
         :param record_type: the record's type (slice, node, authority...)
         :type  record_type: string
         :param pointer:Pointer to where the record is in the origin db,

-        used in case the record comes from a trusted authority.
+            used in case the record comes from a trusted authority.

         :type pointer: integer
         :rtype: RegUser if user, RegSlice if slice, RegNode if node...
         :type pointer: integer
         :rtype: RegUser if user, RegSlice if slice, RegNode if node...

-                or None if record does not exist.
+            or None if record does not exist.

         """
         """

-        return self.records_by_type_pointer.get ( (record_type, pointer), None)
+        return self.records_by_type_pointer.get((record_type, pointer), None)

 
 
 
 

-    def update_just_added_records_dict (self, record):
+    def update_just_added_records_dict(self, record):

         """
 
         Updates the records_by_type_hrn dictionnary if the record has
         """
 
         Updates the records_by_type_hrn dictionnary if the record has


@@ -146,114 +146,30 @@ class SlabImporter:
         """
         rec_tuple = (record.type, record.hrn)
         if rec_tuple in self.records_by_type_hrn:
         """
         rec_tuple = (record.type, record.hrn)
         if rec_tuple in self.records_by_type_hrn:

-            self.logger.warning ("IotlabImporter.update_just_added_records_dict:\
-                        duplicate (%s,%s)"%rec_tuple)
+            self.logger.warning("IotlabImporter.update_just_added_records_dict:\
+                        duplicate (%s,%s)" % rec_tuple)

             return
             return

-        self.records_by_type_hrn [ rec_tuple ] = record
+        self.records_by_type_hrn[rec_tuple] = record

 
 

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-    def import_sites_and_nodes(self, iotlabdriver):
-=======
-    def import_sites_and_nodes(self, slabdriver):
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-        """
-
-        Gets all the sites and nodes from OAR, process the information,
-        creates hrns and RegAuthority for sites, and feed them to the database.
-        For each site, import the site's nodes to the DB by calling
-        import_nodes.
-
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-        :param iotlabdriver: IotlabDriver object, used to have access to iotlabdriver
-        methods and fetching info on sites and nodes.
-        :type iotlabdriver: IotlabDriver
-        """
-
-        sites_listdict  = iotlabdriver.iotlab_api.GetSites()
-        nodes_listdict  = iotlabdriver.iotlab_api.GetNodes()
-=======
-        :param slabdriver: SlabDriver object, used to have access to slabdriver
-        methods and fetching info on sites and nodes.
-        :type slabdriver: SlabDriver
-        """
-
-        sites_listdict  = slabdriver.slab_api.GetSites()
-        nodes_listdict  = slabdriver.slab_api.GetNodes()
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-        nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
-        for site in sites_listdict:
-            site_hrn = site['name']
-            site_record = self.find_record_by_type_hrn ('authority', site_hrn)
-            if not site_record:
-                try:
-                    urn = hrn_to_urn(site_hrn, 'authority')
-                    if not self.auth_hierarchy.auth_exists(urn):
-                        self.auth_hierarchy.create_auth(urn)
-
-                    auth_info = self.auth_hierarchy.get_auth_info(urn)
-                    site_record = RegAuthority(hrn=site_hrn, \
-                                               gid=auth_info.get_gid_object(),
-                                               pointer='-1',
-                                               authority=get_authority(site_hrn))
-                    site_record.just_created()
-                    dbsession.add(site_record)
-                    dbsession.commit()
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-                    self.logger.info("IotlabImporter: imported authority (site) \
-=======
-                    self.logger.info("SlabImporter: imported authority (site) \
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-                         %s" % site_record)
-                    self.update_just_added_records_dict(site_record)
-                except SQLAlchemyError:
-                    # if the site import fails then there is no point in
-                    # trying to import the
-                    # site's child records(node, slices, persons), so skip them.
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-                    self.logger.log_exc("IotlabImporter: failed to import site. \
-=======
-                    self.logger.log_exc("SlabImporter: failed to import site. \
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-                        Skipping child records")
-                    continue
-            else:
-                # xxx update the record ...
-                pass
-
-
-            site_record.stale = False
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-            self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver)
-
-            return

 
     def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver):
 
     def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver):

-=======
-            self.import_nodes(site['node_ids'], nodes_by_id, slabdriver)
-
-            return
-
-    def import_nodes(self, site_node_ids, nodes_by_id, slabdriver):
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

         """
 
         """
 

-        Creates appropriate hostnames and RegNode records for
-        each node in site_node_ids, based on the information given by the
-        dict nodes_by_id that was made from data from OAR.
-        Saves the records to the DB.
+        Creates appropriate hostnames and RegNode records for each node in
+        site_node_ids, based on the information given by the dict nodes_by_id
+        that was made from data from OAR. Saves the records to the DB.

 
         :param site_node_ids: site's node ids
         :type site_node_ids: list of integers
         :param nodes_by_id: dictionary , key is the node id, value is the a dict
 
         :param site_node_ids: site's node ids
         :type site_node_ids: list of integers
         :param nodes_by_id: dictionary , key is the node id, value is the a dict

-        with node information.
+            with node information.

         :type nodes_by_id: dictionary
         :type nodes_by_id: dictionary

-        :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver
-        attributes.
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-        :type iotlabdriver:IotlabDriver
-=======
-        :type slabdriver:SlabDriver
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
+        :param iotlabdriver: IotlabDriver object, used to have access to
+            iotlabdriver attributes.
+        :type iotlabdriver: IotlabDriver
+
+        :returns: None
+        :rtype: None

 
         """
 
 
         """
 


@@ -261,120 +177,147 @@ class SlabImporter:
             try:
                 node = nodes_by_id[node_id]
             except KeyError:
             try:
                 node = nodes_by_id[node_id]
             except KeyError:

-                self.logger.warning ("IotlabImporter: cannot find node_id %s \
-                        - ignored" %(node_id))
+                self.logger.warning("IotlabImporter: cannot find node_id %s \
+                        - ignored" % (node_id))

                 continue
             escaped_hrn =  \
                 continue
             escaped_hrn =  \

-            self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth, \
-            node['hostname'])
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-            self.logger.info("IOTLABIMPORTER node %s " %(node))
-=======
-            self.logger.info("SLABIMPORTER node %s " %(node))
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-            hrn =  node['hrn']
-
+                self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth,
+                                             node['hostname'])
+            self.logger.info("IOTLABIMPORTER node %s " % (node))
+            hrn = node['hrn']

 
             # xxx this sounds suspicious
             if len(hrn) > 64:
                 hrn = hrn[:64]
 
             # xxx this sounds suspicious
             if len(hrn) > 64:
                 hrn = hrn[:64]

-            node_record = self.find_record_by_type_hrn( 'node', hrn )
+            node_record = self.find_record_by_type_hrn('node', hrn)

             if not node_record:
                 pkey = Keypair(create=True)
                 urn = hrn_to_urn(escaped_hrn, 'node')
                 node_gid = \
             if not node_record:
                 pkey = Keypair(create=True)
                 urn = hrn_to_urn(escaped_hrn, 'node')
                 node_gid = \

-                    self.auth_hierarchy.create_gid(urn, \
-                    create_uuid(), pkey)
+                    self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)

 
 

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

                 def iotlab_get_authority(hrn):
                 def iotlab_get_authority(hrn):

-=======
-                def slab_get_authority(hrn):
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
+                    """ Gets the authority part in the hrn.
+                    :param hrn: hrn whose authority we are looking for.
+                    :type hrn: string
+                    :returns: splits the hrn using the '.' separator and returns
+                        the authority part of the hrn.
+                    :rtype: string
+
+                    """

                     return hrn.split(".")[0]
 
                 node_record = RegNode(hrn=hrn, gid=node_gid,
                     return hrn.split(".")[0]
 
                 node_record = RegNode(hrn=hrn, gid=node_gid,

-                                    pointer = '-1',
-                                    authority=iotlab_get_authority(hrn))
+                                      pointer='-1',
+                                      authority=iotlab_get_authority(hrn))

                 try:
 
                     node_record.just_created()
                     dbsession.add(node_record)
                     dbsession.commit()
                 try:
 
                     node_record.just_created()
                     dbsession.add(node_record)
                     dbsession.commit()

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-                    self.logger.info("IotlabImporter: imported node: %s" \
-                                % node_record)
-                    self.update_just_added_records_dict(node_record)
-                except SQLAlchemyError:
-                    self.logger.log_exc("IotlabImporter: \
-=======
-                    self.logger.info("SlabImporter: imported node: %s" \
-                                % node_record)
+                    self.logger.info("IotlabImporter: imported node: %s"
+                                     % node_record)

                     self.update_just_added_records_dict(node_record)
                 except SQLAlchemyError:
                     self.update_just_added_records_dict(node_record)
                 except SQLAlchemyError:

-                    self.logger.log_exc("SlabImporter: \
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-                                    failed to import node")
+                    self.logger.log_exc("IotlabImporter: failed to import node")

             else:
                 #TODO:  xxx update the record ...
                 pass
             node_record.stale = False
             else:
                 #TODO:  xxx update the record ...
                 pass
             node_record.stale = False

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

 
 

+    def import_sites_and_nodes(self, iotlabdriver):
+        """

 
 

-    def init_person_key (self, person, iotlab_key):
-=======
+        Gets all the sites and nodes from OAR, process the information,
+        creates hrns and RegAuthority for sites, and feed them to the database.
+        For each site, import the site's nodes to the DB by calling
+        import_nodes.

 
 

+        :param iotlabdriver: IotlabDriver object, used to have access to
+            iotlabdriver methods and fetching info on sites and nodes.
+        :type iotlabdriver: IotlabDriver
+        """

 
 

-    def init_person_key (self, person, slab_key):
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
+        sites_listdict = iotlabdriver.iotlab_api.GetSites()
+        nodes_listdict = iotlabdriver.iotlab_api.GetNodes()
+        nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
+        for site in sites_listdict:
+            site_hrn = site['name']
+            site_record = self.find_record_by_type_hrn ('authority', site_hrn)
+            self.logger.info("IotlabImporter: import_sites_and_nodes \
+                                    (site) %s \r\n " % site_record)
+            if not site_record:
+                try:
+                    urn = hrn_to_urn(site_hrn, 'authority')
+                    if not self.auth_hierarchy.auth_exists(urn):
+                        self.auth_hierarchy.create_auth(urn)
+
+                    auth_info = self.auth_hierarchy.get_auth_info(urn)
+                    site_record = \
+                        RegAuthority(hrn=site_hrn,
+                                     gid=auth_info.get_gid_object(),
+                                     pointer='-1',
+                                     authority=get_authority(site_hrn))
+                    site_record.just_created()
+                    dbsession.add(site_record)
+                    dbsession.commit()
+                    self.logger.info("IotlabImporter: imported authority \
+                                    (site) %s" % site_record)
+                    self.update_just_added_records_dict(site_record)
+                except SQLAlchemyError:
+                    # if the site import fails then there is no point in
+                    # trying to import the
+                    # site's child records(node, slices, persons), so skip them.
+                    self.logger.log_exc("IotlabImporter: failed to import \
+                        site. Skipping child records")
+                    continue
+            else:
+                # xxx update the record ...
+                pass
+
+            site_record.stale = False
+            self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver)
+
+        return
+
+
+
+    def init_person_key(self, person, iotlab_key):

         """
 
         Returns a tuple pubkey and pkey.
 
         :param person Person's data.
         :type person: dict
         """
 
         Returns a tuple pubkey and pkey.
 
         :param person Person's data.
         :type person: dict

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

         :param iotlab_key: SSH public key, from LDAP user's data.
         :param iotlab_key: SSH public key, from LDAP user's data.

-=======
-        :param slab_key: SSH public key, from LDAP user's data.
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-        RSA type supported.
+            RSA type supported.

         :type iotlab_key: string
         :rtype (string, Keypair)
         :type iotlab_key: string
         :rtype (string, Keypair)

+

         """
         pubkey = None
         """
         pubkey = None

-        if  person['pkey']:
+        if person['pkey']:

             # randomly pick first key in set
             # randomly pick first key in set

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

             pubkey = iotlab_key
             pubkey = iotlab_key

-=======
-            pubkey = slab_key
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

 
             try:
                 pkey = convert_public_key(pubkey)
             except TypeError:
                 #key not good. create another pkey
 
             try:
                 pkey = convert_public_key(pubkey)
             except TypeError:
                 #key not good. create another pkey

-                self.logger.warn('IotlabImporter: \
+                self.logger.warn("IotlabImporter: \

                                     unable to convert public \
                                     unable to convert public \

-                                    key for %s' %person['hrn'])
+                                    key for %s" % person['hrn'])

                 pkey = Keypair(create=True)
 
         else:
             # the user has no keys.
             #Creating a random keypair for the user's gid
             self.logger.warn("IotlabImporter: person %s does not have a  \
                 pkey = Keypair(create=True)
 
         else:
             # the user has no keys.
             #Creating a random keypair for the user's gid
             self.logger.warn("IotlabImporter: person %s does not have a  \

-                        public key" %(person['hrn']))
+                        public key" % (person['hrn']))

             pkey = Keypair(create=True)
         return (pubkey, pkey)
 
             pkey = Keypair(create=True)
         return (pubkey, pkey)
 

-
-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

     def import_persons_and_slices(self, iotlabdriver):
     def import_persons_and_slices(self, iotlabdriver):

-=======
-    def import_persons_and_slices(self, slabdriver):
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

         """
 
         Gets user data from LDAP, process the information.
         """
 
         Gets user data from LDAP, process the information.


@@ -385,31 +328,18 @@ class SlabImporter:
         import the user's slice onto the database as well by calling
         import_slice.
 
         import the user's slice onto the database as well by calling
         import_slice.
 

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-        :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver
-=======
-        :param slabdriver:SlabDriver object, used to have access to slabdriver
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-        attributes.
-        :type iotlabdriver:IotlabDriver
+        :param iotlabdriver: IotlabDriver object, used to have access to
+            iotlabdriver attributes.
+        :type iotlabdriver: IotlabDriver

         """
         """

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

         ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons()
         ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons()

-        self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n" \
-=======
-        ldap_person_listdict = slabdriver.slab_api.GetPersons()
-        self.logger.info("SLABIMPORT \t ldap_person_listdict %s \r\n" \
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-                %(ldap_person_listdict))
+        self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
+                         % (ldap_person_listdict))

 
          # import persons
 
          # import persons

-        for person in ldap_person_listdict :
+        for person in ldap_person_listdict:

 
 

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-            self.logger.info("IotlabImporter: person :" %(person))
-=======
-            self.logger.info("SlabImporter: person :" %(person))
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
+            self.logger.info("IotlabImporter: person :" % (person))

             if 'ssh-rsa' not in person['pkey']:
                 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
                 #won't be imported
             if 'ssh-rsa' not in person['pkey']:
                 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
                 #won't be imported


@@ -423,80 +353,66 @@ class SlabImporter:
             person_urn = hrn_to_urn(person_hrn, 'user')
 
 
             person_urn = hrn_to_urn(person_hrn, 'user')
 
 

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-            self.logger.info("IotlabImporter: users_rec_by_email %s " \
-=======
-            self.logger.info("SlabImporter: users_rec_by_email %s " \
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-                                            %(self.users_rec_by_email))
+            self.logger.info("IotlabImporter: users_rec_by_email %s "
+                             % (self.users_rec_by_email))

 
             #Check if user using person['email'] from LDAP is already registered
             #in SFA. One email = one person. In this case, do not create another
             #record for this person
 
             #Check if user using person['email'] from LDAP is already registered
             #in SFA. One email = one person. In this case, do not create another
             #record for this person

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

             #person_hrn returned by GetPerson based on iotlab root auth +
             #person_hrn returned by GetPerson based on iotlab root auth +

-=======
-            #person_hrn returned by GetPerson based on senslab root auth +
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

             #uid ldap
             user_record = self.find_record_by_type_hrn('user', person_hrn)
 
             #uid ldap
             user_record = self.find_record_by_type_hrn('user', person_hrn)
 

-            if not user_record and  person['email'] in self.users_rec_by_email:
+            if not user_record and person['email'] in self.users_rec_by_email:

                 user_record = self.users_rec_by_email[person['email']]
                 person_hrn = user_record.hrn
                 person_urn = hrn_to_urn(person_hrn, 'user')
 
 
                 user_record = self.users_rec_by_email[person['email']]
                 person_hrn = user_record.hrn
                 person_urn = hrn_to_urn(person_hrn, 'user')
 
 

-            slice_record = self.find_record_by_type_hrn ('slice', slice_hrn)
+            slice_record = self.find_record_by_type_hrn('slice', slice_hrn)

 
 

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

             iotlab_key = person['pkey']
             iotlab_key = person['pkey']

-=======
-            slab_key = person['pkey']
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

             # new person
             if not user_record:
                 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
             # new person
             if not user_record:
                 (pubkey, pkey) = self.init_person_key(person, iotlab_key)

-                if pubkey is not None and pkey is not None :
+                if pubkey is not None and pkey is not None:

                     person_gid = \
                     person_gid = \

-                    self.auth_hierarchy.create_gid(person_urn, \
-                    create_uuid(), pkey)
+                        self.auth_hierarchy.create_gid(person_urn,
+                                                       create_uuid(), pkey)

                     if person['email']:
                     if person['email']:

-                        self.logger.debug( "SLAB IMPORTER \
-                            PERSON EMAIL OK email %s " %(person['email']))
+                        self.logger.debug("IOTLAB IMPORTER \
+                            PERSON EMAIL OK email %s " % (person['email']))

                         person_gid.set_email(person['email'])
                         person_gid.set_email(person['email'])

-                        user_record = RegUser(hrn=person_hrn, \
-                                            gid=person_gid,
-                                            pointer='-1',
-                                            authority=get_authority(person_hrn),
-                                            email=person['email'])
+                        user_record = \
+                            RegUser(hrn=person_hrn,
+                                    gid=person_gid,
+                                    pointer='-1',
+                                    authority=get_authority(person_hrn),
+                                    email=person['email'])

                     else:
                     else:

-                        user_record = RegUser(hrn=person_hrn, \
-                                            gid=person_gid,
-                                            pointer='-1',
-                                            authority=get_authority(person_hrn))
+                        user_record = \
+                            RegUser(hrn=person_hrn,
+                                    gid=person_gid,
+                                    pointer='-1',
+                                    authority=get_authority(person_hrn))

 
                     if pubkey:
                         user_record.reg_keys = [RegKey(pubkey)]
                     else:
 
                     if pubkey:
                         user_record.reg_keys = [RegKey(pubkey)]
                     else:

-                        self.logger.warning("No key found for user %s" \
-                        %(user_record))
+                        self.logger.warning("No key found for user %s"
+                                            % (user_record))

 
                         try:
                             user_record.just_created()
                             dbsession.add (user_record)
                             dbsession.commit()
 
                         try:
                             user_record.just_created()
                             dbsession.add (user_record)
                             dbsession.commit()

-                            self.logger.info("IotlabImporter: imported person %s"\
-                            %(user_record))
-                            self.update_just_added_records_dict( user_record )
+                            self.logger.info("IotlabImporter: imported person \
+                                            %s" % (user_record))
+                            self.update_just_added_records_dict(user_record)

 
                         except SQLAlchemyError:
 
                         except SQLAlchemyError:

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

                             self.logger.log_exc("IotlabImporter: \
                             self.logger.log_exc("IotlabImporter: \

-=======
-                            self.logger.log_exc("SlabImporter: \
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-                                failed to import person  %s"%(person))
+                                failed to import person  %s" % (person))

             else:
                 # update the record ?
                 # if user's primary key has changed then we need to update
             else:
                 # update the record ?
                 # if user's primary key has changed then we need to update


@@ -504,25 +420,21 @@ class SlabImporter:
                 sfa_keys = user_record.reg_keys
 
                 new_key = False
                 sfa_keys = user_record.reg_keys
 
                 new_key = False

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py
-                if iotlab_key is not sfa_keys :
-=======
-                if slab_key is not sfa_keys :
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
+                if iotlab_key is not sfa_keys:

                     new_key = True
                 if new_key:
                     self.logger.info("IotlabImporter: \t \t USER UPDATE \
                     new_key = True
                 if new_key:
                     self.logger.info("IotlabImporter: \t \t USER UPDATE \

-                        person: %s" %(person['hrn']))
-                    (pubkey, pkey) = self.init_person_key (person, iotlab_key)
+                        person: %s" % (person['hrn']))
+                    (pubkey, pkey) = self.init_person_key(person, iotlab_key)

                     person_gid = \
                     person_gid = \

-                        self.auth_hierarchy.create_gid(person_urn, \
-                        create_uuid(), pkey)
+                        self.auth_hierarchy.create_gid(person_urn,
+                                                       create_uuid(), pkey)

                     if not pubkey:
                         user_record.reg_keys = []
                     else:
                         user_record.reg_keys = [RegKey(pubkey)]
                     if not pubkey:
                         user_record.reg_keys = []
                     else:
                         user_record.reg_keys = [RegKey(pubkey)]

-                    self.logger.info("IotlabImporter: updated person: %s" \
-                    % (user_record))
+                    self.logger.info("IotlabImporter: updated person: %s"
+                                     % (user_record))

 
                 if person['email']:
                     user_record.email = person['email']
 
                 if person['email']:
                     user_record.email = person['email']


@@ -531,12 +443,8 @@ class SlabImporter:
                 dbsession.commit()
                 user_record.stale = False
             except SQLAlchemyError:
                 dbsession.commit()
                 user_record.stale = False
             except SQLAlchemyError:

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

                 self.logger.log_exc("IotlabImporter: \
                 self.logger.log_exc("IotlabImporter: \

-=======
-                self.logger.log_exc("SlabImporter: \
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
-                failed to update person  %s"%(person))
+                failed to update person  %s"% (person))

 
             self.import_slice(slice_hrn, slice_record, user_record)
 
 
             self.import_slice(slice_hrn, slice_record, user_record)
 


@@ -559,55 +467,45 @@ class SlabImporter:
 
         .. todo::Update the record if a slice record already exists.
         """
 
         .. todo::Update the record if a slice record already exists.
         """

-        if not slice_record :
+        if not slice_record:

             pkey = Keypair(create=True)
             urn = hrn_to_urn(slice_hrn, 'slice')
             slice_gid = \
             pkey = Keypair(create=True)
             urn = hrn_to_urn(slice_hrn, 'slice')
             slice_gid = \

-                self.auth_hierarchy.create_gid(urn, \
-                create_uuid(), pkey)
-            slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid,
-                                        pointer='-1',
-                                        authority=get_authority(slice_hrn))
+                self.auth_hierarchy.create_gid(urn,
+                                               create_uuid(), pkey)
+            slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
+                                    pointer='-1',
+                                    authority=get_authority(slice_hrn))

             try:
                 slice_record.just_created()
                 dbsession.add(slice_record)
                 dbsession.commit()
 
 
             try:
                 slice_record.just_created()
                 dbsession.add(slice_record)
                 dbsession.commit()
 
 

-                self.update_just_added_records_dict ( slice_record )
+                self.update_just_added_records_dict(slice_record)

 
             except SQLAlchemyError:
 
             except SQLAlchemyError:

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

                 self.logger.log_exc("IotlabImporter: failed to import slice")
 
         #No slice update upon import in iotlab
                 self.logger.log_exc("IotlabImporter: failed to import slice")
 
         #No slice update upon import in iotlab

-=======
-                self.logger.log_exc("SlabImporter: failed to import slice")
-
-        #No slice update upon import in senslab
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

         else:
             # xxx update the record ...
         else:
             # xxx update the record ...

-            self.logger.warning ("Slice update not yet implemented")
+            self.logger.warning("Slice update not yet implemented")

             pass
         # record current users affiliated with the slice
 
 
             pass
         # record current users affiliated with the slice
 
 

-        slice_record.reg_researchers =  [user_record]
+        slice_record.reg_researchers = [user_record]

         try:
             dbsession.commit()
             slice_record.stale = False
         except SQLAlchemyError:
         try:
             dbsession.commit()
             slice_record.stale = False
         except SQLAlchemyError:

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

             self.logger.log_exc("IotlabImporter: failed to update slice")
             self.logger.log_exc("IotlabImporter: failed to update slice")

-=======
-            self.logger.log_exc("SlabImporter: failed to update slice")
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py

 
 
 
 

-    def run (self, options):
+    def run(self, options):

         """
         """

-        Create the special iotlab table, iotlab_xp, in the iotlab database.
+        Create the special iotlab table, testbed_xp, in the iotlab database.

         Import everything (users, slices, nodes and sites from OAR
         and LDAP) into the SFA database.
         Delete stale records that are no longer in OAR or LDAP.
         Import everything (users, slices, nodes and sites from OAR
         and LDAP) into the SFA database.
         Delete stale records that are no longer in OAR or LDAP.


@@ -616,15 +514,13 @@ class SlabImporter:
         """
         config = Config()
 
         """
         config = Config()
 

-<<<<<<< HEAD:sfa/importer/iotlabimporter.py

         iotlabdriver = IotlabDriver(config)
         iotlabdriver = IotlabDriver(config)

-
+        iotlab_db = TestbedAdditionalSfaDB(config)

         #Create special slice table for iotlab
 
         #Create special slice table for iotlab
 

-        if not iotlabdriver.db.exists('iotlab_xp'):
-            iotlabdriver.db.createtable()
-            self.logger.info ("IotlabImporter.run:  iotlab_xp table created ")
-
+        if not iotlab_db.exists('testbed_xp'):
+            iotlab_db.createtable()
+            self.logger.info("IotlabImporter.run:  testbed_xp table created ")

 
         # import site and node records in site into the SFA db.
         self.import_sites_and_nodes(iotlabdriver)
 
         # import site and node records in site into the SFA db.
         self.import_sites_and_nodes(iotlabdriver)


@@ -633,53 +529,30 @@ class SlabImporter:
 
          ### remove stale records
         # special records must be preserved
 
          ### remove stale records
         # special records must be preserved

-        system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth,  \
-                                        iotlabdriver.hrn+ '.slicemanager']
-=======
-        slabdriver = SlabDriver(config)
-
-        #Create special slice table for senslab
-
-        if not slabdriver.db.exists('slab_xp'):
-            slabdriver.db.createtable()
-            self.logger.info ("SlabImporter.run:  slab_xp table created ")
-
-
-        # import site and node records in site into the SFA db.
-        self.import_sites_and_nodes(slabdriver)
-        #import users and slice into the SFA DB.
-        self.import_persons_and_slices(slabdriver)
-
-         ### remove stale records
-        # special records must be preserved
-        system_hrns = [slabdriver.hrn, slabdriver.slab_api.root_auth,  \
-                                        slabdriver.hrn+ '.slicemanager']
->>>>>>> 3fe7429... SA:sfa/importer/slabimporter.py
+        system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth,
+                       iotlabdriver.hrn + '.slicemanager']

         for record in self.all_records:
             if record.hrn in system_hrns:
                 record.stale = False
             if record.peer_authority:
                 record.stale = False
 
         for record in self.all_records:
             if record.hrn in system_hrns:
                 record.stale = False
             if record.peer_authority:
                 record.stale = False
 

-

         for record in self.all_records:
             if record.type == 'user':
         for record in self.all_records:
             if record.type == 'user':

-                self.logger.info("IotlabImporter: stale records: hrn %s %s" \
-                                            %(record.hrn,record.stale) )
+                self.logger.info("IotlabImporter: stale records: hrn %s %s"
+                                 % (record.hrn, record.stale))

             try:
                 stale = record.stale
             try:
                 stale = record.stale

-            except :
+            except:

                 stale = True
                 stale = True

-                self.logger.warning("stale not found with %s"%record)
+                self.logger.warning("stale not found with %s" % record)

             if stale:
             if stale:

-                self.logger.info("IotlabImporter: deleting stale record: %s" \
-                %(record))
+                self.logger.info("IotlabImporter: deleting stale record: %s"
+                                 % (record))

 
                 try:
                     dbsession.delete(record)
                     dbsession.commit()
                 except SQLAlchemyError:
 
                 try:
                     dbsession.delete(record)
                     dbsession.commit()
                 except SQLAlchemyError:

-                    self.logger.log_exc("IotlabImporter: failed to delete stale \
-                    record %s" %(record) )
-
-
+                    self.logger.log_exc("IotlabImporter: failed to delete \
+                        stale record %s" % (record))