+""" File defining the importer class and all the methods needed to import
+the nodes, users and slices from OAR and LDAP to the SFA database.
+Also creates the iotlab specific databse and table to keep track
+of which slice hrn contains which job.
+"""
from sfa.util.config import Config
from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
from sfa.iotlab.iotlabdriver import IotlabDriver
-from sfa.iotlab.iotlabpostgres import IotlabDB, Iotlab_xp
+from sfa.iotlab.iotlabpostgres import IotlabDB
from sfa.trust.certificate import Keypair, convert_public_key
from sfa.trust.gid import create_uuid
from sfa.storage.alchemy import dbsession
from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
- RegUser, RegKey
+ RegUser, RegKey
from sqlalchemy.exc import SQLAlchemyError
-
class IotlabImporter:
"""
IotlabImporter class, generic importer_class. Used to populate the SFA DB
or deleted.
"""
- def __init__ (self, auth_hierarchy, loc_logger):
+ def __init__(self, auth_hierarchy, loc_logger):
"""
Sets and defines import logger and the authority name. Gathers all the
- records already registerd in the SFA DB, broke them into 3 dicts,
- by type and hrn, by email and by type and pointer.
+ records already registerd in the SFA DB, broke them into 3 dicts, by
+ type and hrn, by email and by type and pointer.
:param auth_hierarchy: authority name
:type auth_hierarchy: string
#create hash by (type,hrn)
#used to know if a given record is already known to SFA
self.records_by_type_hrn = \
- dict([( (record.type,record.hrn), record) \
- for record in self.all_records])
+ dict([((record.type, record.hrn), record)
+ for record in self.all_records])
self.users_rec_by_email = \
- dict([ (record.email, record) \
- for record in self.all_records if record.type == 'user'])
+ dict([(record.email, record)
+ for record in self.all_records if record.type == 'user'])
# create hash by (type,pointer)
self.records_by_type_pointer = \
- dict([ ( (str(record.type), record.pointer) , record) \
- for record in self.all_records if record.pointer != -1])
-
+ dict([((str(record.type), record.pointer), record)
+ for record in self.all_records if record.pointer != -1])
@staticmethod
def hostname_to_hrn_escaped(root_auth, hostname):
"""
- Returns a node's hrn based on its hostname and the root
- authority and by removing special caracters from the hostname.
+ Returns a node's hrn based on its hostname and the root authority and by
+ removing special caracters from the hostname.
:param root_auth: root authority name
:param hostname: nodes's hostname
:type hostname: string
:rtype: string
"""
- return '.'.join( [root_auth, Xrn.escape(hostname)] )
+ return '.'.join([root_auth, Xrn.escape(hostname)])
@staticmethod
:type person_hrn: string
:rtype: string
"""
- return (person_hrn +'_slice')
+ return (person_hrn + '_slice')
- def add_options (self, parser):
+ def add_options(self, parser):
+ """
+ .. warning:: not used
+ """
# we don't have any options for now
pass
def find_record_by_type_hrn(self, record_type, hrn):
"""
-
- Returns the record associated with a given hrn and hrn type.
- Returns None if the key tuple is not in dictionary.
+ Finds the record associated with the hrn and its type given in parameter
+ if the tuple (hrn, type hrn) is an existing key in the dictionary.
:param record_type: the record's type (slice, node, authority...)
:type record_type: string
:param hrn: Human readable name of the object's record
:type hrn: string
- :rtype: RegUser if user, RegSlice if slice, RegNode if node...
- or None if record does not exist.
+ :returns: Returns the record associated with a given hrn and hrn type.
+ Returns None if the key tuple is not in the dictionary.
+ :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
+ record does not exist.
"""
- return self.records_by_type_hrn.get ( (record_type, hrn), None)
+ return self.records_by_type_hrn.get((record_type, hrn), None)
- def locate_by_type_pointer (self, record_type, pointer):
+ def locate_by_type_pointer(self, record_type, pointer):
"""
Returns the record corresponding to the key pointer and record
:param record_type: the record's type (slice, node, authority...)
:type record_type: string
:param pointer:Pointer to where the record is in the origin db,
- used in case the record comes from a trusted authority.
+ used in case the record comes from a trusted authority.
:type pointer: integer
:rtype: RegUser if user, RegSlice if slice, RegNode if node...
- or None if record does not exist.
+ or None if record does not exist.
"""
- return self.records_by_type_pointer.get ( (record_type, pointer), None)
+ return self.records_by_type_pointer.get((record_type, pointer), None)
- def update_just_added_records_dict (self, record):
+ def update_just_added_records_dict(self, record):
"""
Updates the records_by_type_hrn dictionnary if the record has
"""
rec_tuple = (record.type, record.hrn)
if rec_tuple in self.records_by_type_hrn:
- self.logger.warning ("IotlabImporter.update_just_added_records_dict:\
- duplicate (%s,%s)"%rec_tuple)
+ self.logger.warning("IotlabImporter.update_just_added_records_dict:\
+ duplicate (%s,%s)" % rec_tuple)
return
- self.records_by_type_hrn [ rec_tuple ] = record
+ self.records_by_type_hrn[rec_tuple] = record
def import_sites_and_nodes(self, iotlabdriver):
"""
For each site, import the site's nodes to the DB by calling
import_nodes.
- :param iotlabdriver: IotlabDriver object, used to have access to iotlabdriver
- methods and fetching info on sites and nodes.
+ :param iotlabdriver: IotlabDriver object, used to have access to
+ iotlabdriver methods and fetching info on sites and nodes.
:type iotlabdriver: IotlabDriver
"""
- sites_listdict = iotlabdriver.iotlab_api.GetSites()
- nodes_listdict = iotlabdriver.iotlab_api.GetNodes()
+ sites_listdict = iotlabdriver.iotlab_api.GetSites()
+ nodes_listdict = iotlabdriver.iotlab_api.GetNodes()
nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
for site in sites_listdict:
site_hrn = site['name']
self.auth_hierarchy.create_auth(urn)
auth_info = self.auth_hierarchy.get_auth_info(urn)
- site_record = RegAuthority(hrn=site_hrn, \
- gid=auth_info.get_gid_object(),
- pointer='-1',
- authority=get_authority(site_hrn))
+ site_record = \
+ RegAuthority(hrn=site_hrn,
+ gid=auth_info.get_gid_object(),
+ pointer='-1',
+ authority=get_authority(site_hrn))
site_record.just_created()
dbsession.add(site_record)
dbsession.commit()
- self.logger.info("IotlabImporter: imported authority (site) \
- %s" % site_record)
+ self.logger.info("IotlabImporter: imported authority \
+ (site) %s" % site_record)
self.update_just_added_records_dict(site_record)
except SQLAlchemyError:
# if the site import fails then there is no point in
# trying to import the
# site's child records(node, slices, persons), so skip them.
- self.logger.log_exc("IotlabImporter: failed to import site. \
- Skipping child records")
+ self.logger.log_exc("IotlabImporter: failed to import \
+ site. Skipping child records")
continue
else:
# xxx update the record ...
pass
-
site_record.stale = False
self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver)
:param nodes_by_id: dictionary , key is the node id, value is the a dict
with node information.
:type nodes_by_id: dictionary
- :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver
- attributes.
+ :param iotlabdriver:IotlabDriver object, used to have access to
+ iotlabdriver attributes.
:type iotlabdriver:IotlabDriver
"""
try:
node = nodes_by_id[node_id]
except KeyError:
- self.logger.warning ("IotlabImporter: cannot find node_id %s \
- - ignored" %(node_id))
+ self.logger.warning("IotlabImporter: cannot find node_id %s \
+ - ignored" % (node_id))
continue
escaped_hrn = \
- self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth, \
- node['hostname'])
- self.logger.info("IOTLABIMPORTER node %s " %(node))
- hrn = node['hrn']
-
+ self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth,
+ node['hostname'])
+ self.logger.info("IOTLABIMPORTER node %s " % (node))
+ hrn = node['hrn']
# xxx this sounds suspicious
if len(hrn) > 64:
hrn = hrn[:64]
- node_record = self.find_record_by_type_hrn( 'node', hrn )
+ node_record = self.find_record_by_type_hrn('node', hrn)
if not node_record:
pkey = Keypair(create=True)
urn = hrn_to_urn(escaped_hrn, 'node')
node_gid = \
- self.auth_hierarchy.create_gid(urn, \
- create_uuid(), pkey)
+ self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
def iotlab_get_authority(hrn):
+ """ Gets the authority part in the hrn.
+ :param hrn: hrn whose authority we are looking for.
+ :type hrn: string
+ :returns: splits the hrn using the '.' separator and returns
+ the authority part of the hrn.
+ :rtype: string
+
+ """
return hrn.split(".")[0]
node_record = RegNode(hrn=hrn, gid=node_gid,
- pointer = '-1',
- authority=iotlab_get_authority(hrn))
+ pointer='-1',
+ authority=iotlab_get_authority(hrn))
try:
node_record.just_created()
dbsession.add(node_record)
dbsession.commit()
- self.logger.info("IotlabImporter: imported node: %s" \
- % node_record)
+ self.logger.info("IotlabImporter: imported node: %s"
+ % node_record)
self.update_just_added_records_dict(node_record)
except SQLAlchemyError:
- self.logger.log_exc("IotlabImporter: \
- failed to import node")
+ self.logger.log_exc("IotlabImporter: failed to import node")
else:
#TODO: xxx update the record ...
pass
node_record.stale = False
-
- def init_person_key (self, person, iotlab_key):
+ def init_person_key(self, person, iotlab_key):
"""
Returns a tuple pubkey and pkey.
:param person Person's data.
:type person: dict
:param iotlab_key: SSH public key, from LDAP user's data.
- RSA type supported.
+ RSA type supported.
:type iotlab_key: string
:rtype (string, Keypair)
+
"""
pubkey = None
- if person['pkey']:
+ if person['pkey']:
# randomly pick first key in set
pubkey = iotlab_key
pkey = convert_public_key(pubkey)
except TypeError:
#key not good. create another pkey
- self.logger.warn('IotlabImporter: \
+ self.logger.warn("IotlabImporter: \
unable to convert public \
- key for %s' %person['hrn'])
+ key for %s" % person['hrn'])
pkey = Keypair(create=True)
else:
# the user has no keys.
#Creating a random keypair for the user's gid
self.logger.warn("IotlabImporter: person %s does not have a \
- public key" %(person['hrn']))
+ public key" % (person['hrn']))
pkey = Keypair(create=True)
return (pubkey, pkey)
-
def import_persons_and_slices(self, iotlabdriver):
"""
import the user's slice onto the database as well by calling
import_slice.
- :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver
- attributes.
+ :param iotlabdriver:IotlabDriver object, used to have access to
+ iotlabdriver attributes.
:type iotlabdriver:IotlabDriver
"""
ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons()
- self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n" \
- %(ldap_person_listdict))
+ self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
+ % (ldap_person_listdict))
# import persons
- for person in ldap_person_listdict :
+ for person in ldap_person_listdict:
- self.logger.info("IotlabImporter: person :" %(person))
+ self.logger.info("IotlabImporter: person :" % (person))
if 'ssh-rsa' not in person['pkey']:
#people with invalid ssh key (ssh-dss, empty, bullshit keys...)
#won't be imported
person_urn = hrn_to_urn(person_hrn, 'user')
- self.logger.info("IotlabImporter: users_rec_by_email %s " \
- %(self.users_rec_by_email))
+ self.logger.info("IotlabImporter: users_rec_by_email %s "
+ % (self.users_rec_by_email))
#Check if user using person['email'] from LDAP is already registered
#in SFA. One email = one person. In this case, do not create another
#uid ldap
user_record = self.find_record_by_type_hrn('user', person_hrn)
- if not user_record and person['email'] in self.users_rec_by_email:
+ if not user_record and person['email'] in self.users_rec_by_email:
user_record = self.users_rec_by_email[person['email']]
person_hrn = user_record.hrn
person_urn = hrn_to_urn(person_hrn, 'user')
- slice_record = self.find_record_by_type_hrn ('slice', slice_hrn)
+ slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
iotlab_key = person['pkey']
# new person
if not user_record:
(pubkey, pkey) = self.init_person_key(person, iotlab_key)
- if pubkey is not None and pkey is not None :
+ if pubkey is not None and pkey is not None:
person_gid = \
- self.auth_hierarchy.create_gid(person_urn, \
- create_uuid(), pkey)
+ self.auth_hierarchy.create_gid(person_urn,
+ create_uuid(), pkey)
if person['email']:
- self.logger.debug( "IOTLAB IMPORTER \
- PERSON EMAIL OK email %s " %(person['email']))
+ self.logger.debug("IOTLAB IMPORTER \
+ PERSON EMAIL OK email %s " % (person['email']))
person_gid.set_email(person['email'])
- user_record = RegUser(hrn=person_hrn, \
- gid=person_gid,
- pointer='-1',
- authority=get_authority(person_hrn),
- email=person['email'])
+ user_record = \
+ RegUser(hrn=person_hrn,
+ gid=person_gid,
+ pointer='-1',
+ authority=get_authority(person_hrn),
+ email=person['email'])
else:
- user_record = RegUser(hrn=person_hrn, \
- gid=person_gid,
- pointer='-1',
- authority=get_authority(person_hrn))
+ user_record = \
+ RegUser(hrn=person_hrn,
+ gid=person_gid,
+ pointer='-1',
+ authority=get_authority(person_hrn))
if pubkey:
user_record.reg_keys = [RegKey(pubkey)]
else:
- self.logger.warning("No key found for user %s" \
- %(user_record))
+ self.logger.warning("No key found for user %s"
+ % (user_record))
try:
user_record.just_created()
dbsession.add (user_record)
dbsession.commit()
- self.logger.info("IotlabImporter: imported person %s"\
- %(user_record))
- self.update_just_added_records_dict( user_record )
+ self.logger.info("IotlabImporter: imported person \
+ %s" % (user_record))
+ self.update_just_added_records_dict(user_record)
except SQLAlchemyError:
self.logger.log_exc("IotlabImporter: \
- failed to import person %s"%(person))
+ failed to import person %s" % (person))
else:
# update the record ?
# if user's primary key has changed then we need to update
sfa_keys = user_record.reg_keys
new_key = False
- if iotlab_key is not sfa_keys :
+ if iotlab_key is not sfa_keys:
new_key = True
if new_key:
self.logger.info("IotlabImporter: \t \t USER UPDATE \
- person: %s" %(person['hrn']))
- (pubkey, pkey) = self.init_person_key (person, iotlab_key)
+ person: %s" % (person['hrn']))
+ (pubkey, pkey) = self.init_person_key(person, iotlab_key)
person_gid = \
- self.auth_hierarchy.create_gid(person_urn, \
- create_uuid(), pkey)
+ self.auth_hierarchy.create_gid(person_urn,
+ create_uuid(), pkey)
if not pubkey:
user_record.reg_keys = []
else:
user_record.reg_keys = [RegKey(pubkey)]
- self.logger.info("IotlabImporter: updated person: %s" \
- % (user_record))
+ self.logger.info("IotlabImporter: updated person: %s"
+ % (user_record))
if person['email']:
user_record.email = person['email']
user_record.stale = False
except SQLAlchemyError:
self.logger.log_exc("IotlabImporter: \
- failed to update person %s"%(person))
+ failed to update person %s"% (person))
self.import_slice(slice_hrn, slice_record, user_record)
.. todo::Update the record if a slice record already exists.
"""
- if not slice_record :
+ if not slice_record:
pkey = Keypair(create=True)
urn = hrn_to_urn(slice_hrn, 'slice')
slice_gid = \
- self.auth_hierarchy.create_gid(urn, \
- create_uuid(), pkey)
- slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid,
- pointer='-1',
- authority=get_authority(slice_hrn))
+ self.auth_hierarchy.create_gid(urn,
+ create_uuid(), pkey)
+ slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
+ pointer='-1',
+ authority=get_authority(slice_hrn))
try:
slice_record.just_created()
dbsession.add(slice_record)
dbsession.commit()
- self.update_just_added_records_dict ( slice_record )
+ self.update_just_added_records_dict(slice_record)
except SQLAlchemyError:
self.logger.log_exc("IotlabImporter: failed to import slice")
#No slice update upon import in iotlab
else:
# xxx update the record ...
- self.logger.warning ("Slice update not yet implemented")
+ self.logger.warning("Slice update not yet implemented")
pass
# record current users affiliated with the slice
- slice_record.reg_researchers = [user_record]
+ slice_record.reg_researchers = [user_record]
try:
dbsession.commit()
slice_record.stale = False
self.logger.log_exc("IotlabImporter: failed to update slice")
- def run (self, options):
+ def run(self, options):
"""
Create the special iotlab table, iotlab_xp, in the iotlab database.
Import everything (users, slices, nodes and sites from OAR
if not iotlab_db.exists('iotlab_xp'):
iotlab_db.createtable()
- self.logger.info ("IotlabImporter.run: iotlab_xp table created ")
-
+ self.logger.info("IotlabImporter.run: iotlab_xp table created ")
# import site and node records in site into the SFA db.
self.import_sites_and_nodes(iotlabdriver)
### remove stale records
# special records must be preserved
- system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth, \
- iotlabdriver.hrn+ '.slicemanager']
+ system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth,
+ iotlabdriver.hrn + '.slicemanager']
for record in self.all_records:
if record.hrn in system_hrns:
record.stale = False
if record.peer_authority:
record.stale = False
-
for record in self.all_records:
if record.type == 'user':
- self.logger.info("IotlabImporter: stale records: hrn %s %s" \
- %(record.hrn,record.stale) )
+ self.logger.info("IotlabImporter: stale records: hrn %s %s"
+ % (record.hrn, record.stale))
try:
stale = record.stale
- except :
+ except:
stale = True
- self.logger.warning("stale not found with %s"%record)
+ self.logger.warning("stale not found with %s" % record)
if stale:
- self.logger.info("IotlabImporter: deleting stale record: %s" \
- %(record))
+ self.logger.info("IotlabImporter: deleting stale record: %s"
+ % (record))
try:
dbsession.delete(record)
dbsession.commit()
except SQLAlchemyError:
- self.logger.log_exc("IotlabImporter: failed to delete stale \
- record %s" %(record) )
-
-
+ self.logger.log_exc("IotlabImporter: failed to delete \
+ stale record %s" % (record))
git://git.onelab.eu / sfa.git / commitdiff