### $URL$
#
##
-# Import PLC records into the Geni database. It is indended that this tool be
-# run once to create Geni records that reflect the current state of the
+# Import PLC records into the SFA database. It is indended that this tool be
+# run once to create SFA records that reflect the current state of the
# planetlab database.
#
# The import tool assumes that the existing PLC hierarchy should all be part
import getopt
import sys
import tempfile
-
+import logging.handlers
+import logging
+from sfa.util.record import *
+from sfa.util.table import SfaTable
+from sfa.util.namespace import *
+from sfa.util.config import Config
from sfa.trust.certificate import convert_public_key, Keypair
from sfa.trust.trustedroot import *
-
from sfa.trust.hierarchy import *
-from sfa.util.record import *
-from sfa.util.genitable import *
-from sfa.util.misc import *
-from sfa.util.config import *
-
-# get PL account settings from config module
-pl_auth = get_pl_auth()
-
-def connect_shell():
- global pl_auth, shell
-
- # get PL account settings from config module
- pl_auth = get_pl_auth()
-
- # connect to planetlab
- if "Url" in pl_auth:
- from sfa.util import remoteshell
- shell = remoteshell.RemoteShell()
- else:
- import PLC.Shell
- shell = PLC.Shell.Shell(globals = globals())
-
- return shell
-
-# connect to planetlab
-shell = connect_shell()
-
-##
-# Two authorities are specified: the root authority and the level1 authority.
-
-#root_auth = "plc"
-#level1_auth = None
-
-#root_auth = "planetlab"
-#level1_auth = "planetlab.us"
-config = Config()
-
-root_auth = config.SFA_REGISTRY_ROOT_AUTH
-level1_auth = config.SFA_REGISTRY_LEVEL1_AUTH
-if not level1_auth or level1_auth in ['']:
- level1_auth = None
-
-def un_unicode(str):
- if isinstance(str, unicode):
- return str.encode("ascii", "ignore")
- else:
- return str
-
-def cleanup_string(str):
- # pgsql has a fit with strings that have high ascii in them, so filter it
- # out when generating the hrns.
- tmp = ""
- for c in str:
- if ord(c) < 128:
- tmp = tmp + c
- str = tmp
-
- str = un_unicode(str)
- str = str.replace(" ", "_")
- str = str.replace(".", "_")
- str = str.replace("(", "_")
- str = str.replace("'", "_")
- str = str.replace(")", "_")
- str = str.replace('"', "_")
- return str
+from sfa.plc.api import *
+from sfa.trust.gid import create_uuid
+from sfa.plc.sfaImport import *
+from sfa.util.report import trace, error
def process_options():
global hrn
name = opt[0]
val = opt[1]
-def get_auth_table(auth_name):
- AuthHierarchy = Hierarchy()
- auth_info = AuthHierarchy.get_auth_info(auth_name)
- table = GeniTable(hrn=auth_name,
- cninfo=auth_info.get_dbinfo())
+def load_keys(filename):
+ keys = {}
+ tmp_dict = {}
+ try:
+ execfile(filename, tmp_dict)
+ if 'keys' in tmp_dict:
+ keys = tmp_dict['keys']
+ return keys
+ except:
+ return keys
- # if the table doesn't exist, then it means we haven't put any records
- # into this authority yet.
+def save_keys(filename, keys):
+ f = open(filename, 'w')
+ f.write("keys = %s" % str(keys))
+ f.close()
+def main():
+ # setup the logger
+ LOGFILE='/var/log/sfa_import_plc.log'
+ logging.basicConfig(level=logging.INFO,
+ format='%(asctime)s - %(message)s',
+ filename=LOGFILE)
+ rotate_handler = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes=1000000, backupCount=5)
+ logger = logging.getLogger()
+ logger.addHandler(rotate_handler)
+
+ process_options()
+ config = Config()
+ if not config.SFA_REGISTRY_ENABLED:
+ sys.exit(0)
+ root_auth = config.SFA_REGISTRY_ROOT_AUTH
+ level1_auth = config.SFA_REGISTRY_LEVEL1_AUTH
+ keys_filename = config.config_path + os.sep + 'person_keys.py'
+ sfaImporter = sfaImport(logger)
+ shell = sfaImporter.shell
+ plc_auth = sfaImporter.plc_auth
+ AuthHierarchy = sfaImporter.AuthHierarchy
+ TrustedRoots = sfaImporter.TrustedRoots
+ table = SfaTable()
if not table.exists():
- report.trace("Import: creating table for authority " + auth_name)
table.create()
- return table
-
-def person_to_hrn(parent_hrn, person):
- # the old way - Lastname_Firstname
- #personname = person['last_name'] + "_" + person['first_name']
-
- # the new way - use email address up to the "@"
- personname = person['email'].split("@")[0]
-
- personname = cleanup_string(personname)
-
- hrn = parent_hrn + "." + personname
- return hrn
-
-def import_person(parent_hrn, person):
- AuthHierarchy = Hierarchy()
- hrn = person_to_hrn(parent_hrn, person)
-
- # ASN.1 will have problems with hrn's longer than 64 characters
- if len(hrn) > 64:
- hrn = hrn[:64]
-
- report.trace("Import: importing person " + hrn)
-
- table = get_auth_table(parent_hrn)
-
- key_ids = []
- if 'key_ids' in person:
- key_ids = person["key_ids"]
-
- # get the user's private key from the SSH keys they have uploaded
- # to planetlab
- keys = shell.GetKeys(pl_auth, key_ids)
- key = keys[0]['key']
- pkey =convert_public_key(key)
- else:
- # the user has no keys
- report.trace(" person " + hrn + " does not have a PL public key")
-
- # if a key is unavailable, then we still need to put something in the
- # user's GID. So make one up.
- pkey = Keypair(create=True)
-
- # create the gid
- person_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
- person_record = table.resolve("user", hrn)
- if not person_record:
- report.trace(" inserting user record for " + hrn)
- person_record = GeniRecord(name=hrn, gid=person_gid, type="user", pointer=person['person_id'])
- table.insert(person_record)
- else:
- report.trace(" updating user record for " + hrn)
- person_record = GeniRecord(name=hrn, gid=person_gid, type="user", pointer=person['person_id'])
- table.update(person_record)
-
-def import_slice(parent_hrn, slice):
- AuthHierarchy = Hierarchy()
- slicename = slice['name'].split("_",1)[-1]
- slicename = cleanup_string(slicename)
-
- if not slicename:
- report.error("Import_Slice: failed to parse slice name " + slice['name'])
- return
-
- hrn = parent_hrn + "." + slicename
- report.trace("Import: importing slice " + hrn)
-
- table = get_auth_table(parent_hrn)
-
- slice_record = table.resolve("slice", hrn)
- if not slice_record:
- pkey = Keypair(create=True)
- slice_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
- slice_record = GeniRecord(name=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'])
- report.trace(" inserting slice record for " + hrn)
- table.insert(slice_record)
-
-def import_node(parent_hrn, node):
- AuthHierarchy = Hierarchy()
- nodename = node['hostname'].split(".")[0]
- nodename = cleanup_string(nodename)
-
- if not nodename:
- report.error("Import_node: failed to parse node name " + node['hostname'])
- return
-
- hrn = parent_hrn + "." + nodename
-
- # ASN.1 will have problems with hrn's longer than 64 characters
- if len(hrn) > 64:
- hrn = hrn[:64]
-
- report.trace("Import: importing node " + hrn)
-
- table = get_auth_table(parent_hrn)
-
- node_record = table.resolve("node", hrn)
- if not node_record:
- pkey = Keypair(create=True)
- node_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
- node_record = GeniRecord(name=hrn, gid=node_gid, type="node", pointer=node['node_id'])
- report.trace(" inserting node record for " + hrn)
- table.insert(node_record)
-
-def import_site(parent_hrn, site):
- AuthHierarchy = Hierarchy()
- sitename = site['login_base']
- sitename = cleanup_string(sitename)
-
- hrn = parent_hrn + "." + sitename
+ if not level1_auth or level1_auth in ['']:
+ level1_auth = None
- # Hardcode 'internet2' into the hrn for sites hosting
- # internet2 nodes. This is a special operation for some vini
- # sites only
- if ".vini" in parent_hrn and parent_hrn.endswith('vini'):
- if sitename.startswith("ii"):
- sitename = sitename.replace("ii", "")
- hrn = ".".join([parent_hrn, "internet2", sitename])
- elif sitename.startswith("nlr"):
- hrn = ".".join([parent_hrn, "internet2", sitename])
- sitename = sitename.replace("nlr", "")
-
- report.trace("Import_Site: importing site " + hrn)
-
- # create the authority
- if not AuthHierarchy.auth_exists(hrn):
- AuthHierarchy.create_auth(hrn)
-
- auth_info = AuthHierarchy.get_auth_info(hrn)
-
- table = get_auth_table(parent_hrn)
-
- auth_record = table.resolve("authority", hrn)
- if not auth_record:
- auth_record = GeniRecord(name=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id'])
- report.trace(" inserting authority record for " + hrn)
- table.insert(auth_record)
-
- if 'person_ids' in site:
- for person_id in site['person_ids']:
- persons = shell.GetPersons(pl_auth, [person_id])
- if persons:
- try:
- import_person(hrn, persons[0])
- except:
- report.trace("Failed to import: %s" % persons[0])
- if 'slice_ids' in site:
- for slice_id in site['slice_ids']:
- slices = shell.GetSlices(pl_auth, [slice_id])
- if slices:
- try:
- import_slice(hrn, slices[0])
- except:
- report.trace("Failed to import: %s" % slices[0])
- if 'node_ids' in site:
- for node_id in site['node_ids']:
- nodes = shell.GetNodes(pl_auth, [node_id])
- if nodes:
- try:
- import_node(hrn, nodes[0])
- except:
- report.trace("Failed to import: %s" % nodes[0])
-
-def create_top_level_auth_records(hrn):
- parent_hrn = get_authority(hrn)
- print hrn, ":", parent_hrn
- if not parent_hrn:
- parent_hrn = hrn
- auth_info = AuthHierarchy.get_auth_info(parent_hrn)
- table = get_auth_table(parent_hrn)
-
- auth_record = table.resolve("authority", hrn)
- if not auth_record:
- auth_record = GeniRecord(name=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1)
- report.trace(" inserting authority record for " + hrn)
- table.insert(auth_record)
-
-def main():
- global AuthHierarchy
- global TrustedRoots
-
- process_options()
-
- AuthHierarchy = Hierarchy()
- TrustedRoots = TrustedRootList()
-
- print "Import: creating top level authorities"
-
- if not AuthHierarchy.auth_exists(root_auth):
- AuthHierarchy.create_auth(root_auth)
-
- create_top_level_auth_records(root_auth)
- if level1_auth:
+ if not level1_auth:
+ sfaImporter.create_top_level_auth_records(root_auth)
+ import_auth = root_auth
+ else:
if not AuthHierarchy.auth_exists(level1_auth):
AuthHierarchy.create_auth(level1_auth)
- create_top_level_auth_records(level1_auth)
+ sfaImporter.create_top_level_auth_records(level1_auth)
import_auth = level1_auth
- else:
- import_auth = root_auth
- print "Import: adding", root_auth, "to trusted list"
- root = AuthHierarchy.get_auth_info(root_auth)
- TrustedRoots.add_gid(root.get_gid_object())
+ trace("Import: adding" + import_auth + "to trusted list", logger)
+ authority = AuthHierarchy.get_auth_info(import_auth)
+ TrustedRoots.add_gid(authority.get_gid_object())
- connect_shell()
-
- sites = shell.GetSites(pl_auth, {'peer_id': None})
- # create a fake internet2 site first
- i2site = {'name': 'Internet2', 'abbreviated_name': 'I2',
+ if ".vini" in import_auth and import_auth.endswith('vini'):
+ # create a fake internet2 site first
+ i2site = {'name': 'Internet2', 'abbreviated_name': 'I2',
'login_base': 'internet2', 'site_id': -1}
- import_site(import_auth, i2site)
+ sfaImporter.import_site(import_auth, i2site)
+
+ # create dict of all existing sfa records
+ existing_records = {}
+ existing_hrns = []
+ key_ids = []
+ person_keys = {}
+ results = table.find()
+ for result in results:
+ existing_records[(result['hrn'], result['type'])] = result
+ existing_hrns.append(result['hrn'])
+
+ # Get all plc sites
+ sites = shell.GetSites(plc_auth, {'peer_id': None})
+ sites_dict = {}
+ for site in sites:
+ sites_dict[site['login_base']] = site
+ # Get all plc users
+ persons = shell.GetPersons(plc_auth, {'peer_id': None}, ['person_id', 'email', 'key_ids', 'site_ids'])
+ persons_dict = {}
+ for person in persons:
+ persons_dict[person['person_id']] = person
+ key_ids.extend(person['key_ids'])
+
+ # Get all public keys
+ keys = shell.GetKeys(plc_auth, {'peer_id': None, 'key_id': key_ids})
+ keys_dict = {}
+ for key in keys:
+ keys_dict[key['key_id']] = key['key']
+
+ # create a dict of person keys keyed on key_id
+ old_person_keys = load_keys(keys_filename)
+ for person in persons:
+ pubkeys = []
+ for key_id in person['key_ids']:
+ pubkeys.append(keys_dict[key_id])
+ person_keys[person['person_id']] = pubkeys
+
+ # Get all plc nodes
+ nodes = shell.GetNodes(plc_auth, {'peer_id': None}, ['node_id', 'hostname', 'site_id'])
+ nodes_dict = {}
+ for node in nodes:
+ nodes_dict[node['node_id']] = node
+
+ # Get all plc slices
+ slices = shell.GetSlices(plc_auth, {'peer_id': None}, ['slice_id', 'name'])
+ slices_dict = {}
+ for slice in slices:
+ slices_dict[slice['slice_id']] = slice
+
+ # start importing
for site in sites:
- import_site(import_auth, site)
-
+ site_hrn = import_auth + "." + site['login_base']
+ # import if hrn is not in list of existing hrns or if the hrn exists
+ # but its not a site record
+ if site_hrn not in existing_hrns or \
+ (site_hrn, 'authority') not in existing_records:
+ site_hrn = sfaImporter.import_site(import_auth, site)
+
+ # import node records
+ for node_id in site['node_ids']:
+ if node_id not in nodes_dict:
+ continue
+ node = nodes_dict[node_id]
+ hrn = hostname_to_hrn(import_auth, site['login_base'], node['hostname'])
+ if hrn not in existing_hrns or \
+ (hrn, 'node') not in existing_records:
+ sfaImporter.import_node(site_hrn, node)
+
+ # import slices
+ for slice_id in site['slice_ids']:
+ if slice_id not in slices_dict:
+ continue
+ slice = slices_dict[slice_id]
+ hrn = slicename_to_hrn(import_auth, slice['name'])
+ if hrn not in existing_hrns or \
+ (hrn, 'slice') not in existing_records:
+ sfaImporter.import_slice(site_hrn, slice)
+
+ # import persons
+ for person_id in site['person_ids']:
+ if person_id not in persons_dict:
+ continue
+ person = persons_dict[person_id]
+ hrn = email_to_hrn(site_hrn, person['email'])
+ old_keys = []
+ new_keys = []
+ if person_id in old_person_keys:
+ old_keys = old_person_keys[person_id]
+ if person_id in person_keys:
+ new_keys = person_keys[person_id]
+ update_record = False
+ for key in new_keys:
+ if key not in old_keys:
+ update_record = True
+
+ if hrn not in existing_hrns or \
+ (hrn, 'user') not in existing_records or update_record:
+ sfaImporter.import_person(site_hrn, person)
+
+ # remove stale records
+ for (record_hrn, type) in existing_records.keys():
+ record = existing_records[(record_hrn, type)]
+ # if this is the interface name dont do anything
+ if record_hrn == import_auth or record['peer_authority']:
+ continue
+ # dont delete vini's internet2 placeholdder record
+ # normally this would be deleted becuase it does not have a plc record
+ if ".vini" in import_auth and import_auth.endswith('vini') and \
+ record_hrn.endswith("internet2"):
+ continue
+
+ found = False
+
+ if type == 'authority':
+ for site in sites:
+ site_hrn = import_auth + "." + site['login_base']
+ if site_hrn == record_hrn and site['site_id'] == record['pointer']:
+ found = True
+ break
+
+ elif type == 'user':
+ login_base = get_leaf(get_authority(record_hrn))
+ username = get_leaf(record_hrn)
+ if login_base in sites_dict:
+ site = sites_dict[login_base]
+ for person in persons:
+ tmp_username = person['email'].split("@")[0]
+ alt_username = person['email'].split("@")[0].replace(".", "_")
+ if username in [tmp_username, alt_username] and \
+ site['site_id'] in person['site_ids'] and \
+ person['person_id'] == record['pointer']:
+ found = True
+ break
+
+ elif type == 'slice':
+ slicename = hrn_to_pl_slicename(record_hrn)
+ for slice in slices:
+ if slicename == slice['name'] and \
+ slice['slice_id'] == record['pointer']:
+ found = True
+ break
+
+ elif type == 'node':
+ login_base = get_leaf(get_authority(record_hrn))
+ nodename = get_leaf(record_hrn)
+ if login_base in sites_dict:
+ site = sites_dict[login_base]
+ for node in nodes:
+ tmp_nodename = node['hostname'].split(".")[0]
+ if tmp_nodename == nodename and \
+ node['site_id'] == site['site_id'] and \
+ node['node_id'] == record['pointer']:
+ found = True
+ break
+ else:
+ continue
+
+ if not found:
+ record_object = existing_records[(record_hrn, type)]
+ sfaImporter.delete_record(record_hrn, type)
+
+ # save pub keys
+ trace('Import: saving current pub keys', logger)
+ save_keys(keys_filename, person_keys)
+
if __name__ == "__main__":
main()
git://git.onelab.eu / sfa.git / blobdiff