From 37a296de43dfa860362d4482cfb2339ee55736dc Mon Sep 17 00:00:00 2001 From: Tony Mack Date: Sun, 1 Jan 2012 16:05:42 -0500 Subject: [PATCH] moved plc specific code out of sfa.importer.sfaImporter. Refactored sfa.importer.sfa-import.plc and sfa.importer.sfaImporter --- sfa/importer/sfa-import-plc.py | 61 ++++++++++--- sfa/importer/sfaImport.py | 159 +++------------------------------ sfa/storage/record.py | 68 +++++++++++++- sfa/storage/row.py | 1 + 4 files changed, 129 insertions(+), 160 deletions(-) diff --git a/sfa/importer/sfa-import-plc.py b/sfa/importer/sfa-import-plc.py index 8558f65e..d827396b 100755 --- a/sfa/importer/sfa-import-plc.py +++ b/sfa/importer/sfa-import-plc.py @@ -19,12 +19,14 @@ import getopt import sys from sfa.util.config import Config -from sfa.util.xrn import Xrn, get_leaf, get_authority +from sfa.util.xrn import Xrn, get_leaf, get_authority, hrn_to_urn from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename - from sfa.storage.table import SfaTable - -from sfa.importer.sfaImport import sfaImport +from sfa.storage.record import SfaRecord +from sfa.trust.gid import create_uuid +from sfa.trust.certificate import convert_public_key, Keypair +from sfa.importer.sfaImport import sfaImport, _cleanup_string +from sfa.util.sfalogging import logger def process_options(): @@ -141,7 +143,13 @@ def main(): # but its not a site record if site_hrn not in existing_hrns or \ (site_hrn, 'authority') not in existing_records: - sfaImporter.import_site(site_hrn, site) + logger.info("Import: site %s " % site_hrn) + urn = hrn_to_urn(site_hrn, 'authority') + if not sfaImporter.AuthHierarchy.auth_exists(urn): + sfaImporter.AuthHierarchy.create_auth(urn) + auth_info = sfaImporter.AuthHierarchy.get_auth_info(urn) + auth_record = SfaRecord(hrn=site_hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id']) + auth_record.sync(verbose=True) # import node records for node_id in site['node_ids']: @@ -151,9 +159,15 @@ def main(): site_auth = get_authority(site_hrn) site_name = get_leaf(site_hrn) hrn = hostname_to_hrn(site_auth, site_name, node['hostname']) + if len(hrn) > 64: + hrn = hrn[:64] if hrn not in existing_hrns or \ (hrn, 'node') not in existing_records: - sfaImporter.import_node(hrn, node) + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'node') + node_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id'], authority=get_authority(hrn)) + node_record.sync(verbose=True) # import slices for slice_id in site['slice_ids']: @@ -161,9 +175,16 @@ def main(): continue slice = slices_dict[slice_id] hrn = slicename_to_hrn(interface_hrn, slice['name']) + #slicename = slice['name'].split("_",1)[-1] + #slicename = _cleanup_string(slicename) if hrn not in existing_hrns or \ (hrn, 'slice') not in existing_records: - sfaImporter.import_slice(site_hrn, slice) + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'slice') + slice_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'], + authority=get_authority(hrn)) + slice_record.sync(verbose=True) # import persons for person_id in site['person_ids']: @@ -171,6 +192,11 @@ def main(): continue person = persons_dict[person_id] hrn = email_to_hrn(site_hrn, person['email']) + if len(hrn) > 64: + hrn = hrn[:64] + + # if user's primary key has chnaged then we need to update the + # users gid by forcing a update here old_keys = [] new_keys = [] if person_id in old_person_keys: @@ -184,8 +210,22 @@ def main(): if hrn not in existing_hrns or \ (hrn, 'user') not in existing_records or update_record: - sfaImporter.import_person(site_hrn, person) - + if 'key_ids' in person and person['key_ids']: + key = new_keys[0] + try: + pkey = convert_public_key(key) + except: + logger.warn('unable to convert public key for %s' % hrn) + pkey = Keypair(create=True) + else: + # the user has no keys. Creating a random keypair for the user's gid + logger.warn("Import: person %s does not have a PL public key"%hrn) + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'user') + person_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", \ + pointer=person['person_id'], authority=get_authority(hrn)) + person_record.sync(verbose=True) # remove stale records system_records = [interface_hrn, root_auth, interface_hrn + '.slicemanager'] @@ -251,7 +291,8 @@ def main(): if not found: record_object = existing_records[(record_hrn, type)] - sfaImporter.delete_record(record_hrn, type) + record = SfaRecord(dict=record_object) + record.delete() # save pub keys logger.info('Import: saving current pub keys') diff --git a/sfa/importer/sfaImport.py b/sfa/importer/sfaImport.py index a37f413a..68cd195e 100644 --- a/sfa/importer/sfaImport.py +++ b/sfa/importer/sfaImport.py @@ -94,14 +94,8 @@ class sfaImport: self.AuthHierarchy.create_top_level_auth(hrn) # create the db record if it doesnt already exist auth_info = self.AuthHierarchy.get_auth_info(hrn) - table = SfaTable() - auth_record = table.find({'type': 'authority', 'hrn': hrn}) - - if not auth_record: - auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1) - auth_record['authority'] = get_authority(auth_record['hrn']) - self.logger.info("Import: inserting authority record for %s"%hrn) - table.insert(auth_record) + auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1, authority=get_authority(hrn)) + auth_record.sync(verbose=True) def create_sm_client_record(self): """ @@ -114,12 +108,9 @@ class sfaImport: self.AuthHierarchy.create_auth(urn) auth_info = self.AuthHierarchy.get_auth_info(hrn) - table = SfaTable() - sm_user_record = table.find({'type': 'user', 'hrn': hrn}) - if not sm_user_record: - record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="user", pointer=-1) - record['authority'] = get_authority(record['hrn']) - table.insert(record) + record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), \ + type="user", pointer=-1, authority=get_authority(hrn)) + record.sync(verbose=True) def create_interface_records(self): """ @@ -127,142 +118,18 @@ class sfaImport: """ # just create certs for all sfa interfaces even if they # arent enabled - interface_hrn = self.config.SFA_INTERFACE_HRN + hrn = self.config.SFA_INTERFACE_HRN interfaces = ['authority+sa', 'authority+am', 'authority+sm'] table = SfaTable() - auth_info = self.AuthHierarchy.get_auth_info(interface_hrn) + auth_info = self.AuthHierarchy.get_auth_info(hrn) pkey = auth_info.get_pkey_object() for interface in interfaces: - interface_record = table.find({'type': interface, 'hrn': interface_hrn}) - if not interface_record: - self.logger.info("Import: interface %s %s " % (interface_hrn, interface)) - urn = hrn_to_urn(interface_hrn, interface) - gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - record = SfaRecord(hrn=interface_hrn, gid=gid, type=interface, pointer=-1) - record['authority'] = get_authority(interface_hrn) - table.insert(record) - - def import_person(self, parent_hrn, person): - """ - Register a user record - """ - hrn = email_to_hrn(parent_hrn, person['email']) - - # ASN.1 will have problems with hrn's longer than 64 characters - if len(hrn) > 64: - hrn = hrn[:64] - - self.logger.info("Import: person %s"%hrn) - key_ids = [] - if 'key_ids' in person and person['key_ids']: - key_ids = person["key_ids"] - # get the user's private key from the SSH keys they have uploaded - # to planetlab - keys = self.shell.GetKeys(key_ids) - key = keys[0]['key'] - pkey = None - try: - pkey = convert_public_key(key) - except: - self.logger.warn('unable to convert public key for %s' % hrn) - if not pkey: - pkey = Keypair(create=True) - else: - # the user has no keys - self.logger.warn("Import: person %s does not have a PL public key"%hrn) - # if a key is unavailable, then we still need to put something in the - # user's GID. So make one up. - pkey = Keypair(create=True) - - # create the gid - urn = hrn_to_urn(hrn, 'user') - person_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - table = SfaTable() - person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", pointer=person['person_id']) - person_record['authority'] = get_authority(person_record['hrn']) - existing_records = table.find({'hrn': hrn, 'type': 'user', 'pointer': person['person_id']}) - if not existing_records: - table.insert(person_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - person_record['record_id'] = existing_record['record_id'] - table.update(person_record) - - def import_slice(self, parent_hrn, slice): - slicename = slice['name'].split("_",1)[-1] - slicename = _cleanup_string(slicename) - - if not slicename: - self.logger.error("Import: failed to parse slice name %s" %slice['name']) - return - - hrn = parent_hrn + "." + slicename - self.logger.info("Import: slice %s"%hrn) - - pkey = Keypair(create=True) - urn = hrn_to_urn(hrn, 'slice') - slice_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id']) - slice_record['authority'] = get_authority(slice_record['hrn']) - table = SfaTable() - existing_records = table.find({'hrn': hrn, 'type': 'slice', 'pointer': slice['slice_id']}) - if not existing_records: - table.insert(slice_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - slice_record['record_id'] = existing_record['record_id'] - table.update(slice_record) - - def import_node(self, hrn, node): - self.logger.info("Import: node %s" % hrn) - # ASN.1 will have problems with hrn's longer than 64 characters - if len(hrn) > 64: - hrn = hrn[:64] - - table = SfaTable() - node_record = table.find({'type': 'node', 'hrn': hrn}) - pkey = Keypair(create=True) - urn = hrn_to_urn(hrn, 'node') - node_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id']) - node_record['authority'] = get_authority(node_record['hrn']) - existing_records = table.find({'hrn': hrn, 'type': 'node', 'pointer': node['node_id']}) - if not existing_records: - table.insert(node_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - node_record['record_id'] = existing_record['record_id'] - table.update(node_record) - - - def import_site(self, hrn, site): - urn = hrn_to_urn(hrn, 'authority') - self.logger.info("Import: site %s"%hrn) - - # create the authority - if not self.AuthHierarchy.auth_exists(urn): - self.AuthHierarchy.create_auth(urn) - - auth_info = self.AuthHierarchy.get_auth_info(urn) - - table = SfaTable() - auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id']) - auth_record['authority'] = get_authority(auth_record['hrn']) - existing_records = table.find({'hrn': hrn, 'type': 'authority', 'pointer': site['site_id']}) - if not existing_records: - table.insert(auth_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - auth_record['record_id'] = existing_record['record_id'] - table.update(auth_record) - - return hrn - - + urn = hrn_to_urn(hrn, interface) + gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + interface_record = SfaRecord(hrn=hrn, type=interface, pointer=-1, + gid = gid, authority=get_authority(hrn)) + interface_record.sync(verbose=True) + def delete_record(self, hrn, type): # delete the record table = SfaTable() diff --git a/sfa/storage/record.py b/sfa/storage/record.py index 46c6699e..964d65c6 100644 --- a/sfa/storage/record.py +++ b/sfa/storage/record.py @@ -5,13 +5,12 @@ ## from types import StringTypes - from sfa.trust.gid import GID - from sfa.storage.parameter import Parameter from sfa.util.xrn import get_authority from sfa.storage.row import Row from sfa.util.xml import XML +from sfa.util.sfalogging import logger class SfaRecord(Row): """ @@ -62,13 +61,14 @@ class SfaRecord(Row): # @param pointer is a pointer to a PLC record # @param dict if !=None, then fill in this record from the dictionary - def __init__(self, hrn=None, gid=None, type=None, pointer=None, peer_authority=None, dict=None, string=None): + def __init__(self, hrn=None, gid=None, type=None, pointer=None, authority=None, peer_authority=None, dict=None, string=None): self.dirty = True self.hrn = None self.gid = None self.type = None self.pointer = None self.set_peer_auth(peer_authority) + self.set_authority(authority) if hrn: self.set_name(hrn) if gid: @@ -112,6 +112,17 @@ class SfaRecord(Row): self['hrn'] = hrn self.dirty = True + def set_authority(self, authority): + """ + Set the authority + """ + if not authority: + authority = "" + self.authority = authority + self['authority'] = authority + self.dirty = True + + ## # Set the GID of the record # @@ -353,9 +364,58 @@ class SfaRecord(Row): else: print " %s: %s" % (key, self[key]) + def summary_string(self): + return "Record(record_id=%s, hrn=%s, type=%s, auth=%s, pointer=%s)" % \ + (self.get('record_id'), self.get('hrn'), self.get('type'), self.get('auth'), \ + self.get('pointer')) + def getdict(self): return dict(self) - + + def sync(self, verbose=False): + """ + Sync this record with the database. + """ + from sfa.storage.table import SfaTable + table = SfaTable() + filter = {} + if self.get('record_id'): + filter['record_id'] = self.get('record_id') + if self.get('hrn') and self.get('type'): + filter['hrn'] = self.get('hrn') + filter['type'] = self.get('type') + if self.get('pointer'): + filter['pointer'] = self.get('pointer') + existing_records = table.find(filter) + if not existing_records: + table.insert(self) + if verbose: + logger.info("Inserted record: %s" %self.summary_string()) + else: + existing_record = existing_records[0] + self['record_id'] = existing_record['record_id'] + table.upate(self) + if verbose: + logger.info("Updated record: %s" % self.summary_string()) + + def delete(self, verbose=False): + """ + Remove record from the database. + """ + from sfa.storage.table import SfaTable + table = SfaTable() + if self.get('record_id'): + filter['record_id'] = self.get('record_id') + if self.get('hrn') and self.get('type'): + filter['hrn'] = self.get('hrn') + filter['type'] = self.get('type') + if self.get('pointer'): + filter['pointer'] = self.get('pointer') + existing_records = table.find(filter) + for record in existing_records: + table.remove(record) + if verbose: + logger.info("Removed record: %s" % self.summary_string()) class UserRecord(SfaRecord): diff --git a/sfa/storage/row.py b/sfa/storage/row.py index 129f5147..ef668445 100644 --- a/sfa/storage/row.py +++ b/sfa/storage/row.py @@ -55,3 +55,4 @@ class Row(dict): raise SfaInvalidArgument, "'%s' not in the future" % human return human + -- 2.43.0