From 1971124b0a49cfbd7bce902722a1512baf8562f3 Mon Sep 17 00:00:00 2001 From: Tony Mack Date: Fri, 4 Sep 2009 00:39:20 +0000 Subject: [PATCH] changed the sfa db schema. All records are now stored in 1 table instead of createing a new table for each site (authority) --- sfa/methods/get_credential.py | 26 +++++---- sfa/methods/list.py | 17 +++--- sfa/methods/register.py | 35 ++++++----- sfa/methods/remove.py | 29 +++++----- sfa/methods/resolve.py | 8 +-- sfa/methods/update.py | 59 ++++++++----------- sfa/plc/api.py | 48 +++++++-------- sfa/plc/sfa-import-plc.py | 6 +- sfa/plc/sfaImport.py | 52 ++++++----------- sfa/trust/auth.py | 22 +------ sfa/util/genitable.py | 106 +++++++++++++++++++--------------- sfa/util/record.py | 35 ++++++----- 12 files changed, 202 insertions(+), 241 deletions(-) diff --git a/sfa/methods/get_credential.py b/sfa/methods/get_credential.py index ff5c1a2a..3e11f88e 100644 --- a/sfa/methods/get_credential.py +++ b/sfa/methods/get_credential.py @@ -7,7 +7,9 @@ from sfa.util.faults import * from sfa.util.method import Method from sfa.util.parameter import Parameter, Mixed from sfa.trust.auth import Auth +from sfa.trust.gid import GID from sfa.util.record import GeniRecord +from sfa.util.genitable import * from sfa.util.debug import log class get_credential(Method): @@ -44,8 +46,8 @@ class get_credential(Method): if not auth_hrn or hrn == self.api.config.SFA_INTERFACE_HRN: auth_hrn = hrn auth_info = self.api.auth.get_auth_info(auth_hrn) - table = self.api.auth.get_auth_table(auth_hrn) - records = table.resolve(type, hrn) + table = GeniTable() + records = table.find({'type': type, 'hrn': hrn}) if not records: raise RecordNotFound(hrn) record = records[0] @@ -59,12 +61,14 @@ class get_credential(Method): # TODO: Check permission that self.client_cred can access the object - object_gid = record.get_gid_object() - new_cred = Credential(subject = object_gid.get_subject()) + gid = record['gid'] + gid_object = GID(string=gid) + + new_cred = Credential(subject = gid_object.get_subject()) new_cred.set_gid_caller(self.api.auth.client_gid) - new_cred.set_gid_object(object_gid) + new_cred.set_gid_object(gid_object) new_cred.set_issuer(key=auth_info.get_pkey_object(), subject=auth_hrn) - new_cred.set_pubkey(object_gid.get_pubkey()) + new_cred.set_pubkey(gid_object.get_pubkey()) new_cred.set_privileges(rights) new_cred.set_delegate(True) @@ -103,13 +107,11 @@ class get_credential(Method): # find a record that matches record = None - table = self.api.auth.get_auth_table(auth_hrn) - records = table.resolve('*', hrn) - for rec in records: - if type in ['*'] or rec.get_type() in [type]: - record = rec - if not record: + table = GeniTable() + records = table.find({'type': type, 'hrn': hrn}) + if not records: raise RecordNotFound(hrn) + record = records[0] gid = record.get_gid_object() peer_cert = self.api.auth.peer_cert if not peer_cert.is_pubkey(gid.get_pubkey()): diff --git a/sfa/methods/list.py b/sfa/methods/list.py index 5ca15aac..73631c76 100644 --- a/sfa/methods/list.py +++ b/sfa/methods/list.py @@ -6,6 +6,7 @@ from sfa.util.method import Method from sfa.util.parameter import Parameter, Mixed from sfa.trust.auth import Auth from sfa.util.record import GeniRecord +from sfa.util.genitable import GeniTable from sfa.server.registry import Registries from sfa.util.prefixTree import prefixTree from sfa.trust.credential import Credential @@ -30,11 +31,11 @@ class list(Method): def call(self, cred, hrn, caller_cred=None): self.api.auth.check(cred, 'list') - if caller_cred==None: - caller_cred=cred + if caller_cred==None: + caller_cred=cred - #log the call - self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, Credential(string=caller_cred).get_gid_caller().get_hrn(), hrn, self.name)) + #log the call + self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, Credential(string=caller_cred).get_gid_caller().get_hrn(), hrn, self.name)) records = [] # load all know registry names into a prefix tree and attempt to find @@ -52,7 +53,7 @@ class list(Method): # if the best match (longest matching hrn) is not the local registry, # forward the request if registry_hrn != self.api.hrn: - credential = self.api.getCredential() + credential = self.api.getCredential() try: record_list = registries[registry_hrn].list(credential, hrn, caller_cred=caller_cred) records = [record.as_dict() for record in record_list] @@ -65,7 +66,7 @@ class list(Method): if not self.api.auth.hierarchy.auth_exists(hrn): raise MissingAuthority(hrn) - table = self.api.auth.get_auth_table(hrn) - records = table.list() - + table = GeniTable() + records = table.find({'authority': hrn}) + return records diff --git a/sfa/methods/register.py b/sfa/methods/register.py index a27cedfd..31220ea3 100644 --- a/sfa/methods/register.py +++ b/sfa/methods/register.py @@ -9,8 +9,8 @@ from sfa.util.misc import * from sfa.util.method import Method from sfa.util.parameter import Parameter, Mixed from sfa.util.record import GeniRecord +from sfa.util.genitable import GeniTable from sfa.util.debug import log - from sfa.trust.auth import Auth from sfa.trust.gid import create_uuid @@ -38,12 +38,12 @@ class register(Method): def call(self, cred, record_dict): self.api.auth.check(cred, "register") record = GeniRecord(dict = record_dict) - type = record.get_type() - name = record.get_name() - self.api.auth.verify_object_permission(name) - auth_name = self.api.auth.get_authority(name) + table = GeniTable() + type = record['type'] + hrn = record['hrn'] + auth_name = get_authority(hrn) + self.api.auth.verify_object_permission(hrn) auth_info = self.api.auth.get_auth_info(auth_name) - table = self.api.auth.get_auth_table(auth_name) pub_key = None # make sure record has a gid if 'gid' not in record: @@ -56,31 +56,30 @@ class register(Method): pub_key = record['key'] pkey = convert_public_key(pub_key) - gid_object = self.api.auth.hierarchy.create_gid(name, uuid, pkey) + gid_object = self.api.auth.hierarchy.create_gid(hrn, uuid, pkey) gid = gid_object.save_to_string(save_parents=True) record['gid'] = gid record.set_gid(gid) # check if record already exists - existing_records = table.resolve(type, name) + existing_records = table.find({'type': type, 'hrn': hrn}) if existing_records: - raise ExistingRecord(name) + raise ExistingRecord(hrn) if type in ["authority"]: # update the tree - if not self.api.auth.hierarchy.auth_exists(name): - self.api.auth.hierarchy.create_auth(name) + if not self.api.auth.hierarchy.auth_exists(hrn): + self.api.auth.hierarchy.create_auth(hrn) # authorities are special since they are managed by the registry # rather than by the caller. We create our own GID for the # authority rather than relying on the caller to supply one. # get the GID from the newly created authority - child_auth_info = self.api.auth.get_auth_info(name) gid = auth_info.get_gid_object() record.set_gid(gid.save_to_string(save_parents=True)) - pl_record = self.api.geni_fields_to_pl_fields(type, name, record) + pl_record = self.api.geni_fields_to_pl_fields(type, hrn, record) sites = self.api.plshell.GetSites(self.api.plauth, [pl_record['login_base']]) if not sites: pointer = self.api.plshell.AddSite(self.api.plauth, pl_record) @@ -90,7 +89,7 @@ class register(Method): record.set_pointer(pointer) elif (type == "slice"): - pl_record = self.api.geni_fields_to_pl_fields(type, name, record) + pl_record = self.api.geni_fields_to_pl_fields(type, hrn, record) slices = self.api.plshell.GetSlices(self.api.plauth, [pl_record['name']]) if not slices: pointer = self.api.plshell.AddSlice(self.api.plauth, pl_record) @@ -107,18 +106,18 @@ class register(Method): if 'enabled' in record and record['enabled']: self.api.plshell.UpdatePerson(self.api.plauth, pointer, {'enabled': record['enabled']}) - login_base = get_leaf(auth_info.hrn) + login_base = get_leaf(auth_name) self.api.plshell.AddPersonToSite(self.api.plauth, pointer, login_base) # What roles should this user have? self.api.plshell.AddRoleToPerson(self.api.plauth, 'user', pointer) record.set_pointer(pointer) - # Add the user's key + # Add the user's key if pub_key: self.api.plshell.AddPersonKey(self.api.plauth, pointer, {'key_type' : 'ssh', 'key' : pub_key}) elif (type == "node"): - pl_record = self.api.geni_fields_to_pl_fields(type, name, record) + pl_record = self.api.geni_fields_to_pl_fields(type, hrn, record) login_base = hrn_to_pl_login_base(auth_name) nodes = self.api.plshell.GetNodes(self.api.plauth, [pl_record['hostname']]) if not nodes: @@ -132,7 +131,7 @@ class register(Method): # SFA upcalls may exist in PLCAPI and they could have already added the # record for us. Lets check if the record already exists - existing_records = table.resolve(type, name) + existing_records = table.find({'type': type, 'hrn': hrn}) if not existing_records: table.insert(record) diff --git a/sfa/methods/remove.py b/sfa/methods/remove.py index dade3fb8..e66fb30c 100644 --- a/sfa/methods/remove.py +++ b/sfa/methods/remove.py @@ -6,6 +6,7 @@ from sfa.util.method import Method from sfa.util.parameter import Parameter, Mixed from sfa.trust.auth import Auth from sfa.util.record import GeniRecord +from sfa.util.genitable import GeniTable from sfa.util.debug import log class remove(Method): @@ -33,29 +34,31 @@ class remove(Method): def call(self, cred, type, hrn): self.api.auth.check(cred, "remove") self.api.auth.verify_object_permission(hrn) - auth_name = self.api.auth.get_authority(hrn) - table = self.api.auth.get_auth_table(auth_name) - record_list = table.resolve(type, hrn) - if not record_list: + table = GeniTable() + filter = {'hrn': hrn} + if type not in ['all', '*']: + filter['type'] = type + records = table.find(filter) + if not records: raise RecordNotFound(hrn) - record = record_list[0] + record = records[0] type = record['type'] if type == "user": - persons = self.api.plshell.GetPersons(self.api.plauth, record.get_pointer()) + persons = self.api.plshell.GetPersons(self.api.plauth, record['pointer']) # only delete this person if he has site ids. if he doesnt, it probably means # he was just removed from a site, not actually deleted if persons and persons[0]['site_ids']: - self.api.plshell.DeletePerson(self.api.plauth, record.get_pointer()) + self.api.plshell.DeletePerson(self.api.plauth, record['pointer']) elif type == "slice": - if self.api.plshell.GetSlices(self.api.plauth, record.get_pointer()): - self.api.plshell.DeleteSlice(self.api.plauth, record.get_pointer()) + if self.api.plshell.GetSlices(self.api.plauth, record['pointer']): + self.api.plshell.DeleteSlice(self.api.plauth, record['pointer']) elif type == "node": - if self.api.plshell.GetNodes(self.api.plauth, record.get_pointer()): - self.api.plshell.DeleteNode(self.api.plauth, record.get_pointer()) + if self.api.plshell.GetNodes(self.api.plauth, record['pointer']): + self.api.plshell.DeleteNode(self.api.plauth, record['pointer']) elif type == "authority": - if self.api.plshell.GetSites(self.api.plauth, record.get_pointer()): - self.api.plshell.DeleteSite(self.api.plauth, record.get_pointer()) + if self.api.plshell.GetSites(self.api.plauth, record['pointer']): + self.api.plshell.DeleteSite(self.api.plauth, record['pointer']) else: raise UnknownGeniType(type) diff --git a/sfa/methods/resolve.py b/sfa/methods/resolve.py index 28d6d290..6e6f7b68 100644 --- a/sfa/methods/resolve.py +++ b/sfa/methods/resolve.py @@ -6,6 +6,7 @@ from sfa.util.method import Method from sfa.util.parameter import Parameter, Mixed from sfa.trust.auth import Auth from sfa.util.record import GeniRecord +from sfa.util.genitable import GeniTable from sfa.util.debug import log from sfa.server.registry import Registries from sfa.util.prefixTree import prefixTree @@ -58,11 +59,8 @@ class resolve(Method): traceback.print_exc() # if we still havnt found the record yet, try the local registry - auth_hrn = self.api.auth.get_authority(hrn) - if not auth_hrn: - auth_hrn = hrn - table = self.api.auth.get_auth_table(auth_hrn) - records = table.resolve('*', hrn) + table = GeniTable() + records = table.find(hrn) if not records: raise RecordNotFound(hrn) for record in records: diff --git a/sfa/methods/update.py b/sfa/methods/update.py index 768116a4..ff40f58a 100644 --- a/sfa/methods/update.py +++ b/sfa/methods/update.py @@ -6,6 +6,7 @@ from sfa.util.method import Method from sfa.util.parameter import Parameter, Mixed from sfa.trust.auth import Auth from sfa.util.record import GeniRecord +from sfa.util.genitable import GeniTable from sfa.trust.certificate import Keypair, convert_public_key from sfa.trust.gid import * from sfa.util.debug import log @@ -33,36 +34,32 @@ class update(Method): def call(self, cred, record_dict): self.api.auth.check(cred, "update") - record = GeniRecord(dict = record_dict) - type = record.get_type() - self.api.auth.verify_object_permission(record.get_name()) - auth_name = self.api.auth.get_authority(record.get_name()) - if not auth_name: - auth_name = record.get_name() - table = self.api.auth.get_auth_table(auth_name) - + new_record = GeniRecord(dict = record_dict) + type = new_record['type'] + hrn = new_record['hrn'] + self.api.auth.verify_object_permission(hrn) + table = GeniTable() # make sure the record exists - existing_record_list = table.resolve(type, record.get_name()) - if not existing_record_list: - raise RecordNotFound(record.get_name()) - existing_record = existing_record_list[0] - + records = table.find({'type': type, 'hrn': hrn}) + if not records: + raise RecordNotFound(hrn) + record = records[0] + # Update_membership needs the membership lists in the existing record # filled in, so it can see if members were added or removed - self.api.fill_record_info(existing_record) + self.api.fill_record_info(record) # Use the pointer from the existing record, not the one that the user # gave us. This prevents the user from inserting a forged pointer - pointer = existing_record.get_pointer() + pointer = record['pointer'] # update the PLC information that was specified with the record if (type == "authority"): - self.api.plshell.UpdateSite(self.api.plauth, pointer, record) + self.api.plshell.UpdateSite(self.api.plauth, pointer, new_record) elif type == "slice": - hrn=record.get_name() - pl_record=self.api.geni_fields_to_pl_fields(type, hrn, record) + pl_record=self.api.geni_fields_to_pl_fields(type, hrn, new_record) if 'name' in pl_record: pl_record.pop('name') self.api.plshell.UpdateSlice(self.api.plauth, pointer, pl_record) @@ -72,7 +69,7 @@ class update(Method): # updated. Ideally we should have a more generic way of doing # this. I copied the field names from UpdatePerson.py... update_fields = {} - all_fields = record + all_fields = new_record for key in all_fields.keys(): if key in ['first_name', 'last_name', 'title', 'email', 'password', 'phone', 'url', 'bio', 'accepted_aup', @@ -80,51 +77,43 @@ class update(Method): update_fields[key] = all_fields[key] self.api.plshell.UpdatePerson(self.api.plauth, pointer, update_fields) - if 'key' in record and record['key']: + if 'key' in new_record and new_record['key']: # must check this key against the previous one if it exists persons = self.api.plshell.GetPersons(self.api.plauth, [pointer], ['key_ids']) person = persons[0] keys = person['key_ids'] keys = self.api.plshell.GetKeys(self.api.plauth, person['key_ids']) key_exists = False - if isinstance(record['key'], list): - new_key = record['key'][0] + if isinstance(new_record['key'], list): + new_key = new_record['key'][0] else: - new_key = record['key'] + new_key = new_record['key'] # Delete all stale keys for key in keys: - if record['key'] != key['key']: + if new_record['key'] != key['key']: self.api.plshell.DeleteKey(self.api.plauth, key['key_id']) else: key_exists = True if not key_exists: self.api.plshell.AddPersonKey(self.api.plauth, pointer, {'key_type': 'ssh', 'key': new_key}) - # find the existing geni record - hrn = record['hrn'] - auth_name = self.api.auth.get_authority(hrn) - auth_info = self.api.auth.get_auth_info(auth_name) - table = self.api.auth.get_auth_table(auth_name) - person_records = table.resolve('user', hrn) - person_record = person_records[0] - # update the openssl key and gid pkey = convert_public_key(new_key) uuid = create_uuid() gid_object = self.api.auth.hierarchy.create_gid(hrn, uuid, pkey) gid = gid_object.save_to_string(save_parents=True) record['gid'] = gid - record.set_gid(gid) + record = GeniRecord(dict=record) table.update(record) elif type == "node": - self.api.plshell.UpdateNode(self.api.plauth, pointer, record) + self.api.plshell.UpdateNode(self.api.plauth, pointer, new_record) else: raise UnknownGeniType(type) # update membership for researchers, pis, owners, operators - self.api.update_membership(existing_record, record) + self.api.update_membership(record, record) return 1 diff --git a/sfa/plc/api.py b/sfa/plc/api.py index 2ac66e94..cc6430b7 100644 --- a/sfa/plc/api.py +++ b/sfa/plc/api.py @@ -19,6 +19,7 @@ from sfa.trust.rights import * from sfa.trust.credential import * from sfa.util.misc import * from sfa.util.sfalogging import * +from sfa.util.genitable import * # See "2.2 Characters" in the XML specification: # @@ -120,7 +121,7 @@ class GeniAPI: self.plshell_version = self.getPLCShellVersion() self.hrn = self.config.SFA_INTERFACE_HRN self.time_format = "%Y-%m-%d %H:%M:%S" - self.logger=get_sfa_logger() + self.logger=get_sfa_logger() def getPLCShell(self): self.plauth = {'Username': self.config.SFA_PLC_USER, @@ -193,12 +194,12 @@ class GeniAPI: if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN: auth_hrn = hrn auth_info = self.auth.get_auth_info(auth_hrn) - table = self.auth.get_auth_table(auth_hrn) - records = table.resolve('*', hrn) + table = GeniTable() + records = table.find(hrn) if not records: raise RecordNotFound record = records[0] - type = record.get_type() + type = record['type'] object_gid = record.get_gid_object() new_cred = Credential(subject = object_gid.get_subject()) new_cred.set_gid_caller(object_gid) @@ -297,8 +298,8 @@ class GeniAPI: @param record: record to fill in field (in/out param) """ - type = record.get_type() - pointer = record.get_pointer() + type = record['type'] + pointer = record['pointer'] auth_hrn = self.hrn login_base = '' # records with pointer==-1 do not have plc info associated with them. @@ -320,7 +321,7 @@ class GeniAPI: raise UnknownGeniType(type) if not pl_res: - raise PlanetLabRecordDoesNotExist(record.get_name()) + raise PlanetLabRecordDoesNotExist(record['hrn']) # convert ids to hrns pl_record = pl_res[0] @@ -360,32 +361,31 @@ class GeniAPI: record.update(pl_record) - def lookup_users(self, auth_table, user_id_list, role="*"): + def lookup_users(self, user_id_list, role="*"): + table = GeniTable() record_list = [] for person_id in user_id_list: - user_records = auth_table.find("user", person_id, "pointer") + user_records = table.find({'type': 'user', 'pointer': person_id}) for user_record in user_records: self.fill_record_info(user_record) user_roles = user_record.get("roles") if (role=="*") or (role in user_roles): - record_list.append(user_record.get_name()) + record_list.append(user_record['hrn']) return record_list def fill_record_geni_info(self, record): geni_info = {} - type = record.get_type() + type = record['type'] if (type == "slice"): - auth_table = self.auth.get_auth_table(self.auth.get_authority(record.get_name())) person_ids = record.get("person_ids", []) - researchers = self.lookup_users(auth_table, person_ids) + researchers = self.lookup_users(person_ids) geni_info['researcher'] = researchers elif (type == "authority"): - auth_table = self.auth.get_auth_table(record.get_name()) person_ids = record.get("person_ids", []) - pis = self.lookup_users(auth_table, person_ids, "pi") - operators = self.lookup_users(auth_table, person_ids, "tech") - owners = self.lookup_users(auth_table, person_ids, "admin") + pis = self.lookup_users(person_ids, "pi") + operators = self.lookup_users(person_ids, "tech") + owners = self.lookup_users(person_ids, "admin") geni_info['pi'] = pis geni_info['operator'] = operators geni_info['owner'] = owners @@ -424,16 +424,10 @@ class GeniAPI: # build a list of the new person ids, by looking up each person to get # their pointer newIdList = [] - for hrn in newList: - auth_hrn = self.auth.get_authority(hrn) - if not auth_hrn: - auth_hrn = hrn - auth_info = self.auth.get_auth_info(auth_hrn) - table = self.auth.get_auth_table(auth_hrn) - records = table.resolve('user', hrn) - if records: - userRecord = records[0] - newIdList.append(userRecord.get_pointer()) + table = GeniTable() + records = table.find({'type': 'user', 'hrn': newList}) + for record in records: + newIdList.append(record['pointer']) # build a list of the old person ids from the person_ids field if oldRecord: diff --git a/sfa/plc/sfa-import-plc.py b/sfa/plc/sfa-import-plc.py index 9b12379c..5da81bbb 100755 --- a/sfa/plc/sfa-import-plc.py +++ b/sfa/plc/sfa-import-plc.py @@ -53,7 +53,9 @@ def main(): plc_auth = sfaImporter.plc_auth AuthHierarchy = sfaImporter.AuthHierarchy TrustedRoots = sfaImporter.TrustedRoots - + table = GeniTable() + table.create() + if not level1_auth or level1_auth in ['']: level1_auth = None @@ -76,7 +78,7 @@ def main(): i2site = {'name': 'Internet2', 'abbreviated_name': 'I2', 'login_base': 'internet2', 'site_id': -1} sfaImporter.import_site(import_auth, i2site) - + for site in sites: sfaImporter.import_site(import_auth, site) diff --git a/sfa/plc/sfaImport.py b/sfa/plc/sfaImport.py index 71fbce75..f362984f 100644 --- a/sfa/plc/sfaImport.py +++ b/sfa/plc/sfaImport.py @@ -83,21 +83,6 @@ class sfaImport: import PLC.Shell self.shell = PLC.Shell.Shell(globals = globals()) - def get_auth_table(self, auth_name): - AuthHierarchy = self.AuthHierarchy - auth_info = AuthHierarchy.get_auth_info(auth_name) - - table = GeniTable(hrn=auth_name, cninfo=auth_info.get_dbinfo()) - - # if the table doesn't exist, then it means we haven't put any records - # into this authority yet. - - if not table.exists(): - trace("Import: creating table for authority " + auth_name) - table.create() - - return table - def create_top_level_auth_records(self, hrn): AuthHierarchy = self.AuthHierarchy @@ -118,9 +103,9 @@ class sfaImport: parent_hrn = hrn auth_info = AuthHierarchy.get_auth_info(parent_hrn) - table = self.get_auth_table(parent_hrn) + table = GeniTable() + auth_record = table.find({'type': 'authority', 'hrn': hrn}) - auth_record = table.resolve("authority", hrn) if not auth_record: auth_record = GeniRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1) trace(" inserting authority record for " + hrn) @@ -137,7 +122,7 @@ class sfaImport: trace("Import: importing person " + hrn) - table = self.get_auth_table(parent_hrn) + table = GeniTable() key_ids = [] if 'key_ids' in person and person['key_ids']: @@ -158,7 +143,7 @@ class sfaImport: # create the gid person_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey) - person_record = table.resolve("user", hrn) + person_record = table.find({'type': 'user', 'hrn': hrn}) if not person_record: trace(" inserting user record for " + hrn) person_record = GeniRecord(hrn=hrn, gid=person_gid, type="user", pointer=person['person_id']) @@ -180,9 +165,9 @@ class sfaImport: hrn = parent_hrn + "." + slicename trace("Import: importing slice " + hrn) - table = self.get_auth_table(parent_hrn) + table = GeniTable() - slice_record = table.resolve("slice", hrn) + slice_record = table.find({'type': 'sslice', 'hrn': hrn}) if not slice_record: pkey = Keypair(create=True) slice_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey) @@ -207,9 +192,9 @@ class sfaImport: trace("Import: importing node " + hrn) - table = self.get_auth_table(parent_hrn) + table = GeniTable() - node_record = table.resolve("node", hrn) + node_record = table.find({'type': 'node', 'hrn': hrn}) if not node_record: pkey = Keypair(create=True) node_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey) @@ -247,9 +232,9 @@ class sfaImport: auth_info = AuthHierarchy.get_auth_info(hrn) - table = self.get_auth_table(parent_hrn) + table = GeniTable() - auth_record = table.resolve("authority", hrn) + auth_record = table.find({'type': 'authority', 'hrn': 'hrn'}) if not auth_record: auth_record = GeniRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id']) trace(" inserting authority record for " + hrn) @@ -282,6 +267,7 @@ class sfaImport: def delete_record(self, parent_hrn, object, type): # get the hrn + table = GeniTable() hrn = None if type in ['slice'] and 'name' in object and object['name']: slice_name = object['name'].split("_")[0] @@ -297,16 +283,14 @@ class sfaImport: hrn = parent_hrn parent_hrn = get_authority(hrn) type = "authority" - # delete the site table - site_table = self.get_auth_table(hrn) - site_table.drop() + # delete all records whos authority is this site + records = table.find({'authority': hrn}) + for record in records: + table.remove(record) else: return # delete the record - table = self.get_auth_table(parent_hrn) - record_list = table.resolve(type, hrn) - if not record_list: - return - record = record_list[0] - table.remove(record) + record_list = table.find({'type': type, 'hrn': hrn}) + for record in record_list: + table.remove(record) diff --git a/sfa/trust/auth.py b/sfa/trust/auth.py index 34cbddcc..923bed4b 100644 --- a/sfa/trust/auth.py +++ b/sfa/trust/auth.py @@ -76,26 +76,6 @@ class Auth: return self.hierarchy.get_auth_info(auth_hrn) - def get_auth_table(self, auth_name): - """ - Given an authority name, return the database table for that authority. - If the databse table does not exist, then one will be automatically - created. - - @param auth_name human readable name of authority - """ - auth_info = self.get_auth_info(auth_name) - table = GeniTable(hrn=auth_name, - cninfo=auth_info.get_dbinfo()) - # if the table doesn't exist, then it means we haven't put any records - # into this authority yet. - - if not table.exists(): - print >> log, "Registry: creating table for authority", auth_name - table.create() - - return table - def veriry_auth_belongs_to_me(self, name): """ Verify that an authority belongs to our hierarchy. @@ -161,7 +141,7 @@ class Auth: verify_cancreate_credential() """ - type = record.get_type() + type = record['type'] if src_cred: cred_object_hrn = src_cred.get_gid_object().get_hrn() else: diff --git a/sfa/util/genitable.py b/sfa/util/genitable.py index 4ad582ba..f3cc2ce1 100644 --- a/sfa/util/genitable.py +++ b/sfa/util/genitable.py @@ -8,30 +8,31 @@ ### $URL$ import report - +import pgdb from pg import DB, ProgrammingError - from sfa.trust.gid import * from sfa.util.record import * from sfa.util.debug import * +from sfa.util.config import * +from sfa.util.filter import * -class GeniTable: +class GeniTable(list): - GENI_TABLE_PREFIX = "sfa$" + GENI_TABLE_PREFIX = "sfa" - def __init__(self, create=False, hrn="unspecified.default.registry", cninfo=None): - - self.hrn = hrn + def __init__(self, record_filter = None): # pgsql doesn't like table names with "." in them, to replace it with "$" - self.tablename = GeniTable.GENI_TABLE_PREFIX + self.hrn.replace(".", "$") + self.tablename = GeniTable.GENI_TABLE_PREFIX # establish a connection to the pgsql server + cninfo = Config().get_plc_dbinfo() self.cnx = DB(cninfo['dbname'], cninfo['address'], port=cninfo['port'], user=cninfo['user'], passwd=cninfo['password']) - # if asked to create the table, then create it - if create: - self.create() + if record_filter: + records = self.find(record_filter) + for record in reocrds: + self.append(record) def exists(self): tableList = self.cnx.get_tables() @@ -44,16 +45,17 @@ class GeniTable: def create(self): querystr = "CREATE TABLE " + self.tablename + " ( \ - key text, \ - hrn text, \ + record_id serial PRIMARY KEY , \ + hrn text NOT NULL, \ + authority text NOT NULL, \ gid text, \ - type text, \ + type text NOT NULL, \ pointer integer, \ date_created timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, \ last_updated timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP);" template = "CREATE INDEX %s_%s_idx ON %s (%s);" indexes = [template % ( self.tablename, field, self.tablename, field) \ - for field in ['key', 'hrn', 'type','pointer']] + for field in ['hrn', 'type', 'authority', 'pointer']] # IF EXISTS doenst exist in postgres < 8.2 try: self.cnx.query('DROP TABLE IF EXISTS ' + self.tablename) @@ -62,19 +64,19 @@ class GeniTable: self.cnx.query('DROP TABLE ' + self.tablename) except ProgrammingError: pass - + self.cnx.query(querystr) for index in indexes: self.cnx.query(index) def remove(self, record): - query_str = "DELETE FROM " + self.tablename + " WHERE key = '" + record.get_key() + "'" + query_str = "DELETE FROM %s WHERE record_id = %s" % (self.tablename, record['record_id']) self.cnx.query(query_str) def insert(self, record): dont_insert = ['date_created', 'last_updated'] fields = [field for field in record.fields.keys() if field not in dont_insert] - fieldnames = ["key", "pointer"] + fields + fieldnames = ["pointer"] + fields fieldvals = record.get_field_value_strings(fieldnames) query_str = "INSERT INTO " + self.tablename + \ "(" + ",".join(fieldnames) + ") " + \ @@ -92,23 +94,48 @@ class GeniTable: pairs.append(field + " = " + val) update = ", ".join(pairs) - query_str = "UPDATE " + self.tablename+ " SET " + update + " WHERE key = '" + record.get_key() + "'" + query_str = "UPDATE %s SET %s WHERE record_id = %s" % \ + (self.tablename, update, record['record_id']) #print query_str self.cnx.query(query_str) - def find_dict(self, type, value, searchfield): - query_str = "SELECT * FROM " + self.tablename + " WHERE " + searchfield + " = '" + str(value) + "'" - dict_list = self.cnx.query(query_str).dictresult() - result_dict_list = [] - for dict in dict_list: - if (type=="*") or (dict['type'] == type): - result_dict_list.append(dict) - return result_dict_list - - def find(self, type, value, searchfield): - result_dict_list = self.find_dict(type, value, searchfield) + def quote(self, value): + """ + Returns quoted version of the specified value. + """ + + # The pgdb._quote function is good enough for general SQL + # quoting, except for array types. + if isinstance(value, (list, tuple, set)): + return "ARRAY[%s]" % ", ".join(map, self.quote, value) + else: + return pgdb._quote(value) + + def find(self, record_filter = None): + sql = "SELECT * FROM %s WHERE True " % self.tablename + + if isinstance(record_filter, (list, tuple, set)): + ints = filter(lambda x: isinstance(x, (int, long)), record_filter) + strs = filter(lambda x: isinstance(x, StringTypes), record_filter) + record_filter = Filter(GeniRecord.all_fields, {'record_id': ints, 'hrn': strs}) + sql += "AND (%s) %s " % record_filter.sql("OR") + elif isinstance(record_filter, dict): + record_filter = Filter(GeniRecord.all_fields, record_filter) + sql += " AND (%s) %s" % record_filter.sql("AND") + elif isinstance(record_filter, StringTypes): + record_filter = Filter(GeniRecord.all_fields, {'hrn':[record_filter]}) + sql += " AND (%s) %s" % record_filter.sql("AND") + elif isinstance(record_filter, int): + record_filter = Filter(GeniRecord.all_fields, {'record_id':[record_filter]}) + sql += " AND (%s) %s" % record_filter.sql("AND") + results = self.cnx.query(sql).dictresult() + return results + + def findObjects(self, record_filter = None): + + results = self.find(record_filter) result_rec_list = [] - for result in result_dict_list: + for result in results: if result['type'] in ['authority']: result_rec_list.append(AuthorityRecord(dict=result)) elif result['type'] in ['node']: @@ -121,23 +148,6 @@ class GeniTable: result_rec_list.append(GeniRecord(dict=result)) return result_rec_list - def resolve_dict(self, type, hrn): - return self.find_dict(type, hrn, "hrn") - - def resolve(self, type, hrn): - return self.find(type, hrn, "hrn") - - def list_dict(self): - query_str = "SELECT * FROM " + self.tablename - result_dict_list = self.cnx.query(query_str).dictresult() - return result_dict_list - - def list(self): - result_dict_list = self.list_dict() - result_rec_list = [] - for dict in result_dict_list: - result_rec_list.append(GeniRecord(dict=dict).as_dict()) - return result_rec_list def drop(self): try: diff --git a/sfa/util/record.py b/sfa/util/record.py index b76e7ea1..f2a455bf 100644 --- a/sfa/util/record.py +++ b/sfa/util/record.py @@ -14,6 +14,8 @@ from sfa.trust.gid import * import sfa.util.report from sfa.util.rspec import * from sfa.util.parameter import * +from sfa.util.misc import * + class GeniRecord(dict): """ @@ -36,14 +38,20 @@ class GeniRecord(dict): """ ### the wsdl generator assumes this is named 'fields' + internal_fields = { + 'record_id': Parameter(int, 'An id that uniquely identifies this record'), + 'pointer': Parameter(int, 'An id that uniquely identifies this record in an external database ') + } + fields = { + 'authority': Parameter(str, "The authority for this record"), 'hrn': Parameter(str, "Human readable name of object"), 'gid': Parameter(str, "GID of the object"), 'type': Parameter(str, "Record type"), 'last_updated': Parameter(int, 'Date and time of last update'), 'date_created': Parameter(int, 'Date and time this record was created'), } - + all_fields = dict(fields.items() + internal_fields.items()) ## # Create a Geni Record # @@ -71,8 +79,7 @@ class GeniRecord(dict): self.load_from_dict(dict) if string: self.load_from_string(string) - - + def update(self, new_dict): if isinstance(new_dict, list): new_dict = new_dict[0] @@ -96,6 +103,7 @@ class GeniRecord(dict): Set the name of the record """ self.hrn = hrn + self['hrn'] = hrn self.dirty = True ## @@ -110,8 +118,10 @@ class GeniRecord(dict): if isinstance(gid, StringTypes): self.gid = gid + self['gid'] = gid else: self.gid = gid.save_to_string(save_parents=True) + self['gid'] = gid.save_to_string(save_parents=True) self.dirty = True ## @@ -124,6 +134,7 @@ class GeniRecord(dict): Set the type of the record """ self.type = type + self['type'] = type self.dirty = True ## @@ -136,6 +147,7 @@ class GeniRecord(dict): Set the pointer of the record """ self.pointer = pointer + self['pointer'] = pointer self.dirty = True ## @@ -180,19 +192,6 @@ class GeniRecord(dict): """ return GID(string=self.gid) - ## - # Return a key that uniquely identifies this record among all records in - # Geni. This key is used to uniquely identify the record in the Geni - # database. - - def get_key(self): - """ - Return a key that uniquely identifies this record among all records in - Geni. This key is used to uniquely identify the record in the Geni - database. - """ - return self.hrn + "#" + self.type - ## # Returns a list of field names in this record. @@ -211,8 +210,8 @@ class GeniRecord(dict): """ Given a field name ("hrn", "gid", ...) return the value of that field. """ - if fieldname == "key": - val = self.get_key() + if fieldname == "authority": + val = get_authority(self['hrn']) else: try: val = getattr(self, fieldname) -- 2.47.0