From: Tony Mack Date: Sun, 21 Feb 2010 04:17:56 +0000 (+0000) Subject: modify fill_record_pl_info() to handle multiple records more efficiently X-Git-Tag: sfa-0.9-11~81 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=377d8ce849184c5e89c7b7ae70d33761ac6d4972 modify fill_record_pl_info() to handle multiple records more efficiently --- diff --git a/sfa/plc/api.py b/sfa/plc/api.py index a809ca84..9bfd10ae 100644 --- a/sfa/plc/api.py +++ b/sfa/plc/api.py @@ -22,6 +22,11 @@ from sfa.util.api import * from sfa.util.nodemanager import NodeManager from sfa.util.sfalogging import * +def list_to_dict(recs, key): + keys = [rec[key] for rec in recs] + return dict(zip(keys, recs)) + + class SfaAPI(BaseAPI): # flat list of method names @@ -40,6 +45,7 @@ class SfaAPI(BaseAPI): self.SfaTable = SfaTable # Better just be documenting the API if config is None: + print "CONFIG IS NONE" return # Load configuration @@ -51,7 +57,6 @@ class SfaAPI(BaseAPI): self.cert_file = cert_file self.cert = Certificate(filename=self.cert_file) self.credential = None - # Initialize the PLC shell only if SFA wraps a myPLC rspec_type = self.config.get_aggregate_type() if (rspec_type == 'pl' or rspec_type == 'vini'): @@ -235,7 +240,7 @@ class SfaAPI(BaseAPI): return pl_record - def fill_record_pl_info(self, record): + def fill_record_pl_info(self, records): """ Fill in the planetlab specific fields of a SFA record. This involves calling the appropriate PLC method to retrieve the @@ -245,69 +250,136 @@ class SfaAPI(BaseAPI): @param record: record to fill in field (in/out param) """ - type = record['type'] - pointer = record['pointer'] - auth_hrn = self.hrn - login_base = '' - # records with pointer==-1 do not have plc info associated with them. - # for example, the top level authority records which are - # authorities, but not PL "sites" - if pointer == -1: - record.update({}) - return + # get ids by type + node_ids, site_ids, slice_ids = [], [], [] + person_ids, key_ids = [], [] + type_map = {'node': node_ids, 'authority': site_ids, + 'slice': slice_ids, 'user': person_ids} + + for record in records: + for type in type_map: + if type == record['type']: + type_map[type].append(record['pointer']) + + # get pl records + nodes, sites, slices, persons, keys = {}, {}, {}, {}, {} + if node_ids: + node_list = self.plshell.GetNodes(self.plauth, node_ids) + nodes = list_to_dict(node_list, 'node_id') + if site_ids: + site_lists = self.plshell.GetSites(self.plauth, site_ids) + sites = list_to_dict(site_list, 'site_id') + if slice_ids: + slice_list = self.plshell.GetSlices(self.plauth, slice_ids) + slices = list_to_dict(slice_list, 'slice_id') + if person_ids: + person_list = self.plshell.GetPersons(self.plauth, person_ids) + persons = list_to_dict(person_list, 'person_id') + for person in persons: + key_ids.extend(persons[person]['key_ids']) + + pl_records = {'node': nodes, 'authority': sites, + 'slice': slices, 'user': persons} + + if key_ids: + key_list = self.plshell.GetKeys(self.plauth, key_ids) + keys = list_to_dict(key_list, 'key_id') + + # fill record info + for record in records: + # records with pointer==-1 do not have plc info associated with them. + # for example, the top level authority records which are + # authorities, but not PL "sites" + if record['pointer'] == -1: + continue + + for type in pl_records: + if record['type'] == type: + if record['pointer'] in pl_records[type]: + record.update(pl_records[type][record['pointer']]) + break + # fill in key info + if record['type'] == 'user': + pubkeys = [keys[key_id]['key'] for key_id in record['key_ids'] if key_id in keys] + record['keys'] = pubkeys + + # fill in record hrns + records = self.fill_record_hrns(records) + + return records - if (type in ["authority"]): - pl_res = self.plshell.GetSites(self.plauth, [pointer]) - elif (type == "slice"): - pl_res = self.plshell.GetSlices(self.plauth, [pointer]) - elif (type == "user"): - pl_res = self.plshell.GetPersons(self.plauth, [pointer]) - elif (type == "node"): - pl_res = self.plshell.GetNodes(self.plauth, [pointer]) - else: - raise UnknownSfaType(type) - - if not pl_res: - raise PlanetLabRecordDoesNotExist(record['hrn']) + def fill_record_hrns(self, records): + """ + convert pl ids to hrns + """ + # get ids + slice_ids, person_ids, site_ids, node_ids = [], [], [], [] + for record in records: + if 'site_id' in record: + site_ids.append(record['site_id']) + if 'site_ids' in records: + site_ids.extend(record['site_ids']) + if 'person_ids' in record: + person_ids.extend(record['person_ids']) + if 'slice_ids' in record: + slice_ids.extend(record['slice_ids']) + if 'node_ids' in record: + node_ids.extend(record['node_ids']) + + # get pl records + slices, persons, sites, nodes = {}, {}, {}, {} + if site_ids: + site_list = self.plshell.GetSites(self.plauth, site_ids, ['site_id', 'login_base']) + sites = list_to_dict(site_list, 'site_id') + if person_ids: + person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'email']) + persons = list_to_dict(person_list, 'person_id') + if slice_ids: + slice_list = self.plshell.GetSlices(self.plauth, slice_ids, ['slice_id', 'name']) + slices = list_to_dict(slice_list, 'slice_id') + if node_ids: + node_list = self.plshell.GetNodes(self.plauth, node_ids, ['node_id', 'hostname']) + nodes = list_to_dict(node_list, 'node_id') + # convert ids to hrns - pl_record = pl_res[0] - if 'site_id' in pl_record: - sites = self.plshell.GetSites(self.plauth, pl_record['site_id'], ['login_base']) - site = sites[0] - login_base = site['login_base'] - pl_record['site'] = ".".join([auth_hrn, login_base]) - if 'person_ids' in pl_record: - persons = self.plshell.GetPersons(self.plauth, pl_record['person_ids'], ['email']) - emails = [person['email'] for person in persons] - usernames = [email.split('@')[0] for email in emails] - person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames] - pl_record['persons'] = person_hrns - if 'slice_ids' in pl_record: - slices = self.plshell.GetSlices(self.plauth, pl_record['slice_ids'], ['name']) - slicenames = [slice['name'] for slice in slices] - slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames] - pl_record['slices'] = slice_hrns - if 'node_ids' in pl_record: - nodes = self.plshell.GetNodes(self.plauth, pl_record['node_ids'], ['hostname']) - hostnames = [node['hostname'] for node in nodes] - node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames] - pl_record['nodes'] = node_hrns - if 'site_ids' in pl_record: - sites = self.plshell.GetSites(self.plauth, pl_record['site_ids'], ['login_base']) - login_bases = [site['login_base'] for site in sites] - site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases] - pl_record['sites'] = site_hrns - if 'key_ids' in pl_record: - keys = self.plshell.GetKeys(self.plauth, pl_record['key_ids']) - pubkeys = [] - if keys: - pubkeys = [key['key'] for key in keys] - pl_record['keys'] = pubkeys - - record.update(pl_record) - - + for record in records: + + # get all relevant data + type = record['type'] + pointer = record['pointer'] + auth_hrn = self.hrn + login_base = '' + if pointer == -1: + continue + + if 'site_id' in record: + site = sites[record['site_id']] + login_base = site['login_base'] + record['site'] = ".".join([auth_hrn, login_base]) + if 'person_ids' in record: + emails = [persons[person_id]['email'] for person_id in record['person_ids'] \ + if person_id in persons] + usernames = [email.split('@')[0] for email in emails] + person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames] + record['persons'] = person_hrns + if 'slice_ids' in record: + slicenames = [slices[slice_id]['name'] for slice_id in record['slice_ids'] \ + if slice_id in slices] + slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames] + record['slices'] = slice_hrns + if 'node_ids' in record: + hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \ + if node_id in nodes] + node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames] + record['nodes'] = node_hrns + if 'site_ids' in record: + login_bases = [sites[site_id]['login_base'] for site_id in record['site_ids'] \ + if site_id in sites] + site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases] + record['sites'] = site_hrns + + return records def fill_record_sfa_info(self, record): sfa_info = {} @@ -356,13 +428,18 @@ class SfaAPI(BaseAPI): record.update(sfa_info) - def fill_record_info(self, record): + def fill_record_info(self, records): """ Given a SFA record, fill in the PLC specific and SFA specific fields in the record. """ - self.fill_record_pl_info(record) - self.fill_record_sfa_info(record) + if not isinstance(records, list): + records = [records] + + self.fill_record_pl_info(records) + for record in records: + self.fill_record_sfa_info(record) + #self.fill_record_sfa_info(records) def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc): # get a list of the HRNs tht are members of the old and new records