From: Tony Mack Date: Mon, 10 May 2010 22:50:28 +0000 (+0000) Subject: creating tag sfa-0.9-11 X-Git-Tag: sfa-0.9-11~6 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=f3d825457ff4881b8c9658967f8ed159a01dfcaa creating tag sfa-0.9-11 --- f3d825457ff4881b8c9658967f8ed159a01dfcaa diff --cc TODO index 7f442ee6,7f442ee6..31aaa6a8 --- a/TODO +++ b/TODO @@@ -1,41 -1,41 +1,49 @@@ --- test rpms: build/install ++- Tag ++* test rpm build/install ++ ++- Trunk ++* test federation ++* test sub authority import and federation ++ ++- Client ++ * update getNodes to use lxml.etree for parsing the rspec - Stop invalid users * a recently disabled/deleted user may still have a valid cred. Keep a list of valid/invalid users on the aggregate and check callers against this list - Component manager ++ * GetGids - make this work for peer slices * GetTicket - must verify_{site,slice,person,keys} on remote aggregate * Redeem ticket - RedeemTicket/AdminTicket not working. Why? * install the slice and node gid when the slice is created (create NM plugin to execute sfa_component_setup.py ?) --- Protogeni --* agree on standard set of functon calls --* agree on standard set of privs --* on permission error, return priv needed to make call --* cache slice resource states (if aggregate goes down, how do we know what -- slices were on it and recreate them? do we make some sort of transaction log) -- - Registry --* sign peer gids --* update call should attempt to push updates to federated peers if -- the peer has a record for an object that is updated locally --* api.update_membership() shoudl behave more like resolve when looking up records (attempt to resolve records at federated registeries) instead of only looking in the local registry * move db tables into db with less overhead (tokyocabinet?) --* make resolve, fill_record_info more fault tolerent. Skip records with failures --- Auth Service ++- GUI/Auth Service * develop a simple service where users auth using username/passord and receive their cred * service manages users key/cert,creds -- --- GUI -- * requires user's cred (depends on Auth Service above) ++ * gui requires user's cred (depends on Auth Service above) - SM call routing * sfi -a option should send request to sm with an extra argument to specify which am to contact instead of connecting directly to the am (am may not trust client directly) ++- Protogeni ++* agree on standard set of functon calls ++* agree on standard set of privs ++* on permission error, return priv needed to make call ++* cache slice resource states (if aggregate goes down, how do we know what ++ slices were on it and recreate them? do we make some sort of transaction log) ++ ++ ++Questions ++========= ++- SM/Aggregate ++* should the rspec contain only the resources a slice is using or all resources availa and mark what the slice is using. ++ - Initscripts on sfa / geniwrapper * should sfa have native initscript support or should we piggyback off of myplc? * should this be in the rspec diff --cc setup.py index 77e64918,77e64918..932ef0a7 --- a/setup.py +++ b/setup.py @@@ -14,6 -14,6 +14,7 @@@ bins = 'config/gen-sfa-cm-config.py', 'sfa/plc/sfa-import-plc.py', 'sfa/plc/sfa-nuke-plc.py', ++ 'sfa/server/sfa-ca.py', 'sfa/server/sfa-server.py', 'sfa/server/sfa-clean-peer-records.py', 'sfa/server/sfa_component_setup.py', diff --cc sfa.spec index 46af058a,2e32acb8..00cf3ec3 --- a/sfa.spec +++ b/sfa.spec @@@ -34,6 -34,6 +34,7 @@@ Requires: libxslt-pytho Requires: python-ZSI # xmlbuilder depends on lxml Requires: python-lxml ++Requires: python-setuptools # python 2.5 has uuid module added, for python 2.4 we still need it. # we can't really check for if we can load uuid as a python module, @@@ -154,14 -154,14 +155,17 @@@ chkconfig --add sf %post cm chkconfig --add sfa-cm %changelog --* Thu Apr 08 2010 Tony Mack - sfa-0.9-11 ++* Thu May 11 2010 Tony Mack - sfa-0.9-11 - SfaServer now uses a pool of threads to handle requests concurrently - sfa.util.rspec no longer used to process/manage rspecs (deprecated). This is now handled by sfa.plc.network and is not backwards compatible - - PIs can now get a slice credential for any slice at thier site without having to be a member of the slice + - PIs can now get a slice credential for any slice at their site without having to be a member of the slice - Registry records for federated peers (defined in registries.xml, aggregates.xml) updated when sfa service is started - Interfaces will try to fetch and install gids from peers listed in registries.xml/aggregates.xml if gid is not found in /etc/sfa/trusted_roots dir - - Componet manager does not install gid files if slice already has them - + - Component manager does not install gid files if slice already has them - ++- Server automatically fetches and installs peer certificats (defined in registries/aggregates.xml) when service is restarted. ++- fix credential verification exploit (verify that the trusted signer is a parent of the object it it signed) ++- made it easier for root authorities to sign their sub's certifiacate using the sfa-ca.py (sfa/server/sfa-ca.py) tool ++ * Thu Jan 21 2010 anil vengalil - sfa-0.9-10 - This tag is quite same as the previous one (sfa-0.9-9) except that the vini and max aggregate managers are also updated for urn support. Other features are: - - sfa-config-tty now has the same features like plc-config-tty diff --cc sfa/managers/registry_manager_pl.py index 1af2caf5,1af2caf5..24867383 --- a/sfa/managers/registry_manager_pl.py +++ b/sfa/managers/registry_manager_pl.py @@@ -129,7 -129,7 +129,7 @@@ def list(api, xrn, origin_hrn=None) tree = prefixTree() tree.load(registry_hrns) registry_hrn = tree.best_match(hrn) -- ++ #if there was no match then this record belongs to an unknow registry if not registry_hrn: raise MissingAuthority(xrn) diff --cc sfa/plc/sfa-import-plc.py index 160ce77c,603f2960..0350bbb0 --- a/sfa/plc/sfa-import-plc.py +++ b/sfa/plc/sfa-import-plc.py @@@ -85,23 -85,16 +85,16 @@@ def main() if not table.exists(): table.create() - if not level1_auth or level1_auth in ['']: - level1_auth = None - - if not level1_auth: - sfaImporter.create_top_level_auth_records(root_auth) - import_auth = root_auth - else: - if not AuthHierarchy.auth_exists(level1_auth): - AuthHierarchy.create_auth(level1_auth) - sfaImporter.create_top_level_auth_records(level1_auth) - import_auth = level1_auth - - trace("Import: adding " + import_auth + " to trusted list", logger) - authority = AuthHierarchy.get_auth_info(import_auth) + # create root authority + sfaImporter.create_top_level_auth_records(root_auth) - if not root_auth == interface_hrn ++ if not root_auth == interface_hrn: + sfaImporter.create_top_level_auth_records(interface_hrn) + + trace("Import: adding " + interface_hrn + " to trusted list", logger) + authority = AuthHierarchy.get_auth_info(interface_hrn) TrustedRoots.add_gid(authority.get_gid_object()) - if ".vini" in import_auth and import_auth.endswith('vini'): + if ".vini" in interface_hrn and interface_hrn.endswith('vini'): # create a fake internet2 site first i2site = {'name': 'Internet2', 'abbreviated_name': 'I2', 'login_base': 'internet2', 'site_id': -1} @@@ -210,7 -203,7 +203,9 @@@ for (record_hrn, type) in existing_records.keys(): record = existing_records[(record_hrn, type)] # if this is the interface name dont do anything - if record_hrn == import_auth or record['peer_authority']: - if record_hrn == interface_hrn or record['peer_authority']: ++ if record_hrn == interface_hrn or \ ++ record_hrn == root_auth or \ ++ record['peer_authority']: continue # dont delete vini's internet2 placeholdder record # normally this would be deleted becuase it does not have a plc record diff --cc sfa/server/aggregate.py index aecc3b69,aecc3b69..515b9e97 --- a/sfa/server/aggregate.py +++ b/sfa/server/aggregate.py @@@ -30,21 -30,21 +30,13 @@@ class Aggregates(Interfaces) default_dict = {'aggregates': {'aggregate': [Interfaces.default_fields]}} def __init__(self, api, conf_file = "/etc/sfa/aggregates.xml"): -- Interfaces.__init__(self, api, conf_file, 'ma') -- -- def get_connections(self, interfaces): -- """ -- Get connection details for the trusted peer aggregates from file and -- create an connection to each. -- """ -- connections = Interfaces.get_connections(self, interfaces) -- ++ Interfaces.__init__(self, api, conf_file) # set up a connection to the local registry address = self.api.config.SFA_AGGREGATE_HOST port = self.api.config.SFA_AGGREGATE_PORT url = 'http://%(address)s:%(port)s' % locals() local_aggregate = {'hrn': self.api.hrn, 'addr': address, 'port': port} -- self.interfaces.append(local_aggregate) -- connections[self.api.hrn] = xmlrpcprotocol.get_server(url, self.api.key_file, self.api.cert_file) -- return connections ++ self.interfaces[self.api.hrn] = local_aggregate ++ # get connections ++ self.update(self.get_connections()) diff --cc sfa/server/interface.py index 7172ef97,d6dc5c44..05034795 --- a/sfa/server/interface.py +++ b/sfa/server/interface.py @@@ -3,13 -3,13 +3,13 @@@ ### $URL: https://svn.planet-lab.org/svn/sfa/trunk/sfa/server/interface.py $ # -- from sfa.util.faults import * from sfa.util.storage import * from sfa.util.namespace import * from sfa.trust.gid import GID from sfa.util.table import SfaTable from sfa.util.record import SfaRecord ++import traceback import sfa.util.xmlrpcprotocol as xmlrpcprotocol import sfa.util.soapprotocol as soapprotocol @@@ -46,10 -46,10 +46,9 @@@ class Interfaces(dict) # defined by the class default_dict = {} -- # allowed types -- types = ['sa', 'ma'] ++ types = ['authority'] -- def __init__(self, api, conf_file, type): ++ def __init__(self, api, conf_file, type='authority'): if type not in self.types: raise SfaInfaildArgument('Invalid type %s: must be in %s' % (type, self.types)) dict.__init__(self, {}) @@@ -58,11 -58,11 +57,13 @@@ # load config file self.interface_info = XmlStorage(conf_file, self.default_dict) self.interface_info.load() -- self.interfaces = self.interface_info.values()[0].values()[0] -- if not isinstance(self.interfaces, list): -- self.interfaces = [self.interfaces] -- # get connections -- self.update(self.get_connections(self.interfaces)) ++ interfaces = self.interface_info.values()[0].values()[0] ++ if not isinstance(interfaces, list): ++ interfaces = [self.interfaces] ++ self.interfaces = {} ++ for interface in interfaces: ++ self.interfaces[interface['hrn']] = interface ++ def sync_interfaces(self): """ @@@ -75,19 -75,19 +76,19 @@@ # are any missing gids, request a new one from the peer registry. gids_current = self.api.auth.trusted_cert_list hrns_current = [gid.get_hrn() for gid in gids_current] -- hrns_expected = [interface['hrn'] for interface in self.interfaces] ++ hrns_expected = self.interfaces.keys() new_hrns = set(hrns_expected).difference(hrns_current) -- self.get_peer_gids(new_hrns) -- ++ gids = self.get_peer_gids(new_hrns) # update the local db records for these registries -- self.update_db_records(self.type) ++ self.update_db_records(self.type, gids) def get_peer_gids(self, new_hrns): """ Install trusted gids from the specified interfaces. """ ++ peer_gids = [] if not new_hrns: -- return ++ return peer_gids trusted_certs_dir = self.api.config.get_trustedroots_dir() for new_hrn in new_hrns: # the gid for this interface should already be installed @@@ -95,42 -95,42 +96,49 @@@ continue try: # get gid from the registry -- interface = self.get_connections(self.interfaces[new_hrn])[new_hrn] ++ interface_info = self.interfaces[new_hrn] ++ interface = self[new_hrn] trusted_gids = interface.get_trusted_certs() -- # default message -- message = "interface: %s\tunable to install trusted gid for %s" % \ -- (self.api.interface, new_hrn) if trusted_gids: # the gid we want shoudl be the first one in the list, # but lets make sure for trusted_gid in trusted_gids: ++ # default message ++ message = "interface: %s\t" % (self.api.interface) ++ message += "unable to install trusted gid for %s" % \ ++ (new_hrn) gid = GID(string=trusted_gids[0]) ++ peer_gids.append(gid) if gid.get_hrn() == new_hrn: gid_filename = os.path.join(trusted_certs_dir, '%s.gid' % new_hrn) gid.save_to_file(gid_filename, save_parents=True) message = "interface: %s\tinstalled trusted gid for %s" % \ (self.api.interface, new_hrn) -- # log the message -- self.api.logger.info(message) ++ # log the message ++ self.api.logger.info(message) except: message = "interface: %s\tunable to install trusted gid for %s" % \ (self.api.interface, new_hrn) self.api.logger.info(message) ++ traceback.print_exc() # reload the trusted certs list self.api.auth.load_trusted_certs() ++ return peer_gids -- def update_db_records(self, type): ++ def update_db_records(self, type, gids): """ Make sure there is a record in the local db for allowed registries defined in the config file (registries.xml). Removes old records from the db. """ ++ if not gids: ++ return # get hrns we expect to find # ignore records for local interfaces ignore_interfaces = [self.api.config.SFA_INTERFACE_HRN] -- hrns_expected = [interface['hrn'] for interface in self.interfaces \ -- if interface['hrn'] not in ignore_interfaces] ++ hrns_expected = [gid.get_hrn() for gid in gids \ ++ if gid.get_hrn() not in ignore_interfaces] # get hrns that actually exist in the db table = SfaTable() @@@ -143,32 -143,32 +151,31 @@@ table.remove(record) # add new records -- for hrn in hrns_expected: ++ for gid in gids: ++ hrn = gid.get_hrn() if hrn not in hrns_found: record = { 'hrn': hrn, 'type': type, 'pointer': -1, 'authority': get_authority(hrn), ++ 'gid': gid.save_to_string(save_parents=True), } record = SfaRecord(dict=record) table.insert(record) -- -- def get_connections(self, interfaces): ++ def get_connections(self): """ read connection details for the trusted peer registries from file return a dictionary of connections keyed on interface hrn. """ connections = {} required_fields = self.default_fields.keys() -- if not isinstance(interfaces, list): -- interfaces = [interfaces] -- for interface in interfaces: ++ for interface in self.interfaces.values(): # make sure the required fields are present and not null - for key in required_fields: - if not interface.get(key): - continue + if not all([interface.get(key) for key in required_fields]): + continue + hrn, address, port = interface['hrn'], interface['addr'], interface['port'] url = 'http://%(address)s:%(port)s' % locals() # check which client we should use diff --cc sfa/server/registry.py index 5658f37d,5658f37d..0b92f76e --- a/sfa/server/registry.py +++ b/sfa/server/registry.py @@@ -37,19 -37,19 +37,12 @@@ class Registries(Interfaces) default_dict = {'registries': {'registry': [Interfaces.default_fields]}} def __init__(self, api, conf_file = "/etc/sfa/registries.xml"): -- Interfaces.__init__(self, api, conf_file, 'sa') -- -- def get_connections(self, interfaces): -- """ -- read connection details for the trusted peer registries from file return -- a dictionary of connections keyed on interface hrn. -- """ -- connections = Interfaces.get_connections(self, interfaces) -- -- # set up a connection to the local registry ++ Interfaces.__init__(self, api, conf_file) address = self.api.config.SFA_REGISTRY_HOST port = self.api.config.SFA_REGISTRY_PORT url = 'http://%(address)s:%(port)s' % locals() local_registry = {'hrn': self.api.hrn, 'addr': address, 'port': port} -- connections[self.api.hrn] = xmlrpcprotocol.get_server(url, self.api.key_file, self.api.cert_file) -- return connections ++ self.interfaces[self.api.hrn] = local_registry ++ ++ # get connections ++ self.update(self.get_connections()) diff --cc sfa/server/sfa-server.py index 58bfdb9c,58bfdb9c..5e0905f0 --- a/sfa/server/sfa-server.py +++ b/sfa/server/sfa-server.py @@@ -150,12 -150,12 +150,12 @@@ def init_server(options, config) manager.init_server() --def sync_interfaces(): ++def sync_interfaces(server_key_file, server_cert_file): """ Attempt to install missing trusted gids and db records for our federated interfaces """ -- api = SfaAPI() ++ api = SfaAPI(key_file = server_key_file, cert_file = server_cert_file) registries = Registries(api) aggregates = Aggregates(api) registries.sync_interfaces() @@@ -185,7 -185,7 +185,6 @@@ def main() help="Run as daemon.", default=False) (options, args) = parser.parse_args() -- if (options.daemon): daemon() config = Config() hierarchy = Hierarchy() @@@ -194,8 -194,8 +193,9 @@@ init_server_key(server_key_file, server_cert_file, config, hierarchy) init_server(options, config) -- sync_interfaces() ++ sync_interfaces(server_key_file, server_cert_file) ++ if (options.daemon): daemon() # start registry server if (options.registry): from sfa.server.registry import Registry diff --cc sfa/util/table.py index 779562f8,f6090a03..83646273 --- a/sfa/util/table.py +++ b/sfa/util/table.py @@@ -84,20 -84,21 +84,22 @@@ class SfaTable(list) self.db.do(querystr) for index in indexes: self.db.do(index) + self.db.commit() - + def remove(self, record): -- query_str = "DELETE FROM %s WHERE record_id = %s" % \ -- (self.tablename, record['record_id']) -- self.db.do(query_str) ++ params = {'record_id': record['record_id']} ++ template = "DELETE FROM %s " % self.tablename ++ sql = template + "WHERE record_id = %(record_id)s" ++ self.db.do(sql, params) # if this is a site, remove all records where 'authority' == the # site's hrn - if record['type'] == 'site': - sql = " DELETE FROM %s WHERE authority = %s" % \ - (self.tablename, record['hrn']) - self.db.do(sql) - self.db.commit() + if record['type'] == 'authority': - sql = " DELETE FROM %s WHERE authority = %s" % \ - (self.tablename, record['hrn']) - self.db.do(sql) ++ params = {'authority': record['hrn']} ++ sql = template + "WHERE authority = %(authority)s" ++ self.db.do(sql, params) + self.db.commit() def insert(self, record): db_fields = self.db_fields(record) diff --cc xmlbuilder-0.9/xmlbuilder.egg-info/PKG-INFO index d931c15a,d931c15a..bb65a9de --- a/xmlbuilder-0.9/xmlbuilder.egg-info/PKG-INFO +++ b/xmlbuilder-0.9/xmlbuilder.egg-info/PKG-INFO @@@ -1,80 -1,80 +1,80 @@@ --Metadata-Version: 1.0 --Name: xmlbuilder --Version: 0.9 --Summary: Pythonic way to create xml files --Home-page: http://pypi.python.org/pypi/xmlbuilder --Author: koder --Author-email: koder_dot_mail@gmail_dot_com --License: MIT --Download-URL: http://pypi.python.org/pypi/xmlbuilder --Description: Example of usage: -- ----------------- -- -- -- from __future__ import with_statement -- from xmlbuilder import XMLBuilder -- x = XMLBuilder(format=True) -- with x.root(a = 1): -- with x.data: -- [x << ('node',{'val':i}) for i in range(10)] -- -- print str(x) -- -- will print -- -- <root a="1"> -- <data> -- <node val="0" /> -- <node val="1" /> -- <node val="2" /> -- <node val="3" /> -- <node val="4" /> -- <node val="5" /> -- <node val="6" /> -- <node val="7" /> -- <node val="8" /> -- <node val="9" /> -- </data> -- </root> -- -- Mercurial repo:http://hg.assembla.com/MyPackages/ -- -- Documentations -- -------------- -- `XMLBuilder` is simple library build on top of `ElementTree.TreeBuilder` to -- simplify xml files creation as much as possible. Althow it can produce -- structured result with identated child tags. `XMLBuilder` use python `with` -- statement to define xml tag levels and `<<` operator for simple cases - -- text and tag without childs. -- -- First we need to create xmlbuilder -- -- from xmlbuilder import XMLBuilder -- # params - encoding = 'utf8', -- # builder = None, - ElementTree.TreeBuilder -- # tab_level = None, - current tab l;evel - for formatted output only -- # format = False, - create formatted output -- # tab_step = " " * 4 - indentation step -- xml = XMLBuilder() -- -- -- Use `with` statement to make document structure -- #create and open tag 'root_tag' with text 'text' and attributes -- with xml.root_tag(text,attr1=val1,attr2=val2): -- #create and open tag 'sub_tag' -- with xml.sub_tag(text,attr3=val3): -- #create tag which are not valid python identificator -- with xml('one-more-sub-tag',attr7=val37): -- xml << "Some textual data" -- #here tag 'one-more-sub-tag' are closed -- #Tags without children can be created using `<<` operator -- for val in range(15): -- xml << ('message',"python rocks!"[:i]) -- #create 15 child tag like <message> python r</message> -- #all tags closed -- node = ~x # get etree.ElementTree object -- xml_data = str(x) -- unicode_xml_data = unicode(x) -- --Keywords: xml --Platform: UNKNOWN ++Metadata-Version: 1.0 ++Name: xmlbuilder ++Version: 0.9 ++Summary: Pythonic way to create xml files ++Home-page: http://pypi.python.org/pypi/xmlbuilder ++Author: koder ++Author-email: koder_dot_mail@gmail_dot_com ++License: MIT ++Download-URL: http://pypi.python.org/pypi/xmlbuilder ++Description: Example of usage: ++ ----------------- ++ ++ ++ from __future__ import with_statement ++ from xmlbuilder import XMLBuilder ++ x = XMLBuilder(format=True) ++ with x.root(a = 1): ++ with x.data: ++ [x << ('node',{'val':i}) for i in range(10)] ++ ++ print str(x) ++ ++ will print ++ ++ <root a="1"> ++ <data> ++ <node val="0" /> ++ <node val="1" /> ++ <node val="2" /> ++ <node val="3" /> ++ <node val="4" /> ++ <node val="5" /> ++ <node val="6" /> ++ <node val="7" /> ++ <node val="8" /> ++ <node val="9" /> ++ </data> ++ </root> ++ ++ Mercurial repo:http://hg.assembla.com/MyPackages/ ++ ++ Documentations ++ -------------- ++ `XMLBuilder` is simple library build on top of `ElementTree.TreeBuilder` to ++ simplify xml files creation as much as possible. Althow it can produce ++ structured result with identated child tags. `XMLBuilder` use python `with` ++ statement to define xml tag levels and `<<` operator for simple cases - ++ text and tag without childs. ++ ++ First we need to create xmlbuilder ++ ++ from xmlbuilder import XMLBuilder ++ # params - encoding = 'utf8', ++ # builder = None, - ElementTree.TreeBuilder ++ # tab_level = None, - current tab l;evel - for formatted output only ++ # format = False, - create formatted output ++ # tab_step = " " * 4 - indentation step ++ xml = XMLBuilder() ++ ++ ++ Use `with` statement to make document structure ++ #create and open tag 'root_tag' with text 'text' and attributes ++ with xml.root_tag(text,attr1=val1,attr2=val2): ++ #create and open tag 'sub_tag' ++ with xml.sub_tag(text,attr3=val3): ++ #create tag which are not valid python identificator ++ with xml('one-more-sub-tag',attr7=val37): ++ xml << "Some textual data" ++ #here tag 'one-more-sub-tag' are closed ++ #Tags without children can be created using `<<` operator ++ for val in range(15): ++ xml << ('message',"python rocks!"[:i]) ++ #create 15 child tag like <message> python r</message> ++ #all tags closed ++ node = ~x # get etree.ElementTree object ++ xml_data = str(x) ++ unicode_xml_data = unicode(x) ++ ++Keywords: xml ++Platform: UNKNOWN diff --cc xmlbuilder-0.9/xmlbuilder.egg-info/SOURCES.txt index 57272a81,57272a81..4cc27de9 --- a/xmlbuilder-0.9/xmlbuilder.egg-info/SOURCES.txt +++ b/xmlbuilder-0.9/xmlbuilder.egg-info/SOURCES.txt @@@ -1,6 -1,6 +1,8 @@@ LICENSE MANIFEST.in ++PKG-INFO README.txt ++setup.cfg setup.py xmlbuilder/__init__.py xmlbuilder.egg-info/PKG-INFO