From: Josh Karlin Date: Thu, 25 Mar 2010 13:30:52 +0000 (+0000) Subject: Merged trunk changes 17245:17471 into branch X-Git-Tag: geni-apiv1-totrunk~92 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=d38a0999bef99c4e7aba056c218716b8eb8df5f4;p=sfa.git Merged trunk changes 17245:17471 into branch --- diff --git a/TODO b/TODO index 9d3f58d4..99db1ad4 100644 --- a/TODO +++ b/TODO @@ -1,21 +1,14 @@ -- resolve xmlbuilder dependency - * add xmlbuilder to source - * update makefile - * update specfile - * test rpm - +- test rpms: build/install + - Stop invalid users * a recently disabled/deleted user may still have a valid cred. Keep a list of valid/invalid users on the aggregate and check callers against this list - Component manager + * only call get_gids() if there are slices with no gids installed * GetTicket - must verify_{site,slice,person,keys} on remote aggregate - * REdeem ticket - RedeemTicket/AdminTicket not working. Why? + * Redeem ticket - RedeemTicket/AdminTicket not working. Why? * install the slice and node gid when the slice is created (create NM plugin to execute sfa_component_setup.py ?) - -- sfa.util.api - * preload registries/aggregates into the api object (i.e. api.registries = Registries()) - - Protogeni * agree on standard set of functon calls * agree on standard set of privs @@ -30,10 +23,14 @@ * api.update_membership() shoudl behave more like resolve when looking up records (attempt to resolve records at federated registeries) instead of only looking in the local registry * support generic registry records (dont depend on postgres!) -- Aggregate -* sfa.plc.slices.verify_site() should check if site['max_slices'] needs to be updated -* sfa.plc.slices.verify_slice() should check if slice['expires'] needs to be updated - +- Auth Service + * develop a simple service where users auth using username/passord and + receive their cred + * service manages users key/cert,creds + +- GUI + * requires user's cred (depends on Auth Service above) + - SM call routing * sfi -a option should send request to sm with an extra argument to specify which am to contact instead of connecting directly to the am @@ -43,7 +40,4 @@ * should sfa have native initscript support or should we piggyback off of myplc? * should this be in the rspec -- error messages -* error messages should be easier to understand -* (failing to connect to plcapi shoudl return a helpful message, not a generic internal server error) diff --git a/sfa.spec b/sfa.spec index 311e4a53..b5d9c42c 100644 --- a/sfa.spec +++ b/sfa.spec @@ -40,8 +40,11 @@ Requires: python-lxml # it'll be installed by "devel.pkgs". we have the epel repository so # python-uuid will be provided. but we can test for the python # version. -%define has_py24 %( python -c "import sys;sys.exit(sys.version_info[0:2] == (2,4))" 2> /dev/null; echo $? ) -%if %has_py24 +# %define has_py24 %( python -c "import sys;sys.exit(sys.version_info[0:2] == (2,4))" 2> /dev/null; echo $? ) +# %if %has_py24 +# +# this also didn't work very well. I'll just check for distroname - baris +%if %{distroname} == "centos5" Requires: python-uuid %endif diff --git a/sfa/managers/aggregate_manager_eucalyptus.py b/sfa/managers/aggregate_manager_eucalyptus.py index e1d93033..9b970380 100644 --- a/sfa/managers/aggregate_manager_eucalyptus.py +++ b/sfa/managers/aggregate_manager_eucalyptus.py @@ -365,12 +365,21 @@ def get_rspec(api, xrn, origin_hrn): # Get the instances that belong to the given slice from sqlite3 # XXX use getOne() in production because the slice's hrn is supposed # to be unique. For testing, uniqueness is turned off in the db. - theSlice = list(Slice.select(Slice.q.slice_hrn == hrn))[-1] + # If the slice isn't found in the database, create a record for the + # slice. + matchedSlices = list(Slice.select(Slice.q.slice_hrn == hrn)) + if matchedSlices: + theSlice = matchedSlices[-1] + else: + theSlice = Slice(slice_hrn = hrn) for instance in theSlice.instances: instanceId.append(instance.instance_id) # Get the information about those instances using their ids. - reservations = conn.get_all_instances(instanceId) + if len(instanceId) > 0: + reservations = conn.get_all_instances(instanceId) + else: + reservations = [] for reservation in reservations: for instance in reservation.instances: instances.append(instance) diff --git a/sfa/plc/api.py b/sfa/plc/api.py index cd6b2bde..52f72ffa 100644 --- a/sfa/plc/api.py +++ b/sfa/plc/api.py @@ -69,13 +69,22 @@ class SfaAPI(BaseAPI): self.plauth = {'Username': self.config.SFA_PLC_USER, 'AuthMethod': 'password', 'AuthString': self.config.SFA_PLC_PASSWORD} - # connect via xmlrpc - self.plshell_type = 'xmlrpc' - url = self.config.SFA_PLC_URL - shell = xmlrpclib.Server(url, verbose = 0, allow_none = True) - shell.AuthCheck(self.plauth) - return shell - + + try: + sys.path.append(os.path.dirname(os.path.realpath("/usr/bin/plcsh"))) + self.plshell_type = 'direct' + import PLC.Shell + shell = PLC.Shell.Shell(globals = globals()) + shell.AuthCheck(self.plauth) + return shell + except ImportError: + self.plshell_type = 'xmlrpc' + # connect via xmlrpc + url = self.config.SFA_PLC_URL + shell = xmlrpclib.Server(url, verbose = 0, allow_none = True) + shell.AuthCheck(self.plauth) + return shell + def getPLCShellVersion(self): # We need to figure out what version of PLCAPI we are talking to. # Some calls we need to make later will be different depending on diff --git a/sfa/plc/sfa-nuke-plc.py b/sfa/plc/sfa-nuke-plc.py index 16c12dcd..2b1b41f5 100755 --- a/sfa/plc/sfa-nuke-plc.py +++ b/sfa/plc/sfa-nuke-plc.py @@ -13,7 +13,6 @@ import sys from sfa.trust.hierarchy import * from sfa.util.record import * from sfa.util.table import SfaTable -from sfa.util.config import Config def process_options(): @@ -26,7 +25,8 @@ def main(): process_options() print "Purging SFA records from database" - SfaTable.sfa_records_purge(Config().get_plc_dbinfo()) + table = SfaTable() + table.sfa_records_purge() if __name__ == "__main__": main() diff --git a/sfa/trust/auth.py b/sfa/trust/auth.py index 66887670..3bcc0971 100644 --- a/sfa/trust/auth.py +++ b/sfa/trust/auth.py @@ -80,17 +80,8 @@ class Auth: def verifyPeerCert(self, cert, gid): # make sure the client_gid matches client's certificate - if not cert: - peer_cert = self.peer_cert - else: - peer_cert = cert - - if not gid: - peer_gid = self.client_gid - else: - peer_gid = gid - if not peer_cert.is_pubkey(peer_gid.get_pubkey()): - raise ConnectionKeyGIDMismatch(peer_gid.get_subject()) + if not cert.is_pubkey(gid.get_pubkey()): + raise ConnectionKeyGIDMismatch(gid.get_subject()+":"+cert.get_subject()) def verifyGidRequestHash(self, gid, hash, arglist): key = gid.get_pubkey() diff --git a/sfa/util/PostgreSQL.py b/sfa/util/PostgreSQL.py index 5b349016..984e5149 100644 --- a/sfa/util/PostgreSQL.py +++ b/sfa/util/PostgreSQL.py @@ -44,6 +44,13 @@ if not psycopg2: pgdb.pgdbTypeCache.typecast = unicast(pgdb.pgdbTypeCache.typecast) +def handle_exception(f): + def wrapper(*args, **kwds): + try: return f(*args, **kwds) + except Exception, fault: + raise SfaDBError(str(fault)) + return wrapper + class PostgreSQL: def __init__(self, config): self.config = config @@ -51,6 +58,7 @@ class PostgreSQL: # self.debug = True self.connection = None + @handle_exception def cursor(self): if self.connection is None: # (Re)initialize database connection @@ -132,13 +140,12 @@ class PostgreSQL: return self.rowcount def next_id(self, table_name, primary_key): - sequence = "%(table_name)s_%(primary_key)s_seq" % locals() - sql = "SELECT nextval('%(sequence)s')" % locals() - rows = self.selectall(sql, hashref = False) - if rows: - return rows[0][0] - - return None + sequence = "%(table_name)s_%(primary_key)s_seq" % locals() + sql = "SELECT nextval('%(sequence)s')" % locals() + rows = self.selectall(sql, hashref = False) + if rows: + return rows[0][0] + return None def last_insert_id(self, table_name, primary_key): if isinstance(self.lastrowid, int): @@ -202,7 +209,7 @@ class PostgreSQL: print >> log, query print >> log, "Params:" print >> log, pformat(params) - raise SfaDBError("Please contact support") + raise SfaDBError("Please contact support: %s" % str(e)) return cursor diff --git a/sfa/util/componentserver.py b/sfa/util/componentserver.py index 933ecfef..542b4be0 100644 --- a/sfa/util/componentserver.py +++ b/sfa/util/componentserver.py @@ -13,20 +13,16 @@ import sys import traceback import threading import socket, os - import SocketServer import BaseHTTPServer import SimpleHTTPServer import SimpleXMLRPCServer - from OpenSSL import SSL - from sfa.trust.certificate import Keypair, Certificate from sfa.trust.credential import * - from sfa.util.faults import * from sfa.plc.api import ComponentAPI -from sfa.util.server import verify_callback, SecureXMLRPCServer +from sfa.util.server import verify_callback, ThreadedServer from sfa.util.debug import log @@ -49,7 +45,9 @@ class SecureXMLRpcRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): It was copied out from SimpleXMLRPCServer.py and modified to shutdown the socket cleanly. """ try: - self.api = ComponentAPI(peer_cert = self.server.peer_cert, + peer_cert = Certificate() + peer_cert.load_from_pyopenssl_x509(self.connection.get_peer_certificate()) + self.api = ComponentAPI(peer_cert = peer_cert, interface = self.server.interface, key_file = self.server.key_file, cert_file = self.server.cert_file) @@ -75,6 +73,7 @@ class SecureXMLRpcRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): # internal error, report as HTTP server error self.send_response(500) self.end_headers() + traceback.print_exc() else: # got a valid XML RPC response self.send_response(200) @@ -109,7 +108,7 @@ class ComponentServer(threading.Thread): threading.Thread.__init__(self) self.key = Keypair(filename = key_file) self.cert = Certificate(filename = cert_file) - self.server = SecureXMLRPCServer((ip, port), SecureXMLRpcRequestHandler, key_file, cert_file) + self.server = ThreadedServer((ip, port), SecureXMLRpcRequestHandler, key_file, cert_file) self.trusted_cert_list = None self.register_functions() diff --git a/sfa/util/server.py b/sfa/util/server.py index fece14ff..197297d3 100644 --- a/sfa/util/server.py +++ b/sfa/util/server.py @@ -41,12 +41,6 @@ def verify_callback(conn, x509, err, depth, preverify): #print " depth > 0 in verify_callback" return 0 - # create a Certificate object and load it from the client's x509 - ctx = conn.get_context() - server = ctx.get_app_data() - server.peer_cert = Certificate() - server.peer_cert.load_from_pyopenssl_x509(x509) - # the certificate verification done by openssl checks a number of things # that we aren't interested in, so we look out for those error messages # and ignore them @@ -99,7 +93,9 @@ class SecureXMLRpcRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): It was copied out from SimpleXMLRPCServer.py and modified to shutdown the socket cleanly. """ try: - self.api = SfaAPI(peer_cert = self.server.peer_cert, + peer_cert = Certificate() + peer_cert.load_from_pyopenssl_x509(self.connection.get_peer_certificate()) + self.api = SfaAPI(peer_cert = peer_cert, interface = self.server.interface, key_file = self.server.key_file, cert_file = self.server.cert_file) @@ -121,6 +117,7 @@ class SecureXMLRpcRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): # internal error, report as HTTP server error self.send_response(500) self.end_headers() + traceback.print_exc() else: # got a valid XML RPC response self.send_response(200) @@ -194,7 +191,7 @@ class ThreadPoolMixIn(SocketServer.ThreadingMixIn): Handle one request at a time until doomsday. """ # set up the threadpool - self.requests = Queue(self.numThreads) + self.requests = Queue() for x in range(self.numThreads): t = threading.Thread(target = self.process_request_thread) diff --git a/sfa/util/table.py b/sfa/util/table.py index c1bfdfb1..7eea14a7 100644 --- a/sfa/util/table.py +++ b/sfa/util/table.py @@ -7,7 +7,6 @@ import report import pgdb -from pg import DB, ProgrammingError from sfa.util.PostgreSQL import * from sfa.trust.gid import * @@ -26,9 +25,6 @@ class SfaTable(list): self.tablename = SfaTable.SFA_TABLE_PREFIX self.config = Config() self.db = PostgreSQL(self.config) - # establish a connection to the pgsql server - cninfo = self.config.get_plc_dbinfo() - self.cnx = DB(cninfo['dbname'], cninfo['address'], port=cninfo['port'], user=cninfo['user'], passwd=cninfo['password']) if record_filter: records = self.find(record_filter) @@ -36,13 +32,12 @@ class SfaTable(list): self.append(record) def exists(self): - tableList = self.cnx.get_tables() - if 'public.' + self.tablename in tableList: - return True - if 'public."' + self.tablename + '"' in tableList: + sql = "SELECT * from pg_tables" + tables = self.db.selectall(sql) + tables = filter(lambda row: row['tablename'].startswith(self.SFA_TABLE_PREFIX), tables) + if tables: return True return False - def db_fields(self, obj=None): db_fields = self.db.fields(self.SFA_TABLE_PREFIX) @@ -79,28 +74,28 @@ class SfaTable(list): for field in ['hrn', 'type', 'authority', 'peer_authority', 'pointer']] # IF EXISTS doenst exist in postgres < 8.2 try: - self.cnx.query('DROP TABLE IF EXISTS ' + self.tablename) - except ProgrammingError: + self.db.do('DROP TABLE IF EXISTS ' + self.tablename) + except: try: - self.cnx.query('DROP TABLE ' + self.tablename) - except ProgrammingError: + self.db.do('DROP TABLE' + self.tablename) + except: pass - self.cnx.query(querystr) + self.db.do(querystr) for index in indexes: - self.cnx.query(index) + self.db.do(index) def remove(self, record): query_str = "DELETE FROM %s WHERE record_id = %s" % \ (self.tablename, record['record_id']) - self.cnx.query(query_str) + self.db.do(query_str) # if this is a site, remove all records where 'authority' == the # site's hrn if record['type'] == 'site': sql = " DELETE FROM %s WHERE authority = %s" % \ (self.tablename, record['hrn']) - self.cnx.query(sql) + self.db.do(sql) def insert(self, record): db_fields = self.db_fields(record) @@ -132,19 +127,10 @@ class SfaTable(list): self.db.commit() def quote_string(self, value): - return str(self.quote(value)) + return str(self.db.quote(value)) def quote(self, value): - """ - Returns quoted version of the specified value. - """ - - # The pgdb._quote function is good enough for general SQL - # quoting, except for array types. - if isinstance(value, (list, tuple, set)): - return "ARRAY[%s]" % ", ".join(map, self.quote_string, value) - else: - return pgdb._quote(value) + return self.db.quote(value) def find(self, record_filter = None, columns=None): if not columns: @@ -168,7 +154,7 @@ class SfaTable(list): record_filter = Filter(SfaRecord.all_fields, {'record_id':[record_filter]}) sql += " AND (%s) %s" % record_filter.sql("AND") - results = self.cnx.query(sql).dictresult() + results = self.db.selectall(sql) if isinstance(results, dict): results = [results] return results @@ -193,22 +179,12 @@ class SfaTable(list): def drop(self): try: - self.cnx.query('DROP TABLE IF EXISTS ' + self.tablename) - except ProgrammingError: + self.db.do('DROP TABLE IF EXISTS ' + self.tablename) + except: try: - self.cnx.query('DROP TABLE ' + self.tablename) - except ProgrammingError: + self.db.do('DROP TABLE ' + self.tablename) + except: pass - @staticmethod - def sfa_records_purge(cninfo): - - cnx = DB(cninfo['dbname'], cninfo['address'], - port=cninfo['port'], user=cninfo['user'], passwd=cninfo['password']) - tableList = cnx.get_tables() - for table in tableList: - if table.startswith(SfaTable.SFA_TABLE_PREFIX) or \ - table.startswith('public.' + SfaTable.SFA_TABLE_PREFIX) or \ - table.startswith('public."' + SfaTable.SFA_TABLE_PREFIX): - report.trace("dropping table " + table) - cnx.query("DROP TABLE " + table) + def sfa_records_purge(): + self.drop()