From: Ciro Scognamiglio Date: Tue, 17 Mar 2015 17:07:24 +0000 (+0100) Subject: manifold backend, proxy and api cleanup X-Git-Tag: myslice-1.3~56^2~1 X-Git-Url: http://git.onelab.eu/?p=unfold.git;a=commitdiff_plain;h=8203b2f617faf5f0f36184952677fa1077172093 manifold backend, proxy and api cleanup --- diff --git a/auth/manifoldbackend.py b/auth/manifoldbackend.py index afb87821..c7ac7faa 100644 --- a/auth/manifoldbackend.py +++ b/auth/manifoldbackend.py @@ -5,6 +5,8 @@ from django.contrib.auth.models import User from manifoldapi.manifoldapi import ManifoldAPI, ManifoldException, ManifoldResult from manifold.core.query import Query +from myslice.settings import config, logger, DEBUG + # Name my backend 'ManifoldBackend' class ManifoldBackend: @@ -13,6 +15,8 @@ class ManifoldBackend: def authenticate(self, token=None): if not token: return None + + person = {} try: username = token['username'] @@ -22,15 +26,13 @@ class ManifoldBackend: auth = {'AuthMethod': 'password', 'Username': username, 'AuthString': password} api = ManifoldAPI(auth) sessions_result = api.forward(Query.create('local:session').to_dict()) - print "result" sessions = sessions_result.ok_value() - print "ok" if not sessions: - print "GetSession failed", sessions_result.error() + logger.error("GetSession failed", sessions_result.error()) return - print "first", sessions session = sessions[0] - + logger.debug("SESSION : %s" % session) + # Change to session authentication api.auth = {'AuthMethod': 'session', 'session': session['session']} self.api = api @@ -40,19 +42,19 @@ class ManifoldBackend: persons_result = api.forward(Query.get('local:user').to_dict()) persons = persons_result.ok_value() if not persons: - print "GetPersons failed",persons_result.error() + logger.error("GetPersons failed",persons_result.error()) return person = persons[0] - print "PERSON=", person + logger.debug("PERSON : %s" % person) + #logger.info("%s %s <%s> logged in" % (person['config']['first_name'], person['config']['last_name'], person['config']['email'])) request.session['manifold'] = {'auth': api.auth, 'person': person, 'expires': session['expires']} except ManifoldException, e: - print "ManifoldBackend.authenticate caught ManifoldException, returning corresponding ManifoldResult" - return e.manifold_result + logger.error("Manifold Auth Backend: %s" % e.manifold_result) except Exception, e: - print "E: manifoldbackend", e - import traceback - traceback.print_exc() + logger.error("Manifold Auth Backend: %s" % e) + #import traceback + #traceback.print_exc() return None try: diff --git a/manifoldapi/manifoldapi.py b/manifoldapi/manifoldapi.py index 796d0a52..9a999910 100644 --- a/manifoldapi/manifoldapi.py +++ b/manifoldapi/manifoldapi.py @@ -1,38 +1,18 @@ # Manifold API Python interface import copy, xmlrpclib, ssl - -from myslice.configengine import ConfigEngine +from myslice.settings import config, logger, DEBUG from django.contrib import messages from django.shortcuts import redirect from manifoldresult import ManifoldResult, ManifoldCode, ManifoldException from manifold.core.result_value import ResultValue -debug=False -debug=True debug_deep=False #debug_deep=True -########## ugly stuff for hopefully nicer debug messages -def mytruncate (obj, l): - # we will add '..' - l1=l-2 - repr="%s"%obj - return (repr[:l1]+'..') if len(repr)>l1 else repr - -from time import time, gmtime, strftime -from math import trunc -def mytime (start=None): - gm=gmtime() - t=time() - msg=strftime("%H:%M:%S-", gmtime())+"%03d"%((t-trunc(t))*1000) - if start is not None: msg += " (%03fs)"%(t-start) - return t,msg -########## - class ManifoldAPI: - def __init__ (self, auth=None, cainfo=None): + def __init__(self, auth=None, cainfo=None): self.auth = auth self.cainfo = cainfo @@ -40,7 +20,7 @@ class ManifoldAPI: self.trace = [] self.calls = {} self.multicall = False - self.url = ConfigEngine().manifold_url() + self.url = config.manifold_url() # Manifold uses a self signed certificate # https://www.python.org/dev/peps/pep-0476/ @@ -49,37 +29,6 @@ class ManifoldAPI: else : self.server = xmlrpclib.Server(self.url, verbose=False, allow_none=True) - def __repr__ (self): return "ManifoldAPI[%s]"%self.url - - def _print_value (self, value): - print "+++",'value', - if isinstance (value,list): print "[%d]"%len(value), - elif isinstance (value,dict): print "{%d}"%len(value), - print mytruncate (value,80) - - # a one-liner to give a hint of what the return value looks like - def _print_result (self, result): - if not result: print "[no/empty result]" - elif isinstance (result,str): print "result is '%s'"%result - elif isinstance (result,list): print "result is a %d-elts list"%len(result) - elif isinstance (result,dict): - print "result is a dict with %d keys : %s"%(len(result),result.keys()) - for (k,v) in result.iteritems(): - if v is None: continue - if k=='value': self._print_value(v) - else: print '+++',k,':',mytruncate (v,30) - else: print "[dont know how to display result] %s"%result - - # how to display a call - def _repr_query (self,methodName, query): - try: action=query['action'] - except: action="???" - try: subject=query['object'] - except: subject="???" - # most of the time, we run 'forward' - if methodName=='forward': return "forward(%s(%s))"%(action,subject) - else: return "%s(%s)"%(action,subject) - # xxx temporary code for scaffolding a ManifolResult on top of an API that does not expose error info # as of march 2013 we work with an API that essentially either returns the value, or raises # an xmlrpclib.Fault exception with always the same 8002 code @@ -87,55 +36,64 @@ class ManifoldAPI: # (looks like sessions are rather short-lived), for now the choice is to map these errors on # a SESSION_EXPIRED code def __getattr__(self, methodName): + def func(*args, **kwds): - # shorthand - def repr(): return self._repr_query (methodName, args[0]) + import time + + start = time.time() try: - if debug: - start,msg = mytime() - print "====>",msg,"ManifoldAPI.%s"%repr(),"url",self.url - # No password in the logs - logAuth = copy.copy(self.auth) - for obfuscate in ['Authring','session']: - if obfuscate in logAuth: logAuth[obfuscate]="XXX" - if debug_deep: print "=> auth",logAuth - if debug_deep: print "=> args",args,"kwds",kwds - annotations = { - 'authentication': self.auth - } - args += (annotations,) - result=getattr(self.server, methodName)(*args, **kwds) - print "%s%r" %(methodName, args) - if debug: - print '<= result=', - self._print_result(result) - end,msg = mytime(start) - print "<====",msg,"backend call %s returned"%(repr()) + #logger.debug("MANIFOLD %s( %s( %s ) ) to %s" % (methodName, args[0]['action'], args[0]['object'], self.url)) + + if ('Username' in self.auth) : + username = self.auth['Username'] + else : + username = "-" + + args += ({ 'authentication': self.auth },) + + result = getattr(self.server, methodName)(*args, **kwds) + + logger.debug("MANIFOLD %s( %s( %s ) ) as %s to %s executed in %s seconds -> %s" % + (methodName, + args[0]['action'] or '', + args[0]['object'] or '', + username, + self.url, + (time.time() - start), + args)) return ResultValue(**result) - except Exception,error: - print "** MANIFOLD API ERROR **" - if debug: - print "===== xmlrpc catch-all exception:",error + except Exception, error: + if True: + print "===== xmlrpc catch-all exception:", error import traceback traceback.print_exc(limit=3) + if "Connection refused" in error: raise ManifoldException ( ManifoldResult (code=ManifoldCode.SERVER_UNREACHABLE, - output="%s answered %s"%(self.url,error))) + output="%s answered %s" % (self.url,error))) # otherwise - print "<==== ERROR On ManifoldAPI.%s"%repr() - raise ManifoldException ( ManifoldResult (code=ManifoldCode.SERVER_UNREACHABLE, output="%s"%error) ) + logger.error("MANIFOLD %s( %s( %s ) ) as %s to %s executed in %s seconds -> %s" % + (methodName, + args[0]['action'] or '', + args[0]['object'] or '', + username, + self.url, + (time.time() - start), + args)) + logger.error("MANIFOLD %s", error) + raise ManifoldException ( ManifoldResult (code = ManifoldCode.SERVER_UNREACHABLE, output = "%s" % error) ) return func def _execute_query(request, query, manifold_api_session_auth): - manifold_api = ManifoldAPI(auth=manifold_api_session_auth) - print "-"*80 - print query - print query.to_dict() - print "-"*80 + + manifold_api = ManifoldAPI(auth = manifold_api_session_auth) + + logger.debug("MANIFOLD QUERY : %s" % " ".join(str(query).split())) + #logger.debug("MANIFOLD DICT : %s" % query.to_dict()) result = manifold_api.forward(query.to_dict()) if result['code'] == 2: # this is gross; at the very least we need to logout() @@ -148,8 +106,7 @@ def _execute_query(request, query, manifold_api_session_auth): #raise Exception, 'Error running query: %r' % result if result['code'] == 1: - print "WARNING" - print result['description'] + log.warning("MANIFOLD : %s" % result['description']) # XXX Handle errors #Error running query: {'origin': [0, 'XMLRPCAPI'], 'code': 2, 'description': 'No such session: No row was found for one()', 'traceback': 'Traceback (most recent call last):\n File "/usr/local/lib/python2.7/dist-packages/manifold/core/xmlrpc_api.py", line 68, in xmlrpc_forward\n user = Auth(auth).check()\n File "/usr/local/lib/python2.7/dist-packages/manifold/auth/__init__.py", line 245, in check\n return self.auth_method.check()\n File "/usr/local/lib/python2.7/dist-packages/manifold/auth/__init__.py", line 95, in check\n raise AuthenticationFailure, "No such session: %s" % e\nAuthenticationFailure: No such session: No row was found for one()\n', 'type': 2, 'ts': None, 'value': None} @@ -162,10 +119,12 @@ def execute_query(request, query): #raise Exception, "User not authenticated" host = request.get_host() return redirect('/') + manifold_api_session_auth = request.session['manifold']['auth'] + return _execute_query(request, query, manifold_api_session_auth) def execute_admin_query(request, query): - admin_user, admin_password = ConfigEngine().manifold_admin_user_password() + admin_user, admin_password = config.manifold_admin_user_password() admin_auth = {'AuthMethod': 'password', 'Username': admin_user, 'AuthString': admin_password} return _execute_query(request, query, admin_auth) diff --git a/manifoldapi/manifoldproxy.py b/manifoldapi/manifoldproxy.py index 9385bfa3..a6f29c34 100644 --- a/manifoldapi/manifoldproxy.py +++ b/manifoldapi/manifoldproxy.py @@ -11,7 +11,8 @@ from manifold.core.result_value import ResultValue from manifoldapi import ManifoldAPI from manifoldresult import ManifoldException from manifold.util.log import Log -from myslice.configengine import ConfigEngine + +from myslice.settings import config, logger, DEBUG # register activity import activity.slice @@ -30,23 +31,23 @@ debug_empty=False # as well as # static/js/manifold.js def proxy (request,format): - """the view associated with /manifold/proxy/ -with the query passed using POST""" + """the view associated with /manifold/proxy/ with the query passed using POST""" # expecting a POST if request.method != 'POST': - print "manifoldproxy.api: unexpected method %s -- exiting"%request.method - return + logger.error("MANIFOLDPROXY unexpected method %s -- exiting" % request.method) + return HttpResponse ({"ret":0}, mimetype="application/json") # we only support json for now # if needed in the future we should probably cater for # format_in : how is the query encoded in POST # format_out: how to serve the results if format != 'json': - print "manifoldproxy.proxy: unexpected format %s -- exiting"%format + logger.error("MANIFOLDPROXY unexpected format %s -- exiting" % format) return HttpResponse ({"ret":0}, mimetype="application/json") try: # translate incoming POST request into a query object - if debug: print 'manifoldproxy.proxy: request.POST',request.POST + #logger.debug("MANIFOLDPROXY request.POST %s" % request.POST) + manifold_query = Query() #manifold_query = ManifoldQuery() manifold_query.fill_from_POST(request.POST) @@ -55,24 +56,20 @@ with the query passed using POST""" # We allow some requests to use the ADMIN user account if (manifold_query.get_from() == 'local:user' and manifold_query.get_action() == 'create') \ or (manifold_query.get_from() == 'local:platform' and manifold_query.get_action() == 'get'): - admin_user, admin_password = ConfigEngine().manifold_admin_user_password() + admin_user, admin_password = config.manifold_admin_user_password() manifold_api_session_auth = {'AuthMethod': 'password', 'Username': admin_user, 'AuthString': admin_password} else: if 'manifold' in request.session: - print '===> manifoldproxy.proxy: before auth', manifold_query manifold_api_session_auth = request.session['manifold']['auth'] else: - json_answer=json.dumps({'code':0,'value':[]}) - return HttpResponse (json_answer, mimetype="application/json") + return HttpResponse (json.dumps({'code':0,'value':[]}), mimetype="application/json") if debug_empty and manifold_query.action.lower()=='get': - json_answer=json.dumps({'code':0,'value':[]}) - print "By-passing : debug_empty & 'get' request : returning a fake empty list" - return HttpResponse (json_answer, mimetype="application/json") + return HttpResponse (json.dumps({'code':0,'value':[]}), mimetype="application/json") # actually forward manifold_api= ManifoldAPI(auth=manifold_api_session_auth) - if debug: print '===> manifoldproxy.proxy: sending to backend', manifold_query + # for the benefit of the python code, manifoldAPI raises an exception if something is wrong # however in this case we want to propagate the complete manifold result to the js world @@ -83,7 +80,6 @@ with the query passed using POST""" and isinstance(result['description'], (tuple, list, set, frozenset)): result [ 'description' ] = [ ResultValue.to_html (x) for x in result['description'] ] - print "=> MANIFOLD PROXY executing: " + manifold_query.action.lower() # # register activity # @@ -107,7 +103,7 @@ with the query passed using POST""" return HttpResponse (json_answer, mimetype="application/json") except Exception,e: - print "** PROXY ERROR **",e + logger.error("MANIFOLDPROXY %s" % e) import traceback traceback.print_exc() return HttpResponse ({"ret":0}, mimetype="application/json") diff --git a/myslice/components.py b/myslice/components.py index 51cac47f..84e392c3 100644 --- a/myslice/components.py +++ b/myslice/components.py @@ -1,8 +1,7 @@ from django.conf.urls import include, url -from myslice.configengine import ConfigEngine +from myslice.settings import config, logger def list(): - config = ConfigEngine() if config.myslice.components : return config.myslice.components.split(',') else : @@ -15,9 +14,9 @@ def urls(): __import__(component) u.append( url(r'^%s/' % component, include('%s.urls' % component)) ) except Exception, e: - print "-> Cannot load component (%s): %s" % (component, e) + logger.info("Cannot load component (%s): %s" % (component, e)) else: - print "-> Loaded component %s" % component + logger.info("Loaded component %s" % component) return u diff --git a/myslice/settings.py b/myslice/settings.py index 6a4518ce..3f79a19b 100644 --- a/myslice/settings.py +++ b/myslice/settings.py @@ -1,15 +1,24 @@ -#from __future__ import print_function import os.path +import logging +logger = logging.getLogger('myslice') + +# ROOT try: ROOT = os.path.realpath(os.path.dirname(__file__) + '/..') except: import traceback traceback.print_exc() -import myslice.components as components + from myslice.configengine import ConfigEngine +config = ConfigEngine() + +import myslice.components as components + + + # import djcelery # djcelery.setup_loader() @@ -20,14 +29,15 @@ try: except: building=True -config = ConfigEngine() + +# DEBUG if config.myslice.debug : DEBUG = True else : DEBUG = False -# themes +# theme if config.myslice.theme : theme = config.myslice.theme else : @@ -81,7 +91,7 @@ EMAIL_USE_TLS = False # EMAIL_USE_TLS = False # DEFAULT_FROM_EMAIL = 'testing@example.com' -if config.database : +if config.database.engine : DATABASES = { 'default': { 'ENGINE' : 'django.db.backends.%s' % config.database.engine, @@ -107,7 +117,7 @@ else : 'PORT' : '', } } -print DATABASES + # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. @@ -292,6 +302,39 @@ LOGGING = { }, } } +LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, + 'filters': { + + }, + 'handlers': { + 'null': { + 'level': 'DEBUG', + 'class': 'django.utils.log.NullHandler', + }, + 'debug':{ + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + } + }, + 'loggers': { + 'myslice': { + 'handlers': ['debug'], + 'propagate': True, + 'level': 'DEBUG', + } + } +} AUTHENTICATION_BACKENDS = ('auth.manifoldbackend.ManifoldBackend', 'django.contrib.auth.backends.ModelBackend')