From: Radomir <radomir.klacza@lip6.fr>
Date: Fri, 28 Aug 2015 08:43:12 +0000 (+0200)
Subject: Stats plugin - Adding support for messages local caching in sqlite db if athos server... 
X-Git-Url: http://git.onelab.eu/?p=myslice.git;a=commitdiff_plain;h=1ac10866475f655cadf4c189fb803b49dab37c8f

Stats plugin - Adding support for messages local caching in sqlite db if athos server is not responding.
---

diff --git a/activity/__init__.py b/activity/__init__.py
index b7496724..9c56aaad 100644
--- a/activity/__init__.py
+++ b/activity/__init__.py
@@ -15,7 +15,11 @@ import time
 import datetime
 from myslice.configengine import ConfigEngine
 from myslice.settings import logger
-
+import random
+import os
+import sqlite3 as lite
+import json
+import syslog
 
 config = ConfigEngine()
 if config.activity and config.activity.apikey :
@@ -73,10 +77,31 @@ def logWrite(request, action, message, objects = None):
         result = urllib2.urlopen(server, urllib.urlencode(log))
         logger.info("===============>> activity: {} <{}> {}".format(action, request.user,message))
         content = result.read()
+
+        #checking for not sent data and sending it (50% probability)
+        if random.randint(0,100) < 50:
+            logCheck()
+
     except urllib2.URLError as e:
         logger.error("===============>> activity: connection to {} impossible, could not log action".format(server))
         logger.error(e.strerror)
 
+        dbfile = ''.join([os.path.dirname(os.path.abspath(__file__)), "/errors.db"])
+        conn = None
+        try:
+            conn = lite.connect(dbfile)
+            cur = conn.cursor()
+            cur.execute("""INSERT INTO logs(log) values('%s')""" % json.dumps(log))
+            conn.commit()
+        except lite.Error, e:
+            # this means that writing log into db also failed :(
+            # Last chance to preserve log is to send it to system syslog
+            # however there is no mechanism to pull it from this log - just manually.
+            logger.error('[activity] Error while inserting into sql db: %s' % str(e.args))
+            logger.error("[activity] data to send: '%s'" % json.dumps(log))
+        if conn:
+            conn.close()
+
 def log(request, action, message, objects = None):
     # Create a new thread in Daemon mode to send the log entry
     t = threading.Thread(target=logWrite, args=(request, action, message, objects))
@@ -95,3 +120,38 @@ def getClientIp(request):
 # sign the request with the secret key
 def sign(secret, message):
     return hmac.new(secret, msg=message, digestmod=hashlib.sha256).hexdigest()
+
+#
+# sending the logs cached in sqlite database
+
+def logCheck():
+    """Checking local database for logs adn sending it to monitoring server"""
+    dbfile = ''.join([os.path.dirname(os.path.abspath(__file__)), "/errors.db"])
+    conn = None
+
+    #trying to connect local db adn pull unsent logs
+    try:
+        conn = lite.connect(dbfile)
+        cur = conn.cursor()
+        cur.execute("SELECT rowid, log from logs")
+        notsent = cur.fetchall()
+        for row in notsent:
+            #trying to send unsent data from sqlite db
+            try :
+                urllib2.urlopen(config.activity.server, urllib.urlencode(json.loads(row[1])))
+                #delete those who were sent
+                cur.execute("""DELETE FROM logs where rowid = %s""" % row[0] )
+                conn.commit()
+            except urllib2.URLError as e:
+                # this is just to inform that DB is not working properly
+                logger.error('[activity] Error while sending stats')
+                logger.error(e.strerror)
+
+    except lite.Error, e:
+        #this need to be updated to store information via syslog
+        logger.error('[activity] Error while pulling from local sqlite3 db: %s' % str(e.args))
+        logger.error(e.strerror)
+    if conn:
+        conn.close()
+
+    return