From 5d1186b318a037732efa0c59c9921388bafd086f Mon Sep 17 00:00:00 2001
From: Sapan Bhatia <gwsapan@gmail.com>
Date: Wed, 9 Mar 2011 10:14:49 -0500
Subject: [PATCH] Added cache_utils to the distribution because it includes a
 patch and is not packaged by maintainer

---
 PLCAPI.spec                  |   1 +
 cache_utils/__init__.py      |   1 +
 cache_utils/decorators.py    |  54 +++++++++++++
 cache_utils/group_backend.py | 110 +++++++++++++++++++++++++
 cache_utils/models.py        |   1 +
 cache_utils/tests.py         | 150 +++++++++++++++++++++++++++++++++++
 cache_utils/utils.py         |  60 ++++++++++++++
 7 files changed, 377 insertions(+)
 create mode 100644 cache_utils/__init__.py
 create mode 100644 cache_utils/decorators.py
 create mode 100644 cache_utils/group_backend.py
 create mode 100644 cache_utils/models.py
 create mode 100644 cache_utils/tests.py
 create mode 100644 cache_utils/utils.py

diff --git a/PLCAPI.spec b/PLCAPI.spec
index bc4992b..b9a230a 100644
--- a/PLCAPI.spec
+++ b/PLCAPI.spec
@@ -44,6 +44,7 @@ Requires: python-ldap
 # for memcache
 Requires: python-memcached
 Requires: memcached
+Requires: Django
 ### avoid having yum complain about updates, as stuff is moving around
 # plc.d/api
 Conflicts: MyPLC <= 4.3
diff --git a/cache_utils/__init__.py b/cache_utils/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/cache_utils/__init__.py
@@ -0,0 +1 @@
+
diff --git a/cache_utils/decorators.py b/cache_utils/decorators.py
new file mode 100644
index 0000000..4dd1d4d
--- /dev/null
+++ b/cache_utils/decorators.py
@@ -0,0 +1,54 @@
+#coding: utf-8
+from django.core.cache import cache
+from django.utils.functional import wraps
+from cache_utils.utils import _cache_key, _func_info, _func_type
+
+def cached(timeout, group=None):
+    """ Caching decorator. Can be applied to function, method or classmethod.
+    Supports bulk cache invalidation and invalidation for exact parameter
+    set. Cache keys are human-readable because they are constructed from
+    callable's full name and arguments and then sanitized to make
+    memcached happy.
+
+    It can be used with or without group_backend. Without group_backend
+    bulk invalidation is not supported.
+
+    Wrapped callable gets `invalidate` methods. Call `invalidate` with
+    same arguments as function and the result for these arguments will be
+    invalidated.
+    """
+
+    backend_kwargs = {'group': group} if group else {}
+
+    def _cached(func):
+
+        func_type = _func_type(func)
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+
+            # full name is stored as attribute on first call
+            if not hasattr(wrapper, '_full_name'):
+                name, _args = _func_info(func, args)
+                wrapper._full_name = name
+
+            # try to get the value from cache
+            key = _cache_key(wrapper._full_name, func_type, args, kwargs)
+            value = cache.get(key, **backend_kwargs)
+
+            # in case of cache miss recalculate the value and put it to the cache
+            if value is None:
+                value = func(*args, **kwargs)
+                cache.set(key, value, timeout, **backend_kwargs)
+            return value
+
+        def invalidate(*args, **kwargs):
+            ''' invalidates cache result for function called with passed arguments '''
+            if not hasattr(wrapper, '_full_name'):
+                return
+            key = _cache_key(wrapper._full_name, 'function', args, kwargs)
+            cache.delete(key, **backend_kwargs)
+
+        wrapper.invalidate = invalidate
+        return wrapper
+    return _cached
diff --git a/cache_utils/group_backend.py b/cache_utils/group_backend.py
new file mode 100644
index 0000000..8377477
--- /dev/null
+++ b/cache_utils/group_backend.py
@@ -0,0 +1,110 @@
+"""
+Memcached cache backend with group O(1) invalidation ability, dog-pile
+effect prevention using MintCache algorythm and project version support to allow
+gracefull updates and multiple django projects on same memcached instance.
+Long keys (>250) are truncated and appended with md5 hash.
+"""
+
+import uuid
+import logging
+import sys
+import time
+from django.core.cache.backends.memcached import CacheClass as MemcachedCacheClass
+from django.conf import settings
+from cache_utils.utils import sanitize_memcached_key
+
+# This prefix is appended to the group name to prevent cache key clashes.
+_VERSION_PREFIX = getattr(settings, 'VERSION', "")
+_KEY_PREFIX = "_group::"
+
+# MINT_DELAY is an upper bound on how long any value should take to
+# be generated (in seconds)
+MINT_DELAY = 30
+
+class CacheClass(MemcachedCacheClass):
+
+    def add(self, key, value, timeout=0, group=None):
+        key = self._make_key(group, key)
+
+        refresh_time = timeout + time.time()
+        real_timeout = timeout + MINT_DELAY
+        packed_value = (value, refresh_time, False)
+
+        return super(CacheClass, self).add(key, packed_value, real_timeout)
+
+    def get(self, key, default=None, group=None):
+        key = self._make_key(group, key)
+        packed_value = super(CacheClass, self).get(key, default)
+        if packed_value is None:
+            return default
+        value, refresh_time, refreshed = packed_value
+        if (time.time() > refresh_time) and not refreshed:
+            # Store the stale value while the cache revalidates for another
+            # MINT_DELAY seconds.
+            self.set(key, value, timeout=MINT_DELAY, group=group, refreshed=True)
+            return default
+        return value
+
+    def set(self, key, value, timeout=0, group=None, refreshed=False):
+        key = self._make_key(group, key)
+        refresh_time = timeout + time.time()
+        real_timeout = timeout + MINT_DELAY
+        packed_value = (value, refresh_time, refreshed)
+        return super(CacheClass, self).set(key, packed_value, real_timeout)
+
+    def delete(self, key, group=None):
+        key = self._make_key(group, key)
+        return super(CacheClass, self).delete(key)
+
+    def invalidate_group(self, group):
+        """ Invalidates all cache keys belonging to group """
+        key = "%s%s%s" % (_VERSION_PREFIX, _KEY_PREFIX, group)
+        super(CacheClass, self).delete(key)
+
+    def _make_key(self, group, key, hashkey=None):
+        """ Generates a new cache key which belongs to a group, has
+            _VERSION_PREFIX prepended and is shorter than memcached key length
+            limit.
+        """
+        key = _VERSION_PREFIX + key
+        if group:
+            if not hashkey:
+                hashkey = self._get_hashkey(group)
+            key = "%s:%s-%s" % (group, key, hashkey)
+        return sanitize_memcached_key(key)
+
+    def _get_hashkey(self, group):
+        """ This can be useful sometimes if you're doing a very large number
+            of operations and you want to avoid all of the extra cache hits.
+        """
+        key = "%s%s%s" % (_VERSION_PREFIX, _KEY_PREFIX, group)
+        hashkey = super(CacheClass, self).get(key)
+        if hashkey is None:
+            hashkey = str(uuid.uuid4())
+            super(CacheClass, self).set(key, hashkey)
+        return hashkey
+
+    def clear(self):
+        self._cache.flush_all()
+
+# ======================================
+# I didn't implement methods below to work with MintCache so raise
+# NotImplementedError for them.
+
+    def incr(self, key, delta=1, group=None):
+#        if group:
+#            key = self._make_key(group, key)
+#        return super(CacheClass, self).incr(key, delta)
+        raise NotImplementedError
+
+    def decr(self, key, delta=1, group=None):
+#        if group:
+#            key = self._make_key(group, key)
+#        return super(CacheClass, self).decr(key, delta)
+        raise NotImplementedError
+
+    def get_many(self, keys, group=None):
+#        hashkey = self._get_hashkey(group)
+#        keys = [self._make_key(group, k, hashkey) for k in keys]
+#        return super(CacheClass, self).get_many(keys)
+        raise NotImplementedError
diff --git a/cache_utils/models.py b/cache_utils/models.py
new file mode 100644
index 0000000..da0c0d5
--- /dev/null
+++ b/cache_utils/models.py
@@ -0,0 +1 @@
+# Hello, testrunner!
diff --git a/cache_utils/tests.py b/cache_utils/tests.py
new file mode 100644
index 0000000..b55857e
--- /dev/null
+++ b/cache_utils/tests.py
@@ -0,0 +1,150 @@
+#coding: utf-8
+
+from unittest import TestCase
+
+from django.core.cache import cache
+from cache_utils.decorators import cached
+from cache_utils.utils import sanitize_memcached_key, _func_type, _func_info
+
+def foo(a,b):
+    pass
+
+class Foo(object):
+    def foo(self, a, b):
+        pass
+    @classmethod
+    def bar(cls, x):
+        pass
+
+class FuncTypeTest(TestCase):
+    def assertFuncType(self, func, tp):
+        self.assertEqual(_func_type(func), tp)
+
+    def test_func(self):
+        self.assertFuncType(foo, 'function')
+
+    def test_method(self):
+        self.assertFuncType(Foo.foo, 'method')
+
+    def test_classmethod(self):
+        self.assertFuncType(Foo.bar, 'classmethod')
+
+
+class FuncInfoTest(TestCase):
+    def assertFuncInfo(self, func, args_in, name, args_out):
+        info = _func_info(func, args_in)
+        self.assertEqual(info[0], name)
+        self.assertEqual(info[1], args_out)
+
+    def test_func(self):
+        self.assertFuncInfo(foo, [1,2], 'cache_utils.tests.foo', [1,2])
+
+    def test_method(self):
+        foo_obj = Foo()
+        self.assertFuncInfo(Foo.foo, [foo_obj, 1, 2],
+                            'cache_utils.tests.Foo.foo', [1,2])
+
+    def test_classmethod(self):
+        self.assertFuncInfo(Foo.bar, [Foo, 1],
+                            'cache_utils.tests.Foo.bar', [1])
+
+
+class SanitizeTest(TestCase):
+    def test_sanitize_keys(self):
+        key = u"12345678901234567890123456789012345678901234567890"
+        self.assertTrue(len(key) >= 40)
+        key = sanitize_memcached_key(key, 40)
+        self.assertTrue(len(key) <= 40)
+
+
+class ClearMemcachedTest(TestCase):
+    def tearDown(self):
+        cache._cache.flush_all()
+
+    def setUp(self):
+        cache._cache.flush_all()
+
+
+class InvalidationTest(ClearMemcachedTest):
+
+    def test_group_invalidation(self):
+        cache.set('vasia', 'foo', 60, group='names')
+        cache.set('petya', 'bar', 60, group='names')
+        cache.set('red', 'good', 60, group='colors')
+
+        self.assertEqual(cache.get('vasia', group='names'), 'foo')
+        self.assertEqual(cache.get('petya', group='names'), 'bar')
+        self.assertEqual(cache.get('red', group='colors'), 'good')
+
+        cache.invalidate_group('names')
+        self.assertEqual(cache.get('petya', group='names'), None)
+        self.assertEqual(cache.get('vasia', group='names'), None)
+        self.assertEqual(cache.get('red', group='colors'), 'good')
+
+        cache.set('vasia', 'foo', 60, group='names')
+        self.assertEqual(cache.get('vasia', group='names'), 'foo')
+
+    def test_func_invalidation(self):
+        self.call_count = 0
+
+        @cached(60)
+        def my_func(a, b):
+            self.call_count += 1
+            return self.call_count
+
+        self.assertEqual(my_func(1,2), 1)
+        self.assertEqual(my_func(1,2), 1)
+        self.assertEqual(my_func(3,2), 2)
+        self.assertEqual(my_func(3,2), 2)
+        my_func.invalidate(3,2)
+        self.assertEqual(my_func(1,2), 1)
+        self.assertEqual(my_func(3,2), 3)
+        self.assertEqual(my_func(3,2), 3)
+
+    def test_method_invalidation(self):
+        self.call_count = 0
+        this = self
+
+        class Foo(object):
+            @cached(60)
+            def bar(self, x):
+                this.call_count += 1
+                return this.call_count
+
+        foo = Foo()
+        self.assertEqual(foo.bar(1), 1)
+        self.assertEqual(foo.bar(1), 1)
+        Foo.bar.invalidate(1)
+        self.assertEqual(foo.bar(1), 2)
+
+    def test_invalidate_nonexisting(self):
+        @cached(60)
+        def foo(x):
+            return 1
+        foo.invalidate(5) # this shouldn't raise exception
+
+
+class DecoratorTest(ClearMemcachedTest):
+
+    def test_decorator(self):
+        self._x = 0
+
+        @cached(60, group='test-group')
+        def my_func(params=""):
+            self._x = self._x + 1
+            return u"%d%s" % (self._x, params)
+
+        self.assertEqual(my_func(), "1")
+        self.assertEqual(my_func(), "1")
+
+        self.assertEqual(my_func("x"), u"2x")
+        self.assertEqual(my_func("x"), u"2x")
+
+        self.assertEqual(my_func(u"Василий"), u"3Василий")
+        self.assertEqual(my_func(u"Василий"), u"3Василий")
+
+        self.assertEqual(my_func(u"й"*240), u"4"+u"й"*240)
+        self.assertEqual(my_func(u"й"*240), u"4"+u"й"*240)
+
+        self.assertEqual(my_func(u"Ы"*500), u"5"+u"Ы"*500)
+        self.assertEqual(my_func(u"Ы"*500), u"5"+u"Ы"*500)
diff --git a/cache_utils/utils.py b/cache_utils/utils.py
new file mode 100644
index 0000000..ed7474d
--- /dev/null
+++ b/cache_utils/utils.py
@@ -0,0 +1,60 @@
+from hashlib import md5
+
+CONTROL_CHARACTERS = set([chr(i) for i in range(0,33)])
+CONTROL_CHARACTERS.add(chr(127))
+
+def sanitize_memcached_key(key, max_length=250):
+    """ Removes control characters and ensures that key will
+        not hit the memcached key length limit by replacing
+        the key tail with md5 hash if key is too long.
+    """
+    key = ''.join([c for c in key if c not in CONTROL_CHARACTERS])
+    if len(key) > max_length:
+        hash = md5(key).hexdigest()
+        key = key[:max_length-33]+'-'+hash
+    return key
+
+def _args_to_unicode(args, kwargs):
+    key = ""
+    if args:
+        key += unicode(args)
+    if kwargs:
+        key += unicode(kwargs)
+    return key
+
+
+def _func_type(func):
+    """ returns if callable is a function, method or a classmethod """
+    argnames = func.func_code.co_varnames[:func.func_code.co_argcount]
+    if len(argnames) > 0:
+        if argnames[0] == 'self':
+            return 'method'
+        if argnames[0] == 'cls':
+            return 'classmethod'
+    return 'function'
+
+
+def _func_info(func, args):
+    ''' introspect function's or method's full name.
+    Returns a tuple (name, normalized_args,) with
+    'cls' and 'self' removed from normalized_args '''
+
+    func_type = _func_type(func)
+
+    if func_type == 'function':
+        return ".".join([func.__module__, func.__name__]), args
+
+    class_name = args[0].__class__.__name__
+    if func_type == 'classmethod':
+        class_name = args[0].__name__
+
+    return ".".join([func.__module__, class_name, func.__name__]), args[1:]
+
+
+def _cache_key(func_name, func_type, args, kwargs):
+    """ Construct readable cache key """
+    if func_type == 'function':
+        args_string = _args_to_unicode(args, kwargs)
+    else:
+        args_string = _args_to_unicode(args[1:], kwargs)
+    return sanitize_memcached_key('[cached]%s(%s)' % (func_name, args_string,))
-- 
2.43.0