# for memcache
Requires: python-memcached
Requires: memcached
+Requires: Django
### avoid having yum complain about updates, as stuff is moving around
# plc.d/api
Conflicts: MyPLC <= 4.3
--- /dev/null
+#coding: utf-8
+from django.core.cache import cache
+from django.utils.functional import wraps
+from cache_utils.utils import _cache_key, _func_info, _func_type
+
+def cached(timeout, group=None):
+ """ Caching decorator. Can be applied to function, method or classmethod.
+ Supports bulk cache invalidation and invalidation for exact parameter
+ set. Cache keys are human-readable because they are constructed from
+ callable's full name and arguments and then sanitized to make
+ memcached happy.
+
+ It can be used with or without group_backend. Without group_backend
+ bulk invalidation is not supported.
+
+ Wrapped callable gets `invalidate` methods. Call `invalidate` with
+ same arguments as function and the result for these arguments will be
+ invalidated.
+ """
+
+ backend_kwargs = {'group': group} if group else {}
+
+ def _cached(func):
+
+ func_type = _func_type(func)
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+
+ # full name is stored as attribute on first call
+ if not hasattr(wrapper, '_full_name'):
+ name, _args = _func_info(func, args)
+ wrapper._full_name = name
+
+ # try to get the value from cache
+ key = _cache_key(wrapper._full_name, func_type, args, kwargs)
+ value = cache.get(key, **backend_kwargs)
+
+ # in case of cache miss recalculate the value and put it to the cache
+ if value is None:
+ value = func(*args, **kwargs)
+ cache.set(key, value, timeout, **backend_kwargs)
+ return value
+
+ def invalidate(*args, **kwargs):
+ ''' invalidates cache result for function called with passed arguments '''
+ if not hasattr(wrapper, '_full_name'):
+ return
+ key = _cache_key(wrapper._full_name, 'function', args, kwargs)
+ cache.delete(key, **backend_kwargs)
+
+ wrapper.invalidate = invalidate
+ return wrapper
+ return _cached
--- /dev/null
+"""
+Memcached cache backend with group O(1) invalidation ability, dog-pile
+effect prevention using MintCache algorythm and project version support to allow
+gracefull updates and multiple django projects on same memcached instance.
+Long keys (>250) are truncated and appended with md5 hash.
+"""
+
+import uuid
+import logging
+import sys
+import time
+from django.core.cache.backends.memcached import CacheClass as MemcachedCacheClass
+from django.conf import settings
+from cache_utils.utils import sanitize_memcached_key
+
+# This prefix is appended to the group name to prevent cache key clashes.
+_VERSION_PREFIX = getattr(settings, 'VERSION', "")
+_KEY_PREFIX = "_group::"
+
+# MINT_DELAY is an upper bound on how long any value should take to
+# be generated (in seconds)
+MINT_DELAY = 30
+
+class CacheClass(MemcachedCacheClass):
+
+ def add(self, key, value, timeout=0, group=None):
+ key = self._make_key(group, key)
+
+ refresh_time = timeout + time.time()
+ real_timeout = timeout + MINT_DELAY
+ packed_value = (value, refresh_time, False)
+
+ return super(CacheClass, self).add(key, packed_value, real_timeout)
+
+ def get(self, key, default=None, group=None):
+ key = self._make_key(group, key)
+ packed_value = super(CacheClass, self).get(key, default)
+ if packed_value is None:
+ return default
+ value, refresh_time, refreshed = packed_value
+ if (time.time() > refresh_time) and not refreshed:
+ # Store the stale value while the cache revalidates for another
+ # MINT_DELAY seconds.
+ self.set(key, value, timeout=MINT_DELAY, group=group, refreshed=True)
+ return default
+ return value
+
+ def set(self, key, value, timeout=0, group=None, refreshed=False):
+ key = self._make_key(group, key)
+ refresh_time = timeout + time.time()
+ real_timeout = timeout + MINT_DELAY
+ packed_value = (value, refresh_time, refreshed)
+ return super(CacheClass, self).set(key, packed_value, real_timeout)
+
+ def delete(self, key, group=None):
+ key = self._make_key(group, key)
+ return super(CacheClass, self).delete(key)
+
+ def invalidate_group(self, group):
+ """ Invalidates all cache keys belonging to group """
+ key = "%s%s%s" % (_VERSION_PREFIX, _KEY_PREFIX, group)
+ super(CacheClass, self).delete(key)
+
+ def _make_key(self, group, key, hashkey=None):
+ """ Generates a new cache key which belongs to a group, has
+ _VERSION_PREFIX prepended and is shorter than memcached key length
+ limit.
+ """
+ key = _VERSION_PREFIX + key
+ if group:
+ if not hashkey:
+ hashkey = self._get_hashkey(group)
+ key = "%s:%s-%s" % (group, key, hashkey)
+ return sanitize_memcached_key(key)
+
+ def _get_hashkey(self, group):
+ """ This can be useful sometimes if you're doing a very large number
+ of operations and you want to avoid all of the extra cache hits.
+ """
+ key = "%s%s%s" % (_VERSION_PREFIX, _KEY_PREFIX, group)
+ hashkey = super(CacheClass, self).get(key)
+ if hashkey is None:
+ hashkey = str(uuid.uuid4())
+ super(CacheClass, self).set(key, hashkey)
+ return hashkey
+
+ def clear(self):
+ self._cache.flush_all()
+
+# ======================================
+# I didn't implement methods below to work with MintCache so raise
+# NotImplementedError for them.
+
+ def incr(self, key, delta=1, group=None):
+# if group:
+# key = self._make_key(group, key)
+# return super(CacheClass, self).incr(key, delta)
+ raise NotImplementedError
+
+ def decr(self, key, delta=1, group=None):
+# if group:
+# key = self._make_key(group, key)
+# return super(CacheClass, self).decr(key, delta)
+ raise NotImplementedError
+
+ def get_many(self, keys, group=None):
+# hashkey = self._get_hashkey(group)
+# keys = [self._make_key(group, k, hashkey) for k in keys]
+# return super(CacheClass, self).get_many(keys)
+ raise NotImplementedError
--- /dev/null
+# Hello, testrunner!
--- /dev/null
+#coding: utf-8
+
+from unittest import TestCase
+
+from django.core.cache import cache
+from cache_utils.decorators import cached
+from cache_utils.utils import sanitize_memcached_key, _func_type, _func_info
+
+def foo(a,b):
+ pass
+
+class Foo(object):
+ def foo(self, a, b):
+ pass
+ @classmethod
+ def bar(cls, x):
+ pass
+
+class FuncTypeTest(TestCase):
+ def assertFuncType(self, func, tp):
+ self.assertEqual(_func_type(func), tp)
+
+ def test_func(self):
+ self.assertFuncType(foo, 'function')
+
+ def test_method(self):
+ self.assertFuncType(Foo.foo, 'method')
+
+ def test_classmethod(self):
+ self.assertFuncType(Foo.bar, 'classmethod')
+
+
+class FuncInfoTest(TestCase):
+ def assertFuncInfo(self, func, args_in, name, args_out):
+ info = _func_info(func, args_in)
+ self.assertEqual(info[0], name)
+ self.assertEqual(info[1], args_out)
+
+ def test_func(self):
+ self.assertFuncInfo(foo, [1,2], 'cache_utils.tests.foo', [1,2])
+
+ def test_method(self):
+ foo_obj = Foo()
+ self.assertFuncInfo(Foo.foo, [foo_obj, 1, 2],
+ 'cache_utils.tests.Foo.foo', [1,2])
+
+ def test_classmethod(self):
+ self.assertFuncInfo(Foo.bar, [Foo, 1],
+ 'cache_utils.tests.Foo.bar', [1])
+
+
+class SanitizeTest(TestCase):
+ def test_sanitize_keys(self):
+ key = u"12345678901234567890123456789012345678901234567890"
+ self.assertTrue(len(key) >= 40)
+ key = sanitize_memcached_key(key, 40)
+ self.assertTrue(len(key) <= 40)
+
+
+class ClearMemcachedTest(TestCase):
+ def tearDown(self):
+ cache._cache.flush_all()
+
+ def setUp(self):
+ cache._cache.flush_all()
+
+
+class InvalidationTest(ClearMemcachedTest):
+
+ def test_group_invalidation(self):
+ cache.set('vasia', 'foo', 60, group='names')
+ cache.set('petya', 'bar', 60, group='names')
+ cache.set('red', 'good', 60, group='colors')
+
+ self.assertEqual(cache.get('vasia', group='names'), 'foo')
+ self.assertEqual(cache.get('petya', group='names'), 'bar')
+ self.assertEqual(cache.get('red', group='colors'), 'good')
+
+ cache.invalidate_group('names')
+ self.assertEqual(cache.get('petya', group='names'), None)
+ self.assertEqual(cache.get('vasia', group='names'), None)
+ self.assertEqual(cache.get('red', group='colors'), 'good')
+
+ cache.set('vasia', 'foo', 60, group='names')
+ self.assertEqual(cache.get('vasia', group='names'), 'foo')
+
+ def test_func_invalidation(self):
+ self.call_count = 0
+
+ @cached(60)
+ def my_func(a, b):
+ self.call_count += 1
+ return self.call_count
+
+ self.assertEqual(my_func(1,2), 1)
+ self.assertEqual(my_func(1,2), 1)
+ self.assertEqual(my_func(3,2), 2)
+ self.assertEqual(my_func(3,2), 2)
+ my_func.invalidate(3,2)
+ self.assertEqual(my_func(1,2), 1)
+ self.assertEqual(my_func(3,2), 3)
+ self.assertEqual(my_func(3,2), 3)
+
+ def test_method_invalidation(self):
+ self.call_count = 0
+ this = self
+
+ class Foo(object):
+ @cached(60)
+ def bar(self, x):
+ this.call_count += 1
+ return this.call_count
+
+ foo = Foo()
+ self.assertEqual(foo.bar(1), 1)
+ self.assertEqual(foo.bar(1), 1)
+ Foo.bar.invalidate(1)
+ self.assertEqual(foo.bar(1), 2)
+
+ def test_invalidate_nonexisting(self):
+ @cached(60)
+ def foo(x):
+ return 1
+ foo.invalidate(5) # this shouldn't raise exception
+
+
+class DecoratorTest(ClearMemcachedTest):
+
+ def test_decorator(self):
+ self._x = 0
+
+ @cached(60, group='test-group')
+ def my_func(params=""):
+ self._x = self._x + 1
+ return u"%d%s" % (self._x, params)
+
+ self.assertEqual(my_func(), "1")
+ self.assertEqual(my_func(), "1")
+
+ self.assertEqual(my_func("x"), u"2x")
+ self.assertEqual(my_func("x"), u"2x")
+
+ self.assertEqual(my_func(u"Василий"), u"3Василий")
+ self.assertEqual(my_func(u"Василий"), u"3Василий")
+
+ self.assertEqual(my_func(u"й"*240), u"4"+u"й"*240)
+ self.assertEqual(my_func(u"й"*240), u"4"+u"й"*240)
+
+ self.assertEqual(my_func(u"Ы"*500), u"5"+u"Ы"*500)
+ self.assertEqual(my_func(u"Ы"*500), u"5"+u"Ы"*500)
--- /dev/null
+from hashlib import md5
+
+CONTROL_CHARACTERS = set([chr(i) for i in range(0,33)])
+CONTROL_CHARACTERS.add(chr(127))
+
+def sanitize_memcached_key(key, max_length=250):
+ """ Removes control characters and ensures that key will
+ not hit the memcached key length limit by replacing
+ the key tail with md5 hash if key is too long.
+ """
+ key = ''.join([c for c in key if c not in CONTROL_CHARACTERS])
+ if len(key) > max_length:
+ hash = md5(key).hexdigest()
+ key = key[:max_length-33]+'-'+hash
+ return key
+
+def _args_to_unicode(args, kwargs):
+ key = ""
+ if args:
+ key += unicode(args)
+ if kwargs:
+ key += unicode(kwargs)
+ return key
+
+
+def _func_type(func):
+ """ returns if callable is a function, method or a classmethod """
+ argnames = func.func_code.co_varnames[:func.func_code.co_argcount]
+ if len(argnames) > 0:
+ if argnames[0] == 'self':
+ return 'method'
+ if argnames[0] == 'cls':
+ return 'classmethod'
+ return 'function'
+
+
+def _func_info(func, args):
+ ''' introspect function's or method's full name.
+ Returns a tuple (name, normalized_args,) with
+ 'cls' and 'self' removed from normalized_args '''
+
+ func_type = _func_type(func)
+
+ if func_type == 'function':
+ return ".".join([func.__module__, func.__name__]), args
+
+ class_name = args[0].__class__.__name__
+ if func_type == 'classmethod':
+ class_name = args[0].__name__
+
+ return ".".join([func.__module__, class_name, func.__name__]), args[1:]
+
+
+def _cache_key(func_name, func_type, args, kwargs):
+ """ Construct readable cache key """
+ if func_type == 'function':
+ args_string = _args_to_unicode(args, kwargs)
+ else:
+ args_string = _args_to_unicode(args[1:], kwargs)
+ return sanitize_memcached_key('[cached]%s(%s)' % (func_name, args_string,))
git://git.onelab.eu / plcapi.git / commitdiff