Skip to content

Commit 69d4a27

Browse files
mmalonemmalone
committed
first commit
0 parents  commit 69d4a27

14 files changed

+563
-0
lines changed

‎README

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
An example Django app that uses custom managers, fields, and QuerySets to
2+
transparently cache objects.

‎__init__.py

Whitespace-only changes.

‎app/__init__.py

Whitespace-only changes.

‎app/cache.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
from django.core.cache import cache
2+
from django.utils.encoding import smart_str
3+
import inspect
4+
5+
6+
# Check if the cache backend supports min_compress_len. If so, add it.
7+
if 'min_compress_len' in inspect.getargspec(cache._cache.add)[0] and \
8+
'min_compress_len' in inspect.getargspec(cache._cache.set)[0]:
9+
class CacheClass(cache.__class__):
10+
def add(self, key, value, timeout=None, min_compress_len=150000):
11+
if isinstance(value, unicode):
12+
value = value.encode('utf-8')
13+
# Allow infinite timeouts
14+
if timeout is None:
15+
timeout = self.default_timeout
16+
return self._cache.add(smart_str(key), value, timeout, min_compress_len)
17+
18+
def set(self, key, value, timeout=None, min_compress_len=150000):
19+
if isinstance(value, unicode):
20+
value = value.encode('utf-8')
21+
if timeout is None:
22+
timeout = self.default_timeout
23+
self._cache.set(smart_str(key), value, timeout, min_compress_len)
24+
25+
cache.__class__ = CacheClass

‎app/fields.py

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import functools
2+
from django.db.models.fields.related import ManyToManyField, ReverseManyRelatedObjectsDescriptor, ManyRelatedObjectsDescriptor
3+
from django.db.models.query import QuerySet
4+
from django.db.models import signals
5+
from cache import cache
6+
from types import MethodType
7+
8+
CACHE_DURATION = 60 * 30
9+
10+
def invalidate_cache(obj, field):
11+
cache.set(obj._get_cache_key(field=field), None, 5)
12+
13+
def fix_where(where, modified=False):
14+
def wrap_add(f):
15+
@functools.wraps(f)
16+
def add(self, *args, **kwargs):
17+
"""
18+
Wraps django.db.models.sql.where.add to indicate that a new
19+
'where' condition has been added.
20+
"""
21+
self.modified = True
22+
return f(*args, **kwargs)
23+
return add
24+
where.modified = modified
25+
where.add = MethodType(wrap_add(where.add), where, where.__class__)
26+
return where
27+
28+
29+
def get_pk_list_query_set(superclass):
30+
class PKListQuerySet(superclass):
31+
"""
32+
QuerySet that, when unfiltered, fetches objects individually from
33+
the datastore by pk.
34+
35+
The `pk_list` attribute is a list of primary keys for objects that
36+
should be fetched.
37+
38+
"""
39+
def __init__(self, pk_list=[], from_cache=False, *args, **kwargs):
40+
super(PKListQuerySet, self).__init__(*args, **kwargs)
41+
self.pk_list = pk_list
42+
self.from_cache = from_cache
43+
self.query.where = fix_where(self.query.where)
44+
45+
def iterator(self):
46+
if not self.query.where.modified:
47+
for pk in self.pk_list:
48+
yield self.model._default_manager.get(pk=pk)
49+
else:
50+
superiter = super(PKListQuerySet, self).iterator()
51+
while True:
52+
yield superiter.next()
53+
54+
def _clone(self, *args, **kwargs):
55+
c = super(PKListQuerySet, self)._clone(*args, **kwargs)
56+
c.query.where = fix_where(c.query.where, modified=self.query.where.modified)
57+
c.pk_list = self.pk_list
58+
c.from_cache = self.from_cache
59+
return c
60+
return PKListQuerySet
61+
62+
63+
def get_caching_related_manager(superclass, instance, field_name, related_name):
64+
class CachingRelatedManager(superclass):
65+
def all(self):
66+
key = instance._get_cache_key(field=field_name)
67+
qs = super(CachingRelatedManager, self).get_query_set()
68+
PKListQuerySet = get_pk_list_query_set(qs.__class__)
69+
qs = qs._clone(klass=PKListQuerySet)
70+
pk_list = cache.get(key)
71+
if pk_list is None:
72+
pk_list = qs.values_list('pk', flat=True)
73+
cache.add(key, pk_list, CACHE_DURATION)
74+
else:
75+
qs.from_cache = True
76+
qs.pk_list = pk_list
77+
return qs
78+
79+
def add(self, *objs):
80+
super(CachingRelatedManager, self).add(*objs)
81+
for obj in objs:
82+
invalidate_cache(obj, related_name)
83+
invalidate_cache(instance, field_name)
84+
85+
def remove(self, *objs):
86+
super(CachingRelatedManager, self).remove(*objs)
87+
for obj in objs:
88+
invalidate_cache(obj, related_name)
89+
invalidate_cache(instance, field_name)
90+
91+
def clear(self):
92+
objs = list(self.all())
93+
super(CachingRelatedManager, self).clear()
94+
for obj in objs:
95+
invalidate_cache(obj, related_name)
96+
invalidate_cache(instance, field_name)
97+
return CachingRelatedManager
98+
99+
100+
class CachingReverseManyRelatedObjectsDescriptor(ReverseManyRelatedObjectsDescriptor):
101+
def __get__(self, instance, cls=None):
102+
manager = super(CachingReverseManyRelatedObjectsDescriptor, self).__get__(instance, cls)
103+
104+
CachingRelatedManager = get_caching_related_manager(manager.__class__,
105+
instance,
106+
self.field.name,
107+
self.field.rel.related_name)
108+
109+
manager.__class__ = CachingRelatedManager
110+
return manager
111+
112+
113+
class CachingManyRelatedObjectsDescriptor(ManyRelatedObjectsDescriptor):
114+
def __get__(self, instance, cls=None):
115+
manager = super(CachingManyRelatedObjectsDescriptor, self).__get__(instance, cls)
116+
117+
CachingRelatedManager = get_caching_related_manager(manager.__class__,
118+
instance,
119+
self.related.get_accessor_name(),
120+
self.related.field.name)
121+
122+
manager.__class__ = CachingRelatedManager
123+
return manager
124+
125+
126+
class CachingManyToManyField(ManyToManyField):
127+
def contribute_to_class(self, cls, name):
128+
super(CachingManyToManyField, self).contribute_to_class(cls, name)
129+
setattr(cls, self.name, CachingReverseManyRelatedObjectsDescriptor(self))
130+
131+
def contribute_to_related_class(self, cls, related):
132+
super(CachingManyToManyField, self).contribute_to_related_class(cls, related)
133+
setattr(cls, related.get_accessor_name(), CachingManyRelatedObjectsDescriptor(related))

‎app/fields_bad.py

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
from django.db.models.fields.related import ManyToManyField, ReverseManyRelatedObjectsDescriptor, ManyRelatedObjectsDescriptor
2+
from django.db.models.query import QuerySet
3+
from django.db.models import signals
4+
from cache import cache
5+
6+
def invalidate_cache(obj, field):
7+
cache.set(obj._get_cache_key(field=field), None, 5)
8+
9+
10+
def get_caching_related_manager(superclass, instance, field_name, related_name):
11+
class CachingRelatedManager(superclass):
12+
def all(self):
13+
key = instance._get_cache_key(field=field_name)
14+
qs = cache.get(key)
15+
if qs is None:
16+
qs = super(CachingRelatedManager, self).all()
17+
cache.add(key, qs, 60*30)
18+
qs.from_cache = False
19+
else:
20+
qs.from_cache = True
21+
return qs
22+
23+
def add(self, *objs):
24+
super(CachingRelatedManager, self).add(*objs)
25+
for obj in objs:
26+
invalidate_cache(obj, related_name)
27+
invalidate_cache(instance, field_name)
28+
29+
def remove(self, *objs):
30+
super(CachingRelatedManager, self).remove(*objs)
31+
for obj in objs:
32+
invalidate_cache(obj, related_name)
33+
invalidate_cache(instance, field_name)
34+
35+
def clear(self):
36+
objs = list(self.all())
37+
super(CachingRelatedManager, self).clear()
38+
for obj in objs:
39+
invalidate_cache(obj, related_name)
40+
invalidate_cache(instance, field_name)
41+
return CachingRelatedManager
42+
43+
44+
class CachingReverseManyRelatedObjectsDescriptor(ReverseManyRelatedObjectsDescriptor):
45+
def __get__(self, instance, cls=None):
46+
manager = super(CachingReverseManyRelatedObjectsDescriptor, self).__get__(instance, cls)
47+
48+
CachingRelatedManager = get_caching_related_manager(manager.__class__,
49+
instance,
50+
self.field.name,
51+
self.field.rel.related_name)
52+
53+
manager.__class__ = CachingRelatedManager
54+
return manager
55+
56+
57+
class CachingManyRelatedObjectsDescriptor(ManyRelatedObjectsDescriptor):
58+
def __get__(self, instance, cls=None):
59+
manager = super(CachingManyRelatedObjectsDescriptor, self).__get__(instance, cls)
60+
61+
CachingRelatedManager = get_caching_related_manager(manager.__class__,
62+
instance,
63+
self.related.get_accessor_name(),
64+
self.related.field.name)
65+
66+
manager.__class__ = CachingRelatedManager
67+
return manager
68+
69+
70+
class CachingManyToManyField(ManyToManyField):
71+
def contribute_to_class(self, cls, name):
72+
super(CachingManyToManyField, self).contribute_to_class(cls, name)
73+
setattr(cls, self.name, CachingReverseManyRelatedObjectsDescriptor(self))
74+
75+
def contribute_to_related_class(self, cls, related):
76+
super(CachingManyToManyField, self).contribute_to_related_class(cls, related)
77+
setattr(cls, related.get_accessor_name(), CachingManyRelatedObjectsDescriptor(related))

‎app/fixtures/test.json

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
[
2+
{
3+
"pk": 1,
4+
"model": "app.author",
5+
"fields": {
6+
"name": "Mike Malone"
7+
}
8+
},
9+
{
10+
"pk": 2,
11+
"model": "app.author",
12+
"fields": {
13+
"name": "Willy Wonka"
14+
}
15+
},
16+
{
17+
"pk": 3,
18+
"model": "app.author",
19+
"fields": {
20+
"name": "Harry Potter"
21+
}
22+
},
23+
{
24+
"pk": 1,
25+
"model": "app.site",
26+
"fields": {
27+
"name": "blog"
28+
}
29+
},
30+
{
31+
"pk": 2,
32+
"model": "app.site",
33+
"fields": {
34+
"name": "Site 2"
35+
}
36+
},
37+
{
38+
"pk": 1,
39+
"model": "app.article",
40+
"fields": {
41+
"sites": [],
42+
"name": "Article 1",
43+
"author": 1
44+
}
45+
},
46+
{
47+
"pk": 2,
48+
"model": "app.article",
49+
"fields": {
50+
"sites": [],
51+
"name": "Article 2",
52+
"author": 1
53+
}
54+
},
55+
{
56+
"pk": 3,
57+
"model": "app.article",
58+
"fields": {
59+
"sites": [],
60+
"name": "Article 3",
61+
"author": 1
62+
}
63+
},
64+
{
65+
"pk": 4,
66+
"model": "app.article",
67+
"fields": {
68+
"sites": [],
69+
"name": "Article 4",
70+
"author": 1
71+
}
72+
},
73+
{
74+
"pk": 5,
75+
"model": "app.article",
76+
"fields": {
77+
"sites": [],
78+
"name": "Article 5",
79+
"author": 2
80+
}
81+
}
82+
]

0 commit comments

Comments
 (0)