django---django_redis源码分析
-
-
- 简单示例
- 源码分析
-
简单示例
1、安装
2、配置
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"CONNECTION_POOL_KWARGS": {"max_connections": 100}
# "PASSWORD": "密码",
}
},
# "d1": {
# "BACKEND": "django_redis.cache.RedisCache",
# "LOCATION": "redis://127.0.0.1:6379",
# "OPTIONS": {
# "CLIENT_CLASS": "django_redis.client.DefaultClient",
# "CONNECTION_POOL_KWARGS": {"max_connections": 100}
# # "PASSWORD": "密码",
# }
# }
}
SESSION_ENGINE = 'django.contrib.sessions.backends.cache' # 引擎
SESSION_CACHE_ALIAS = 'default'3、代码演示
from django_redis import get_redis_connection
def django_redis(request):
conn = get_redis_connection("default")
conn.hset("safly","k1","v1")
by = conn.hget("safly","k1")
print(str(by))
return HttpResponse("----")
输出b'v1'
源码分析
先贴出该模块源码的结构图
在上述代码中conn = get_redis_connection("default")，我们就从该入口进行跟踪
# -*- coding: utf-8 -*-
VERSION = (4, 9, 0)
__version__ = '.'.join(map(str, VERSION))
def get_redis_connection(alias='default', write=True):
from django.core.cache import caches
cache = caches[alias]
if not hasattr(cache, "client"):
raise NotImplementedError("This backend does not support this feature")
if not hasattr(cache.client, "get_client"):
raise NotImplementedError("This backend does not support this feature")
return cache.client.get_client(write)首先看from django.core.cache import caches，这是导入了django下的模块，利用了django下的强大的缓存模块功能，接下来cache = caches[alias]，是获取某个缓存对象，例如文件缓存、内存缓存、全站缓存、局部视图缓存等，具体是什么对象呢，一会我们在说，先来看下我们的配置
我们在使用django-redis时候，会在配置文件进行如下配置，然后在get_redis_connection(alias='default', write=True):函数中将alias='default'当默认参数
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"CONNECTION_POOL_KWARGS": {"max_connections": 100}
# "PASSWORD": "密码",
}
},经过我打印输出cache = caches[alias]，是输出如下对象
object at 0x07043DD0> 我们就简单来看下cache = caches[alias]做了什么操作把
caches = CacheHandler()class CacheHandler(object):
"""
A Cache Handler to manage access to Cache instances.
Ensures only one instance of each alias exists per thread.
"""
def __init__(self):
self._caches = local()
def __getitem__(self, alias):
try:
return self._caches.caches[alias]
except AttributeError:
self._caches.caches = {}
except KeyError:
pass
if alias not in settings.CACHES:
raise InvalidCacheBackendError(
"Could not find config for '%s' in settings.CACHES" % alias
)
cache = _create_cache(alias)
self._caches.caches[alias] = cache
return cache
def all(self):
return getattr(self._caches, 'caches', {}).values()我们首先看该类初始化做了什么操作？在CacheHandler类中的__init__方法中，通过 self._caches = local()，在local类中我们主要看下初始方法
def __new__(cls, *args, **kw):
if (args or kw) and (cls.__init__ is object.__init__):
raise TypeError("Initialization arguments are not supported")
self = object.__new__(cls)
impl = _localimpl()
impl.localargs = (args, kw)
impl.locallock = RLock()
object.__setattr__(self, '_local__impl', impl)
# We need to create the thread dict in anticipation of
# __init__ being called, to make sure we don't call it
# again ourselves.
impl.create_dict()
return self在初始化中我们看下impl变量，impl变量是一个类对象，该对象就是初始化一个字典类型数据，
通过impl.locallock = RLock()赋值一个递归锁，主要是用来防止死锁现象，然后通过create_dict来生成一个字典，这个字典是跟特定线程绑定的，
至此我们回过头来看最开始的get_redis_connection函数中，cache = caches[alias]代码，拿到一个CacheHandler对象，该对象初始化构建了一个<_thread._local object at 0x052E64B0>的对象，是一个安全的字典数据类型，避免死锁现象，
然后通过cache = _create_cache(alias)去获取一个缓存示例，例如我们分析的redis缓存实例，也是传递alias='default'参数，加载配置中的CACHES配置，最后通过backend = params.pop('BACKEND')找到配置中的字符串路径，然后通过backend_cls(location, params)生成django_redis.cache.RedisCache缓存实例
我们继续回头看get_redis_connection函数中的cache = caches[alias]
cache最终生成的数据格式如下：
object at 0x06A998F0> 在CacheHandler中self._caches如下
<_thread._local object at 0x04E07480>self._caches.caches如下
{'default': object at 0x061498B0>} 最后看get_redis_connection函数中的return cache.client.get_client(write)
在看之前我们先看看django_redis.cache.RedisCache看看是怎样的一个实例对象呢？
我列出来它的一些方法，属性
client
set
incr_version
add
get
delete
get_many
我们先看下__init__，你还记得么，我们在CacheHandler类中通过cache = _create_cache(alias)生成实例，是去配置中加载的，然后在 return backend_cls(location, params)去构造对象的，这里我去输出下传递的参数，也就是配置中的参数
location：redis://127.0.0.1:6379
params：{'OPTIONS': {'CLIENT_CLASS': 'django_redis.client.DefaultClient', 'CONNECTION_POOL_KWARGS': {'max_connections': 100}}}
然后在RedisCache类中的__init__方法中
self._client_cls = load_class(self._client_cls)依然通过import_module方式去加载
我们输出来看下该client：（通过如下代码获取）
def client(self):
"""
Lazy client connection property.
"""
if self._client is None:
self._client = self._client_cls(self._server, self._params, self)
return self._client
输出如下：
default.DefaultClient object at 0x06A61B10> 那好我们看下该类主要代码如下：我们先看初始方法如下：
def __init__(self, server, params, backend):
self._backend = backend
self._server = server
self._params = params
self.reverse_key = get_key_func(params.get("REVERSE_KEY_FUNCTION") or
"django_redis.util.default_reverse_key")
if not self._server:
raise ImproperlyConfigured("Missing connections string")
if not isinstance(self._server, (list, tuple, set)):
self._server = self._server.split(",")
self._clients = [None] * len(self._server)
self._options = params.get("OPTIONS", {})
self._slave_read_only = self._options.get('SLAVE_READ_ONLY', True)
serializer_path = self._options.get("SERIALIZER", "django_redis.serializers.pickle.PickleSerializer")
serializer_cls = load_class(serializer_path)
compressor_path = self._options.get("COMPRESSOR", "django_redis.compressors.identity.IdentityCompressor")
compressor_cls = load_class(compressor_path)
self._serializer = serializer_cls(options=self._options)
self._compressor = compressor_cls(options=self._options)
self.connection_factory = pool.get_connection_factory(options=self._options)初始化方法中的
serializer_path = self._options.get("SERIALIZER", "django_redis.serializers.pickle.PickleSerializer")
serializer_cls = load_class(serializer_path)
compressor_path = self._options.get("COMPRESSOR", "django_redis.compressors.identity.IdentityCompressor")
compressor_cls = load_class(compressor_path)
self.connection_factory = pool.get_connection_factory(options=self._options)serializer_cls从字面意思我们能猜到是进行序列化和返序列化的，利用pickle，代码省略，不难
compressor_cls 用来减压，增加压力的，比如我们配置中的”max_connections”: 100
self.connection_factory这个就是线程池
另外还有一些在用来操作cache中的数据，我就列出一些
set
incr_version
add
get
persist
delete
delete_many
等等我们继续回头看下get_redis_connection函数中的return cache.client.get_client(write)
def get_client(self, write=True, tried=(), show_index=False):
"""
Method used for obtain a raw redis client.
This function is used by almost all cache backend
operations for obtain a native redis client/connection
instance.
"""
index = self.get_next_client_index(write=write, tried=tried or [])
if self._clients[index] is None:
self._clients[index] = self.connect(index)
if show_index:
return self._clients[index], index
else:
return self._clients[index]cache.client.get_client(write)输出如下：
StrictRedis<ConnectionPool<Connection<host=127.0.0.1,port=6379,db=0>>>StrictRedis这个其实是原生redis里面的，不是django_redis下的
///////////////////////////////////////////////////////////////////////
解析来我在次贴出代码cache.client.get_client(write)中的get_client方法代码
def get_client(self, write=True, tried=(), show_index=False):
"""
Method used for obtain a raw redis client.
This function is used by almost all cache backend
operations for obtain a native redis client/connection
instance.
"""
index = self.get_next_client_index(write=write, tried=tried or [])
if self._clients[index] is None:
self._clients[index] = self.connect(index)
if show_index:
return self._clients[index], index
else:
return self._clients[index]以上代码块中的self._clients[index] = self.connect(index)，跟踪入下代码
def connect(self, index=0):
return self.connection_factory.connect(self._server[index])继续跟踪到DefaultClient类中的如下connect方法
def connect(self, url):
params = self.make_connection_params(url)
connection = self.get_connection(params)
return connection然而DefaultClient中connect返回了如下代码：
def get_connection(self, params):
pool = self.get_or_create_connection_pool(params)
return self.redis_client_cls(connection_pool=pool, **self.redis_client_cls_kwargs)定位到DefaultClient类中的属性connection_factory
然后就跟踪到如下代码,去加载redis里面的redis.client.StrictRedis
redis_client_cls_path = options.get("REDIS_CLIENT_CLASS",
"redis.client.StrictRedis")
self.redis_client_cls = util.load_class(redis_client_cls_path)///////////////////////////刚才我们没有仔细看如下代码，也就是/////////////////////////
ConnectionFactory类，
我们在DefaultClient初始化中的self.connection_factory = pool.get_connection_factory(options=self._options)
def get_connection_factory(path=None, options=None):
if path is None:
path = getattr(settings, "DJANGO_REDIS_CONNECTION_FACTORY",
"django_redis.pool.ConnectionFactory")
cls = util.load_class(path)
return cls(options or {})
该类ConnectionFactory代码百余行，这是django_redis库，自己的简单封装，本身并没有去实现复杂的线程业务，而是通过调用redis下的接口方法去实现，
例如在ConnectionFactory类初始化方法__init__中的如下代码：
pool_cls_path = options.get("CONNECTION_POOL_CLASS",
"redis.connection.ConnectionPool")
self.pool_cls_kwargs = options.get("CONNECTION_POOL_KWARGS", {})
其他省略一个是获取我们配置的参数"max_connections": 100，一个是获取原生的ConnectionPool线程池
之前代码看过get_connection，我们在看看到底是如何链接的？代码我在次贴出来
def get_connection(self, params):
pool = self.get_or_create_connection_pool(params)
return self.redis_client_cls(connection_pool=pool, **self.redis_client_cls_kwargs)
跟踪代码get_or_create_connection_pool方法，讲之前的参数传递进去，也就是配置文件里面的
def get_or_create_connection_pool(self, params):
key = params["url"]
if key not in self._pools:
self._pools[key] = self.get_connection_pool(params)
return self._pools[key]然后通过get_connection_pool返回连接池
def get_connection_pool(self, params):
"""
Given a connection parameters, return a new
connection pool for them.
Overwrite this method if you want a custom
behavior on creating connection pool.
"""
cp_params = dict(params)
cp_params.update(self.pool_cls_kwargs)
pool = self.pool_cls.from_url(**cp_params)
if pool.connection_kwargs.get("password", None) is None:
pool.connection_kwargs["password"] = params.get("password", None)
pool.reset()
return pool最后在客户端就去执行诸如下面的操作，去执行client里面的操作
conn.hset("safly","k1","v1")
conn.hget("safly","k1")操作代码如下
def execute_command(self, *args, **options):
"Execute a command and return a parsed response"
pool = self.connection_pool
command_name = args[0]
connection = pool.get_connection(command_name, **options)
try:
connection.send_command(*args)
return self.parse_response(connection, command_name, **options)
except (ConnectionError, TimeoutError) as e:
connection.disconnect()
if not connection.retry_on_timeout and isinstance(e, TimeoutError):
raise
connection.send_command(*args)
return self.parse_response(connection, command_name, **options)
finally:
pool.release(connection)去获取连接池pool，获取pool.get_connection链接，
然后就去执行redis里面的connection.send_command(*args)方法，最后放到缓存，然后django_redis在去缓存拿即可