GlusterFS之内存池（mem-pool）实现原理及代码详解

最近一直在研究glusterfs的源代码，自己也在上面做了一些小的改动。我最开始研究的是3.2.5这个版本，因为据同行和网上资料显示这个版本目前是最稳定的版本。glusterfs实现比较复杂，具体的设计思想和架构就不详细介绍了，网上有这方面的资料（CSDN博客里面就有很好介绍的文章）。

        研究开源系统的一个好处就是可以充分了解它的实现，如果是看这方面的论文只能了解一些原理性的东西，但是我们真正做项目还需要实际的实现。很多开源系统可能本身不一定就很适合你的系统，但是如果可以改造那么利用它来改造也是很值得划算的。研究开源系统最大的好处就是学习它的优秀的代码，今天这篇博文就是要分享glusterfs里面使用的内存池技术。

        glusterfs实现内存池技术的源文件和头文件分别是mem-pool.c和mem-pool.h，首先看看头文件中内存池对象结构体的定义如下：

[cpp] view plaincopyprint?

struct mem_pool {
        struct list_head  list;//用于管理内存池的标准双向链表
        int               hot_count;//正在使用的内存数量计数
        int               cold_count;//未使用的内存数量计数
        gf_lock_t         lock;
        unsigned long     padded_sizeof_type;//带有填充
        void             *pool;//内存池开始地址
        void             *pool_end;//内存池结束地址
        int               real_sizeof_type;//内存池存放对象的真实大小
        uint64_t          alloc_count;//采用alloc分配的次数
        uint64_t          pool_misses;//内出池缺少次数
        int               max_alloc;//采用alloc分配的最大次数
        int               curr_stdalloc;
        int               max_stdalloc;
        char             *name;
        struct list_head  global_list;//加入到全局的内存池链表
};

然后我们在来分析几个重要的实现函数，第一个函数就是mem_pool_new_fn，它会新建一个内存池对象，然后按照传递进来的内存的大小和个数分配内存，还要加上一些额外存储内容的内存容量，如存放链表指针的因为这些内存池对象本身是通过通用链表来管理的，还有如标识内存是否在被使用的一个标志等。具体看下面代码的实现，关键代码都有注释：

[cpp] view plaincopyprint?

struct mem_pool *
mem_pool_new_fn (unsigned long sizeof_type,
                 unsigned long count, char *name)
{
        struct mem_pool  *mem_pool = NULL;
        unsigned long     padded_sizeof_type = 0;
        void             *pool = NULL;
        int               i = 0;
        int               ret = 0;
        struct list_head *list = NULL;
        jdfs_ctx_t  *ctx = NULL;

        if (!sizeof_type || !count) {
                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                return NULL;
        }
        padded_sizeof_type = sizeof_type + GF_MEM_POOL_PAD_BOUNDARY;//计算大小：对象本身所占内存+链表头+内存池指针+int内存大小（存放in_use变量）

        mem_pool = GF_CALLOC (sizeof (*mem_pool), 1, gf_common_mt_mem_pool);
        if (!mem_pool)
                return NULL;

        ret = gf_asprintf (&mem_pool->name, "%s:%s", THIS->name, name);//哪一个xlator分配什么名字内存
        if (ret < 0)
                return NULL;

        if (!mem_pool->name) {
                GF_FREE (mem_pool);
                return NULL;
        }

        LOCK_INIT (&mem_pool->lock);
        INIT_LIST_HEAD (&mem_pool->list);
        INIT_LIST_HEAD (&mem_pool->global_list);

        mem_pool->padded_sizeof_type = padded_sizeof_type;//总的对齐内存大小
        mem_pool->cold_count = count;//数量：刚开始都是冷的（未使用的）
        mem_pool->real_sizeof_type = sizeof_type;//使用内存池对象的真实内存大小

        pool = GF_CALLOC (count, padded_sizeof_type, gf_common_mt_long);//分配count个padded_sizeof_type大小的内存
        if (!pool) {
                GF_FREE (mem_pool->name);
                GF_FREE (mem_pool);
                return NULL;
        }

        for (i = 0; i < count; i++) {
                list = pool + (i * (padded_sizeof_type));//分配每一个内存对象大小到链表
                INIT_LIST_HEAD (list);
                list_add_tail (list, &mem_pool->list);//加入到内存池的链表中去
        }

        mem_pool->pool = pool;//记录分配的内存区域
        mem_pool->pool_end = pool + (count * (padded_sizeof_type));//内存分配结束的地址

        /* add this pool to the global list */
        ctx = jdfs_ctx_get ();
        if (!ctx)
                goto out;

        list_add (&mem_pool->global_list, &ctx->mempool_list);//加入全局的内存池链表

out:
        return mem_pool;
}

如果我们需要使用这种内存池中的内存，那么就从内存池中拿出一个对象（不同对象需要不同的内存池对象保存，每一个内存池对象只保存一种对象的内存结构）的内存，代码实现和注释如下：

[cpp] view plaincopyprint?

void *
mem_get (struct mem_pool *mem_pool)
{
        struct list_head *list = NULL;
        void             *ptr = NULL;
        int             *in_use = NULL;
        struct mem_pool **pool_ptr = NULL;

        if (!mem_pool) {
                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                return NULL;
        }

        LOCK (&mem_pool->lock);
        {
                mem_pool->alloc_count++;
                if (mem_pool->cold_count) {//内存池中是否还有未使用的内存对象
                        list = mem_pool->list.next;//取出一个
                        list_del (list);//从链表中脱链

                        mem_pool->hot_count++;
                        mem_pool->cold_count--;

                        if (mem_pool->max_alloc < mem_pool->hot_count)//最大以分配的内存是否小于正在使用的内存数量
                                mem_pool->max_alloc = mem_pool->hot_count;

                        ptr = list;
                        in_use = (ptr + GF_MEM_POOL_LIST_BOUNDARY +
                                  GF_MEM_POOL_PTR);//分配内存池对象的时候分配了这个区域来保存次块内存是否在使用
                        *in_use = 1;//标记次块内存正在使用

                        goto fwd_addr_out;
                }

                /* This is a problem area. If we've run out of
                 * chunks in our slab above, we need to allocate
                 * enough memory to service this request.
                 * The problem is, these individual chunks will fail
                 * the first address range check in __is_member. Now, since
                 * we're not allocating a full second slab, we wont have
                 * enough info perform the range check in __is_member.
                 *
                 * I am working around this by performing a regular allocation
                 * , just the way the caller would've done when not using the
                 * mem-pool. That also means, we're not padding the size with
                 * the list_head structure because, this will not be added to
                 * the list of chunks that belong to the mem-pool allocated
                 * initially.
                 *
                 * This is the best we can do without adding functionality for
                 * managing multiple slabs. That does not interest us at present
                 * because it is too much work knowing that a better slab
                 * allocator is coming RSN.
                 */
                mem_pool->pool_misses++;//内存池缺失计数次数加1
                mem_pool->curr_stdalloc++;//系统标准分配次数加1
                if (mem_pool->max_stdalloc < mem_pool->curr_stdalloc)
                        mem_pool->max_stdalloc = mem_pool->curr_stdalloc;
                ptr = GF_CALLOC (1, mem_pool->padded_sizeof_type,
                                 gf_common_mt_mem_pool);//分配一个内存池对象
                gf_log_callingfn ("mem-pool", GF_LOG_DEBUG, "Mem pool is full. "
                                  "Callocing mem");

                /* Memory coming from the heap need not be transformed from a
                 * chunkhead to a usable pointer since it is not coming from
                 * the pool.
                 */
        }
fwd_addr_out:
        pool_ptr = mem_pool_from_ptr (ptr);
        *pool_ptr = (struct mem_pool *)mem_pool;
        ptr = mem_pool_chunkhead2ptr (ptr);//得到真正开始的内存
        UNLOCK (&mem_pool->lock);

        return ptr;
}

当我们使用完一个内存池中的内存结构以后就需要还给内存池以便被以后的程序使用，达到循环使用的目的。但是在归还以前我们首先需要判断是不是内存池对象的一个成员，判断的结果有三种，分别是：是，不是和错误情况（就是它在内存池的内存范围以内，但是不符合内存池对象的大小），实现如下：

[cpp] view plaincopyprint?

static int
__is_member (struct mem_pool *pool, void *ptr)//判断ptr指向的内存是否是pool的成员
{
        if (!pool || !ptr) {
                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                return -1;
        }

        if (ptr < pool->pool || ptr >= pool->pool_end)//ptr如果不再pool开始到结束的范围内就不是
                return 0;

        if ((mem_pool_ptr2chunkhead (ptr) - pool->pool)
            % pool->padded_sizeof_type)//判断是否是一个符合内存块大小的内存对象
                return -1;

        return 1;
}

那么根据上面函数判断的结果，放入内存对象到内存池对象的函数就会做相应的处理，具体代码如下：

[cpp] view plaincopyprint?

void
mem_put (void *ptr)//将ptr放回到内存池中去
{
        struct list_head *list = NULL;
        int    *in_use = NULL;
        void   *head = NULL;
        struct mem_pool **tmp = NULL;
        struct mem_pool *pool = NULL;

        if (!ptr) {
                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                return;
        }

        list = head = mem_pool_ptr2chunkhead (ptr);//得到链表指针
        tmp = mem_pool_from_ptr (head);
        if (!tmp) {
                gf_log_callingfn ("mem-pool", GF_LOG_ERROR,
                                  "ptr header is corrupted");
                return;
        }

        pool = *tmp;
        if (!pool) {
                gf_log_callingfn ("mem-pool", GF_LOG_ERROR,
                                  "mem-pool ptr is NULL");
                return;
        }
        LOCK (&pool->lock);
        {

                switch (__is_member (pool, ptr))
                {
                case 1://是内存池中的内存
                        in_use = (head + GF_MEM_POOL_LIST_BOUNDARY +
                                  GF_MEM_POOL_PTR);//得到是否正在使用变量
                        if (!is_mem_chunk_in_use(in_use)) {//正在使用就暂时不回收
                                gf_log_callingfn ("mem-pool", GF_LOG_CRITICAL,
                                                  "mem_put called on freed ptr %p of mem "
                                                  "pool %p", ptr, pool);
                                break;
                        }
                        pool->hot_count--;
                        pool->cold_count++;
                        *in_use = 0;
                        list_add (list, &pool->list);//加入到内存池中的链表
                        break;
                case -1://错误就终止程序
                        /* For some reason, the address given is within
                         * the address range of the mem-pool but does not align
                         * with the expected start of a chunk that includes
                         * the list headers also. Sounds like a problem in
                         * layers of clouds up above us. ;)
                         */
                        abort ();
                        break;
                case 0://不是内存池中的内存直接释放掉
                        /* The address is outside the range of the mem-pool. We
                         * assume here that this address was allocated at a
                         * point when the mem-pool was out of chunks in mem_get
                         * or the programmer has made a mistake by calling the
                         * wrong de-allocation interface. We do
                         * not have enough info to distinguish between the two
                         * situations.
                         */
                        pool->curr_stdalloc--;//系统分配次数减1
                        GF_FREE (list);
                        break;
                default:
                        /* log error */
                        break;
                }
        }
        UNLOCK (&pool->lock);
}

除了上面介绍的，当然还有销毁内存池的功能函数mem_pool_destroy，辅助分配系统内存的一些封装函数等；另外还有一个对于调试有用的功能，那就是记录分配内存的信息，这些东西相对简单，可以自己直接看源码理解。