Memecache内存管理是采取预分配的形式,避免优先避免频繁malloc和free带来的内存碎片。Memcache是驻欧洲断裂slab的形式来管理内存:每个slab页默认大小为1M,不同的slab里会1到n个分割成大小不同chunk内存块,chunk是实际存储数据的最小单元。存储数据的时候根据数据的大小选择从相应的slab分配空闲chunk来存储。
memcached -u root -f 2 -vv;可以查看Memcache的slabs分配情况
1:Memecache内存管理的代码都在slabs.c里面,slabclass_t是内存管理结构体,memcache本上维护了一个slabclass_t数组。
typedef struct {
unsigned int size; /* 数据存储单元item的大小*/
unsigned int perslab; /* 数据存储单元item的数量 */
void **slots; /* 空闲item链表*/
unsigned int sl_total; /* 已经分配的空闲item数量 */
unsigned int sl_curr; /* 空闲item的数量(当前空闲item的位置) */
void *end_page_ptr; /* 最后一个页面中空闲item的位置 */
unsigned int end_page_free; /* 最后一个页面item的数量 */
unsigned int slabs; /* slab数量 */
void **slab_list; /* slab指针数组 */
unsigned int list_size; /* 已经分配slab指针数组大小*/
unsigned int killing; /* index+1 of dying slab, or zero if none */
} slabclass_t;
2:memcache启动时会调用slabs_init初始化。limit是Memcache最大使用内存的大小,factor是slab_class item大小的增长因子,可以通过memcache -n参数调整。memcache还会调用slabs_preallocate为每个slab_class预分配一个默认1M的slab,以避免在memcache开始阶段频繁的新分配slab。
void slabs_init(size_t limit, double factor) {
int i = POWER_SMALLEST - 1;
unsigned int size = sizeof(item) + settings.chunk_size;
/* Factor of 2.0 means use the default memcached behavior */
if (factor == 2.0 && size < 128)
size = 128;
mem_limit = limit;
memset(slabclass, 0, sizeof(slabclass));
while (++i < POWER_LARGEST && size <= POWER_BLOCK / 2) {
/* Make sure items are always n-byte aligned */
if (size % CHUNK_ALIGN_BYTES)
size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES); //内存对齐
slabclass[i].size = size;
slabclass[i].perslab = POWER_BLOCK / slabclass[i].size;
size *= factor;//chunk大小按增长因子增长
if (settings.verbose > 1) {
fprintf(stderr, "slab class %3d: chunk size %6d perslab %5d\n",
i, slabclass[i].size, slabclass[i].perslab);
}
}
power_largest = i;
slabclass[power_largest].size = POWER_BLOCK;
slabclass[power_largest].perslab = 1;
#ifndef DONT_PREALLOC_SLABS
{
char *pre_alloc = getenv("T_MEMD_SLABS_ALLOC");
if (!pre_alloc || atoi(pre_alloc)) {
slabs_preallocate(limit / POWER_BLOCK);
}
}
#endif
}
void slabs_preallocate (unsigned int maxslabs) {
int i;
unsigned int prealloc = 0;
/* pre-allocate a 1MB slab in every size class so people don't get
confused by non-intuitive "SERVER_ERROR out of memory"
messages. this is the most common question on the mailing
list. if you really don't want this, you can rebuild without
these three lines. */
for(i=POWER_SMALLEST; i<=POWER_LARGEST; i++) {
if (++prealloc > maxslabs)
return;
slabs_newslab(i);
}
}
void *slabs_alloc(size_t size) {
slabclass_t *p;
unsigned char id = slabs_clsid(size);
if (id < POWER_SMALLEST || id > power_largest)
return 0;
p = &slabclass[id];
assert(p->sl_curr == 0 || ((item*)p->slots[p->sl_curr-1])->slabs_clsid == 0);
#ifdef USE_SYSTEM_MALLOC
if (mem_limit && mem_malloced + size > mem_limit)
return 0;
mem_malloced += size;
return malloc(size);
#endif
/* fail unless we have space at the end of a recently allocated page,
we have something on our freelist, or we could allocate a new page */
if (! (p->end_page_ptr || p->sl_curr || slabs_newslab(id)))
return 0;
/* return off our freelist, if we have one */
if (p->sl_curr)
return p->slots[--p->sl_curr];
/* if we recently allocated a whole page, return from that */
if (p->end_page_ptr) {
void *ptr = p->end_page_ptr;
if (--p->end_page_free) {
p->end_page_ptr += p->size;
} else {
p->end_page_ptr = 0;
}
return ptr;
}
return 0; /* shouldn't ever get here */
}
int slabs_newslab(unsigned int id) {
slabclass_t *p = &slabclass[id];
#ifdef ALLOW_SLABS_REASSIGN
int len = POWER_BLOCK;
#else
int len = p->size * p->perslab;
#endif
char *ptr;
if (mem_limit && mem_malloced + len > mem_limit && p->slabs > 0)
return 0;
if (! grow_slab_list(id)) return 0;
ptr = malloc(len);
if (ptr == 0) return 0;
memset(ptr, 0, len);
p->end_page_ptr = ptr;
p->end_page_free = p->perslab;
p->slab_list[p->slabs++] = ptr;
mem_malloced += len;
return 1;
}
void slabs_free(void *ptr, size_t size) {
unsigned char id = slabs_clsid(size);
slabclass_t *p;
assert(((item *)ptr)->slabs_clsid==0);
assert(id >= POWER_SMALLEST && id <= power_largest);
if (id < POWER_SMALLEST || id > power_largest)
return;
p = &slabclass[id];
#ifdef USE_SYSTEM_MALLOC
mem_malloced -= size;
free(ptr);
return;
#endif
if (p->sl_curr == p->sl_total) { /* need more space on the free list */
int new_size = p->sl_total ? p->sl_total*2 : 16; /* 16 is arbitrary */
void **new_slots = realloc(p->slots, new_size*sizeof(void *));
if (new_slots == 0)
return;
p->slots = new_slots;
p->sl_total = new_size;
}
p->slots[p->sl_curr++] = ptr;
return;
}
5:memcache这种内存管理方式也会造成一定的内存浪费:
1):当数据大小 2):现默认一个slabclass_t的大小是1M,如果slabclass_t的大小不是chunk的整数倍,则余下的内存空间将被浪费。 3):Memcache是默认不开启slab resign的,也就是已分配给一个slab,即使该slab不用,也不会分配给其它slab。