目录
- 主要函数
- void *zmalloc(size_t size)
- void *zcalloc(size_t size)
- void zrealloc(void ptr, size_t size)
- void zfree(void *ptr)
- char zstrdup(const char s)
- size_t zmalloc_used_memory(void)
- void zmalloc_set_oom_handler(void (*oom_handler)(size_t))
- size_t zmalloc_get_rss(void)
- int zmalloc_get_allocator_info(size_t allocated, size_t active, size_t *resident)
- size_t zmalloc_get_smap_bytes_by_field(char *field, long pid)
- size_t zmalloc_get_private_dirty(long pid)
- size_t zmalloc_get_memory_size(void)
- 总结
主要函数
void *zmalloc(size_t size);
void *zcalloc(size_t size);
void zrealloc(void ptr, size_t size);
void zfree(void *ptr);
char zstrdup(const char s);
size_t zmalloc_used_memory(void);
void zmalloc_set_oom_handler(void (*oom_handler)(size_t));
size_t zmalloc_get_rss(void);
int zmalloc_get_allocator_info(size_t allocated, size_t active, size_t *resident);
size_t zmalloc_get_private_dirty(long pid);
size_t zmalloc_get_smap_bytes_by_field(char *field, long pid);
size_t zmalloc_get_memory_size(void);
void zlibc_free(void *ptr);
void *zmalloc(size_t size)
void *zmalloc(size_t size) {
void *ptr = malloc(size+PREFIX_SIZE);
if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
update_zmalloc_stat_alloc(zmalloc_size(ptr));
return ptr;
#else
*((size_t*)ptr) = size;
update_zmalloc_stat_alloc(size+PREFIX_SIZE);
return (char*)ptr+PREFIX_SIZE;
#endif
}参数size是需要分配的空间大小。事实上我们需要分配的空间大小为size+PREFIX_SIZE。PREFIX_SIZE是根据平台的不同和HAVE_MALLOC_SIZE宏定义控制的。如果malloc()函数调用失败，就会调用zmalloc_oom_handler()函数来打印异常，并且会终止函数，zmalloc_oom_handler其实是一个函数指针，真正调用的函数是zmalloc_default_oom(),zmalloc_default_oom()函数源码如下:
static void zmalloc_default_oom(size_t size) {
fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",
size);
fflush(stderr);
abort();
}
static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;内存分配成功之后，会依据HAVE_MALLOC_SIZE的控制对前八个字节操作，用以记录分配内存的长度，会在update_zmalloc_stat_alloc()宏定义函数中更新used_memory这个静态变量的值，update_zmalloc_stat_alloc()源码如下:
#define update_zmalloc_stat_alloc(__n) do { \
size_t _n = (__n); \
if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
atomicIncr(used_memory,__n); \
} while(0)if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1));这一行是为了将不为sizeof(long)的_n对sizeof(long)补齐
atomicIncr(used_memory,__n);会调用__atomic_add_fetch(&var,(count),__ATOMIC_RELAXED);用于保证更新used_memory变量的操作是一个原子操作。
void *zcalloc(size_t size)
void *zcalloc(size_t size) {
void *ptr = calloc(1, size+PREFIX_SIZE);
if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
update_zmalloc_stat_alloc(zmalloc_size(ptr));
return ptr;
#else
*((size_t*)ptr) = size;
update_zmalloc_stat_alloc(size+PREFIX_SIZE);
return (char*)ptr+PREFIX_SIZE;
#endif
}zcalloc函数和zmalloc函数处理的思路很是相似，就不做太多的解释了
void zrealloc(void ptr, size_t size)
void *zrealloc(void *ptr, size_t size) {
// 如果没有定义HAVE_MALLOC_SIZE，就说明PREFIX_SIZE宏定义不为0，那么ptr并不是该段内存真正的开始地址
#ifndef HAVE_MALLOC_SIZE
void *realptr;
#endif
size_t oldsize;
void *newptr;
if (ptr == NULL) return zmalloc(size);
// 根据HAVE_MALLOC_SIZE宏定义，oldsize,newptr获取方式不一样，以及更新used_memory的细节
#ifdef HAVE_MALLOC_SIZE
oldsize = zmalloc_size(ptr);
newptr = realloc(ptr,size);
if (!newptr) zmalloc_oom_handler(size);
update_zmalloc_stat_free(oldsize);
update_zmalloc_stat_alloc(zmalloc_size(newptr));
return newptr;
#else
realptr = (char*)ptr-PREFIX_SIZE;
oldsize = *((size_t*)realptr);
newptr = realloc(realptr,size+PREFIX_SIZE);
if (!newptr) zmalloc_oom_handler(size);
*((size_t*)newptr) = size;
update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
update_zmalloc_stat_alloc(size+PREFIX_SIZE);
return (char*)newptr+PREFIX_SIZE;
#endif
}大致思路就是根据新的size进行重新分配内存，并且对used_memory变量进行更新。只不过获取原内存大小方式不一样，根据HAVE_MALLOC_SIZE进行区分。
void zfree(void *ptr)
void zfree(void *ptr) {
#ifndef HAVE_MALLOC_SIZE
void *realptr;
size_t oldsize;
#endif
if (ptr == NULL) return;
#ifdef HAVE_MALLOC_SIZE
update_zmalloc_stat_free(zmalloc_size(ptr));
free(ptr);
#else
realptr = (char*)ptr-PREFIX_SIZE;
oldsize = *((size_t*)realptr);
update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
free(realptr);
#endif
}其实zfree函数和zrealloc函数做法差不到太多，都是对oldsize和realptr对HAVE_MALLOC_SIZE有无声明分别进行操作。
char zstrdup(const char s)
char *zstrdup(const char *s) {
size_t l = strlen(s)+1;
char *p = zmalloc(l);
memcpy(p,s,l);
return p;
}该函数是创建一个字符串副本
size_t zmalloc_used_memory(void)
size_t zmalloc_used_memory(void) {
size_t um;
atomicGet(used_memory,um);
return um;
}获取used_memory变量的值，主要保证原子操作(在atomicGet(used_memory,um);中保证)
void zmalloc_set_oom_handler(void (*oom_handler)(size_t))
void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {
zmalloc_oom_handler = oom_handler;
}主要用来设置内存分配失败处理函数指针zmalloc_oom_handler的值
size_t zmalloc_get_rss(void)
size_t zmalloc_get_rss(void) {
int page = sysconf(_SC_PAGESIZE);
size_t rss;
char buf[4096];
char filename[256];
int fd, count;
char *p, *x;
snprintf(filename,256,"/proc/%d/stat",getpid());
if ((fd = open(filename,O_RDONLY)) == -1) return 0;
if (read(fd,buf,4096) <= 0) {
close(fd);
return 0;
}
close(fd);
p = buf;
count = 23; /* RSS is the 24th field in /proc//stat */
while(p && count--) {
p = strchr(p,' ');
if (p) p++;
}
if (!p) return 0;
x = strchr(p,' ');
if (!x) return 0;
*x = '\0';
rss = strtoll(p,NULL,10);
rss *= page;
return rss;
} 返回驻留集大小
int zmalloc_get_allocator_info(size_t allocated, size_t active, size_t *resident)
#if defined(USE_JEMALLOC)
int zmalloc_get_allocator_info(size_t *allocated,
size_t *active,
size_t *resident) {
uint64_t epoch = 1;
size_t sz;
*allocated = *resident = *active = 0;
/* Update the statistics cached by mallctl. */
sz = sizeof(epoch);
je_mallctl("epoch", &epoch, &sz, &epoch, sz);
sz = sizeof(size_t);
/* Unlike RSS, this does not include RSS from shared libraries and other non
* heap mappings. */
je_mallctl("stats.resident", resident, &sz, NULL, 0);
/* Unlike resident, this doesn't not include the pages jemalloc reserves
* for re-use (purge will clean that). */
je_mallctl("stats.active", active, &sz, NULL, 0);
/* Unlike zmalloc_used_memory, this matches the stats.resident by taking
* into account all allocations done by this process (not only zmalloc). */
je_mallctl("stats.allocated", allocated, &sz, NULL, 0);
return 1;
}
#else
int zmalloc_get_allocator_info(size_t *allocated,
size_t *active,
size_t *resident) {
*allocated = *resident = *active = 0;
return 1;
}
#endif获取分配器的信息，主要在使用jemalloc前提下使用，获取jemalloc分配的信息，详细信息可在http://jemalloc.net/jemalloc.3.html查阅
size_t zmalloc_get_smap_bytes_by_field(char *field, long pid)
#if defined(HAVE_PROC_SMAPS)
size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
char line[1024];
size_t bytes = 0;
int flen = strlen(field);
FILE *fp;
if (pid == -1) {
// /proc/pid/smaps反应了运行时的进程的内存影响，系统的运行时库(so)，堆，栈信息均可在其中看到。
fp = fopen("/proc/self/smaps","r");
} else {
char filename[128];
snprintf(filename,sizeof(filename),"/proc/%ld/smaps",pid);
fp = fopen(filename,"r");
}
if (!fp) return 0;
while(fgets(line,sizeof(line),fp) != NULL) {
if (strncmp(line,field,flen) == 0) {
char *p = strchr(line,'k');
if (p) {
*p = '\0';
bytes += strtol(line+flen,NULL,10) * 1024;
}
}
}
fclose(fp);
return bytes;
}
#else
size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
((void) field);
((void) pid);
return 0;
}
#endif获取/proc/pid/smaps中某一个field的字节大小
size_t zmalloc_get_private_dirty(long pid)
size_t zmalloc_get_private_dirty(long pid) {
return zmalloc_get_smap_bytes_by_field("Private_Dirty:",pid);
}获取Rss中已改写的私有页面页面大小
size_t zmalloc_get_memory_size(void)
size_t zmalloc_get_memory_size(void) {
#if defined(__unix__) || defined(__unix) || defined(unix) || \
(defined(__APPLE__) && defined(__MACH__))
#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))
int mib[2];
mib[0] = CTL_HW;
#if defined(HW_MEMSIZE)
mib[1] = HW_MEMSIZE; /* OSX. --------------------- */
#elif defined(HW_PHYSMEM64)
mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */
#endif
int64_t size = 0; /* 64-bit */
size_t len = sizeof(size);
if (sysctl( mib, 2, &size, &len, NULL, 0) == 0)
return (size_t)size;
return 0L; /* Failed? */
#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
/* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
return (size_t)sysconf(_SC_PHYS_PAGES) * (size_t)sysconf(_SC_PAGESIZE);
#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
/* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
int mib[2];
mib[0] = CTL_HW;
#if defined(HW_REALMEM)
mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */
#elif defined(HW_PHYSMEM)
mib[1] = HW_PHYSMEM; /* Others. ------------------ */
#endif
unsigned int size = 0; /* 32-bit */
size_t len = sizeof(size);
if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)
return (size_t)size;
return 0L; /* Failed? */
#else
return 0L; /* Unknown method to get the data. */
#endif
#else
return 0L; /* Unknown OS. */
#endif
}获取物理内存的字节数
总结
看了redis内存分配的源码后，其实没有相信中的那么难以理解，或许只是心理上的作用，当然也说明redis源码写得真的是好，让我这种渣渣都能轻而易举的看懂，并且注释也很少，这里的函数几乎都是对glibc的malloc中的函数进行了一层包装，并且维护了一个叫做used_memory的全局变量，并且每一次对全局变量的操作都是原子操作。也对一些常用的函数进行了封装，例如：获取rss的大小，获取/proc/pid/smaps文件中某一field占用字节数的大小，获取物理内存字节数等等，总的来说，收益匪浅，没想到内存操作可以做到这样简单。