ptmalloc堆实现
1 概述
在 glibc-2.3.x. 之后，glibc 中集成了ptmalloc2。
可以下载glibc源码查看ptmalloc
http://ftp.gnu.org/gnu/glibc/
查看glibc版本
millionsky@ubuntu-16:~/tmp$ ldd --version ldd (Ubuntu GLIBC 2.23-0ubuntu9) 2.23 |
这里主要参考：
https://ctf-wiki.github.io/ctf-wiki/pwn/heap
本文参考的glibc源码是glibc-2.25.tar.xz
理解ptmalloc堆最后的办法是查看相关资料后看源码。word文档拷贝的时候颜色丢失了，格式也有丢失。先这样。
2 API
2.1 其它
2.1.1 catomic_compare_and_exchange_val_acq
如果*MEM等于OLDVAL，则将*MEM存储为NEWVAL，返回OLDVAL；
/* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
Return the old *MEM value. */
#ifndef catomic_compare_and_exchange_val_acq
# ifdef __arch_c_compare_and_exchange_val_32_acq
# define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
__atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \
mem, newval, oldval)
# else
# define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
atomic_compare_and_exchange_val_acq (mem, newval, oldval)
# endif
#endif
2.1.2 catomic_compare_and_exchange_val_rel
如果*MEM等于OLDVAL，则将*MEM存储为NEWVAL，返回OLDVAL；
#ifndef catomic_compare_and_exchange_val_rel
# ifndef atomic_compare_and_exchange_val_rel
# define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
catomic_compare_and_exchange_val_acq (mem, newval, oldval)
# else
# define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
atomic_compare_and_exchange_val_rel (mem, newval, oldval)
# endif
#endif
2.1.3 Perturb_byte
l perturb_byte：一个字节，用于初始化分配的内存，出于调试的目的。
l __libc_mallopt(int param_number, int value)：设置perturb_byte
l alloc_perturb (char *p, size_t n)：内存初始化为perturb_byte^0xff
l free_perturb (char *p, size_t n)：内存初始化为perturb_byte
l do_set_perturb_byte (int32_t value)：设置perturb_byte
static int perturb_byte;
static void
alloc_perturb (char *p, size_t n)
{
if (__glibc_unlikely (perturb_byte))
memset (p, perturb_byte ^ 0xff, n);
}
static void
free_perturb (char *p, size_t n)
{
if (__glibc_unlikely (perturb_byte))
memset (p, perturb_byte, n);
}
static inline int
__always_inline
do_set_perturb_byte (int32_t value)
{
LIBC_PROBE (memory_mallopt_perturb, 2, value, perturb_byte);
perturb_byte = value;
return 1;
}
int
__libc_mallopt (int param_number, int value){
//......
switch (param_number)
{
case M_PERTURB:
do_set_perturb_byte (value);
break;
}
//......
}
2.2 malloc_init_state
l 初始化arena header(malloc_state)结构：
初始化bins数组，构造bin双链表；
设置NONCONTIGUOUS_BIT标记；
设置fastbin用户数据的最大值：global_max_fast；
设置FASTCHUNKS_BIT标记；
初始化top chunk；
static void
malloc_init_state (mstate av)
{
int i;
mbinptr bin;
/* Establish circular links for normal bins */
for (i = 1; i < NBINS; ++i)
{
bin = bin_at (av, i);
bin->fd = bin->bk = bin;
}
#if MORECORE_CONTIGUOUS
if (av != &main_arena)
#endif
set_noncontiguous (av);
if (av == &main_arena)
set_max_fast (DEFAULT_MXFAST);
av->flags |= FASTCHUNKS_BIT;
av->top = initial_top (av);
}
2.3 Unlink
unlink 用来将一个双向 bin 链表中的一个 chunk 取出来
参数：
AV：arena header(malloc_state)
P ：将要unlink的chunk
BK：P后面的chunk <--
FD：P前面的chunk -->
3个chunk的顺序
-------->地址增加
BK P FD
具体过程如下：
将chunk从FD/BK链表中摘除；
如果是large chunk，则将chunk从fd_nextsize/bk_nextsize链表中摘除
/* Take a chunk off a bin list */ #define unlink(AV, P, BK, FD) { \ FD = P->fd; \ BK = P->bk; \ // 检查3个chunk的BK/FD链接是否一致 if (__builtin_expect (FD->bk != P || BK->fd != P, 0)) \ malloc_printerr (check_action, "corrupted double-linked list", P, AV); \ else { \ // unlink P FD->bk = BK; \ BK->fd = FD; \ // 如果P的大小不属于small bin if (!in_smallbin_range (chunksize_nomask (P)) \ // P的fd_nextsize字段非空，即位于large bin中 // 这里期望fd_nextsize字段为NULL，即大概率为unsorted bin，小概率为large bin && __builtin_expect (P->fd_nextsize != NULL, 0)) { \ // 检查3个chunk的fd_nextsize/bk_nextsize链接是否一致 if (__builtin_expect (P->fd_nextsize->bk_nextsize != P, 0) \ || __builtin_expect (P->bk_nextsize->fd_nextsize != P, 0)) \ malloc_printerr (check_action, \ "corrupted double-linked list (not small)", \ P, AV); \ // 此时P在fd_nextsize链表中 // 如果FD不在fd_nextsize链表中，则将FD加入链表中 if (FD->fd_nextsize == NULL) { \ // 如果P链接到自身，则令FD也链接到自身 if (P->fd_nextsize == P) \ FD->fd_nextsize = FD->bk_nextsize = FD; \ else { \ // 否则我们需要将 FD 插入到 nextsize 形成的双链表中 // 更新FD的fd_nextsize/bk_nextsize // 更新P->fd_nextsize/P->bk_nextsize FD->fd_nextsize = P->fd_nextsize; \ FD->bk_nextsize = P->bk_nextsize; \ P->fd_nextsize->bk_nextsize = FD; \ P->bk_nextsize->fd_nextsize = FD; \ } \ } else { \ // P和FD都在fd_nextsize链表中，将P从fd_nextsize链表中摘除 P->fd_nextsize->bk_nextsize = P->bk_nextsize; \ P->bk_nextsize->fd_nextsize = P->fd_nextsize; \ } \ } \ } \ } |
1. __builtin_expect
l 函数__builtin_expect()是GCC v2.96版本引入的, 其声明如下:
long __builtin_expect(long exp, long c);
参数
exp 为一个整型表达式, 例如: (ptr != NULL)
c 必须是一个编译期常量, 不能使用变量
返回值
等于 第一个参数 exp
功能
GCC 提供了这个内建函数来帮助程序员处理分支预测.
允许程序员将最有可能执行的分支告诉编译，即exp的值很可能是c；
GCC在编译过程中，会将可能性更大的代码紧跟着前面的代码，从而减少指令跳转带来的性能上的下降, 达到优化程序的目的。（此时CPU流水线预取指令会起作用）
2.4 Malloc
见注释，最终调用的是_int_malloc
strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
void * __libc_malloc (size_t bytes) { mstate ar_ptr; void *victim;
// 检查是否有内存分配钩子，如果有，调用钩子并返回 void *(*hook) (size_t, const void *) = atomic_forced_read (__malloc_hook); if (__builtin_expect (hook != NULL, 0)) return (*hook)(bytes, RETURN_ADDRESS (0));
//接着会寻找一个 arena 来试图分配内存 arena_get (ar_ptr, bytes);
//调用 _int_malloc 函数去申请对应的内存 victim = _int_malloc (ar_ptr, bytes);
//尝试再去寻找一个可用的 arena，并分配内存 /* Retry with another arena only if we were able to find a usable arena before. */ if (!victim && ar_ptr != NULL) { LIBC_PROBE (memory_malloc_retry, 1, bytes); ar_ptr = arena_get_retry (ar_ptr, bytes); victim = _int_malloc (ar_ptr, bytes); }
//unlock if (ar_ptr != NULL) __libc_lock_unlock (ar_ptr->mutex);
//判断目前的状态是否满足以下条件 //要么没有申请到内存 //要么是 mmap 的内存 //要么申请到的内存必须在其所分配的arena中 assert (!victim || chunk_is_mmapped (mem2chunk (victim)) || ar_ptr == arena_for_chunk (mem2chunk (victim))); return victim; } libc_hidden_def (__libc_malloc) |
2.5 _int_malloc
static void *
_int_malloc (mstate av, size_t bytes)
_int_malloc 是内存分配的核心函数，其核心思路有如下
1. Fast bin服务请求
2. Small bin服务请求
3. 如果是large chunk，则合并fast bin
4. 循环处理
1) Unsorted bin服务请求
（遍历的unsorted chunk要么匹配用户请求被返回，要么放入对应的bin中）
2) Large bin服务请求
3) next largest bin服务请求
4) top chunk服务请求
5) 合并fast bin，下次尝试
6) 从系统请求内存
具体过程如下：
1. 没有可用的arena，则使用sysmalloc从mmap获取chunk;
2. chunk大小属于fast bin，尝试从fast bin（单链表）链表头部获取;
3. chunk大小属于small bin，尝试从small bin(环形双链表）链表尾部获取;
4. chunk大小属于large bin，先合并 fast bin以解决分片问题；
5. 循环处理
l 这里使用外循环，是因为我们可能在处理中合并了一个满足要求的chunk，需要重新尝试。最多重新尝试一次，否则我们将扩展内存来服务请求。
l unsorted bin服务请求
Ø last remainder chunk服务请求
（用户请求为 small chunk；
last remainder chunk是 unsorted bin 中的唯一的chunk；
Last remainder大小大于用户请求的大小；）
Ø 严格匹配(exact fit)服务请求
Ø 如果是small chunk，则插入到small bin链表的开头
Ø 如果为large chunk，则按大小降序放入large bin（大小相同时总是插入第二个位置）；同时构造fd_nextsize/bk_nextsize链表；
l Large bin服务large chunk请求
如果找到则切分后将剩余的部分放入unsorted bin
l next largest bin服务请求
在下一个非空bin（bin数组中下一个索引）中获取chunk，切分后返还给用户
l top chunk服务请求
l 合并fast bin，下次尝试
l 从系统请求内存
2.5.1 convert
将用户申请的内存大小转换为内部的chunk大小
checked_request2size (bytes, nb);
2.5.2 arena
如果没有可用的arena，则使用sysmalloc从mmap获取chunk
/* There are no usable arenas. Fall back to sysmalloc to get a chunk from
mmap. */
if (__glibc_unlikely (av == NULL))
{
void *p = sysmalloc (nb, av);
if (p != NULL)
alloc_perturb (p, bytes);
return p;
}
2.5.3 Fastbin
l 如果申请的 chunk 的大小位于 fast bin 范围内，则从对应的fast bin链表的头部中获取并返回free chunk；(fast bin是单链表）
/*
If the size qualifies as a fastbin, first check corresponding bin.
This code is safe to execute even if av is not yet initialized, so we
can try it without checking, which saves some time on this fast path.
*/
if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
{
idx = fastbin_index (nb);
mfastbinptr *fb = &fastbin (av, idx);
mchunkptr pp = *fb;
// 获取fast bin链表中的第一个元素，同时更新fastbinsY中的fast bin头指针
// 如果fast bin链表是空的，则退出循环
do
{
victim = pp;
if (victim == NULL)
break;
}
while ((pp = catomic_compare_and_exchange_val_acq (fb, victim->fd, victim))
!= victim);
if (victim != 0)
{
// 检查取到的 chunk 大小是否与相应的 fastbin 索引一致。
if (__builtin_expect (fastbin_index (chunksize (victim)) != idx, 0))
{
errstr = "malloc(): memory corruption (fast)";
errout:
malloc_printerr (check_action, errstr, chunk2mem (victim), av);
return NULL;
}
// 细致的检查。。只有在 DEBUG 的时候有用
check_remalloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
// 如果设置了perturb_type, 则将获取到的chunk初始化为 perturb_type ^ 0xff
alloc_perturb (p, bytes);
return p;
}
}
2.5.4 Small bin
l 如果申请的 chunk 的大小位于 small bin 范围内（small bin是环形循环双链表），则从链表中取出small bin的最后一个chunk返回给用户；
/*
If a small request, check regular bin. Since these "smallbins"
hold one size each, no searching within bins is necessary.
(For a large request, we need to wait until unsorted chunks are
processed to find best fit. But for small ones, fits are exact
anyway, so we can check now, which is faster.)
*/
if (in_smallbin_range (nb))
{
idx = smallbin_index (nb);
bin = bin_at (av, idx);
// 先执行 victim = last(bin)，获取 small bin 的最后一个 chunk
// 如果 victim = bin ，那说明该 bin 为空。
// 如果不相等，那么会有两种情况
if ((victim = last (bin)) != bin)
{
// 第一种情况，small bin 还没有初始化。
if (victim == 0) /* initialization check */
malloc_consolidate (av);
// 第二种情况，small bin 中存在空闲的 chunk
else
{
// 获取 small bin 中倒数第二个 chunk
bck = victim->bk;
// 获取 small bin 中倒数第二个 chunk
if (__glibc_unlikely (bck->fd != victim))
{
errstr = "malloc(): smallbin double linked list corrupted";
goto errout;
}
set_inuse_bit_at_offset (victim, nb);
// 修改 small bin 链表，将 small bin 的最后一个 chunk 取出来
bin->bk = bck;
bck->fd = bin;
if (av != &main_arena)
set_non_main_arena (victim);
check_malloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
alloc_perturb (p, bytes);
return p;
}
}
}
2.5.5 malloc_consolidate
如果是large chunk，继续之前，先合并 fastbins。这可以解决一般由fastbins带来的分片问题。实际上，程序要么请求small chunk，要么请求large chunk，很少会混合请求。因此合并在大多数程序中不会被调用。
/*
If this is a large request, consolidate fastbins before continuing.
While it might look excessive to kill all fastbins before
even seeing if there is space available, this avoids
fragmentation problems normally associated with fastbins.
Also, in practice, programs tend to have runs of either small or
large requests, but less often mixtures, so consolidation is not
invoked all that often in most programs. And the programs that
it is called frequently in otherwise tend to fragment.
*/
else
{
idx = largebin_index (nb);
if (have_fastchunks (av))
malloc_consolidate (av);
}
2.5.6 大循环
l unsorted bin服务请求
Ø last remainder chunk服务请求
（用户请求为 small chunk；
last remainder chunk是 unsorted bin 中的唯一的chunk；
Last remainder大小大于用户请求的大小；）
Ø 严格匹配(exact fit)服务请求
Ø 如果是small chunk，则插入到small bin链表的开头
Ø 如果为large chunk，则按大小降序放入large bin（大小相同时总是插入第二个位置）；同时构造fd_nextsize/bk_nextsize链表；
l Large bin服务large chunk请求
如果找到则切分后将剩余的部分放入unsorted bin
l next largest bin服务请求
在下一个非空bin（bin数组中下一个索引）中获取chunk，切分后返还给用户
l top chunk服务请求
l fast bin服务请求
l 从系统请求内存
1. 这里使用外循环，是因为我们可能在处理中合并了一个满足要求的chunk，需要重新尝试。最多重新尝试一次，否则我们将扩展内存来服务请求。
for (;; )
{
int iters = 0;
//......
}
2.5.7 大循环-unsorted bin遍历
使用bk遍历unsorted bin
2.5.7.1 获取一个unsorted chunk
// 如果 unsorted bin 不为空
while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av))
{
// victim 为 unsorted bin 的最后一个 chunk
// bck 为 unsorted bin 的倒数第二个 chunk
bck = victim->bk;
if (__builtin_expect (chunksize_nomask (victim) <= 2 * SIZE_SZ, 0)
|| __builtin_expect (chunksize_nomask (victim)
> av->system_mem, 0))
malloc_printerr (check_action, "malloc(): memory corruption",
chunk2mem (victim), av);
// 得到victim对应的chunk大小
size = chunksize (victim);
2.5.7.2 last remainder chunk中获取
如果满足以下条件，则使用last remainder chunk服务请求。这会有助于使得连续的small chunk的请求得到的位置相邻。
1) 用户请求为 small chunk
2) last remainder chunk是 unsorted bin 中的唯一的chunk
3) Last remainder大小大于用户请求的大小
/*
If a small request, try to use last remainder if it is the
only chunk in unsorted bin. This helps promote locality for
runs of consecutive small requests. This is the only
exception to best-fit, and applies only when there is
no exact fit for a small chunk.
*/
//满足3个条件
if (in_smallbin_range (nb) &&
bck == unsorted_chunks (av) &&
victim == av->last_remainder &&
(unsigned long) (size) > (unsigned long) (nb + MINSIZE))
{
//从last remainder chunk中取出用户请求的chunk，更新unsorted bin链表和标志
/* split and reattach remainder */
remainder_size = size - nb;
remainder = chunk_at_offset (victim, nb);
unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder;
av->last_remainder = remainder;
remainder->bk = remainder->fd = unsorted_chunks (av);
if (!in_smallbin_range (remainder_size))
{
remainder->fd_nextsize = NULL;
remainder->bk_nextsize = NULL;
}
set_head (victim, nb | PREV_INUSE |
(av != &main_arena ? NON_MAIN_ARENA : 0));
set_head (remainder, remainder_size | PREV_INUSE);
set_foot (remainder, remainder_size);
check_malloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
alloc_perturb (p, bytes);
return p;
}
2.5.7.3 将Unsorted chunk从链表中移除
/* remove from unsorted list */
unsorted_chunks (av)->bk = bck;
bck->fd = unsorted_chunks (av);
移除后要么被使用，要么返回给用户（exact fit），要么放入相关的bin中；
2.5.7.4 精确匹配
如果取出的unsorted chunk精确匹配，则设置标志后，返回给用户；
/* Take now instead of binning if exact fit */
if (size == nb)
{
set_inuse_bit_at_offset (victim, size);
if (av != &main_arena)
set_non_main_arena (victim);
check_malloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
alloc_perturb (p, bytes);
return p;
}
2.5.7.5 unsorted chunk放入small bin
如果是small chunk，则插入到small bin链表的开头
if (in_smallbin_range (size))
{
victim_index = smallbin_index (size);
bck = bin_at (av, victim_index);
fwd = bck->fd;
}
else ......
// 放到对应的 bin 中，插入victim，构成 bck<-->victim<-->fwd
mark_bin (av, victim_index);
victim->bk = bck;
victim->fd = fwd;
fwd->bk = victim;
bck->fd = victim;
#define MAX_ITERS 10000
if (++iters >= MAX_ITERS)
break;
2.5.7.6 unsorted chunk放入large bin
如果为large chunk，则按大小降序放入large bin（大小相同时总是插入第二个位置）；同时构造fd_nextsize/bk_nextsize链表；
else
{
// large bin范围，bck为头结点，fwd为第一个结点
victim_index = largebin_index (size);
bck = bin_at (av, victim_index);
fwd = bck->fd;
/* maintain large bins in sorted order */
// bin为非空链表
if (fwd != bck)
{
/* Or with inuse bit to speed comparisons */
// 加速比较，链表里的chunk都会设置该位
size |= PREV_INUSE;
/* if smaller than smallest, bypass loop below */
//为什么最后一个chunk属于main arena呢？
assert (chunk_main_arena (bck->bk));
//遍历的chunk比当前最小的还要小
if ((unsigned long) (size) < (unsigned long) chunksize_nomask (bck->bk))
{
//bck为最后一个结点，fw为头结点
fwd = bck;
bck = bck->bk;
//victim插入fd_nextsize/bk_nextsize链表
victim->fd_nextsize = fwd->fd;
victim->bk_nextsize = fwd->fd->bk_nextsize;
fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim;
}
else
{//遍历的chunk比当前最小的要大于等于
//为什么第一个chunk属于main arena呢？
assert (chunk_main_arena (fwd));
// 从链表头部开始找到大小小于等于 victim 的 chunk
while ((unsigned long) size < chunksize_nomask (fwd))
{
fwd = fwd->fd_nextsize;
//为什么每个chunk都属于main arena呢？
assert (chunk_main_arena (fwd));
}
// 如果大小一样，插入第二个位置，nextsize指针没有改变，不需要修改
if ((unsigned long) size == (unsigned long) chunksize_nomask (fwd))
/* Always insert in the second position. */
fwd = fwd->fd;
else
{// 如果大小不一样，则插入fd_nextsize/bk_nextsize双链表
victim->fd_nextsize = fwd;
victim->bk_nextsize = fwd->bk_nextsize;
fwd->bk_nextsize = victim;
victim->bk_nextsize->fd_nextsize = victim;
}
bck = fwd->bk;
}
}
else//空链表
victim->fd_nextsize = victim->bk_nextsize = victim;
}
mark_bin (av, victim_index);
victim->bk = bck;
victim->fd = fwd;
fwd->bk = victim;
bck->fd = victim;
2.5.8 大循环-large bin服务large chunk请求
如果请求的是large chunk，就在large bin 中从小到大进行扫描，找到第一个合适的。如果找到则切分后将剩余的部分放入unsorted bin；
/*
If a large request, scan through the chunks of current bin in
sorted order to find smallest that fits. Use the skip list for this.
*/
if (!in_smallbin_range (nb))
{
bin = bin_at (av, idx);
// 如果对应的 bin 为空或者其中最大的chunk也小于用户的请求，那就跳过
/* skip scan if empty or largest chunk is too small */
if ((victim = first (bin)) != bin && (unsigned long) chunksize_nomask (victim) >= (unsigned long) (nb))
{
victim = victim->bk_nextsize;
// 反向遍历链表，直到找到第一个不小于用户请求大小的chunk
while (((unsigned long) (size = chunksize (victim)) < (unsigned long) (nb)))
victim = victim->bk_nextsize;
/* Avoid removing the first entry for a size so that the skip
list does not have to be rerouted. */
// 如果最终取到的chunk不是该bin中的最后一个chunk，并且该chunk与其前面的chunk（fd）
// 的大小相同，那么我们就取其前面的chunk，这样可以避免调整bk_nextsize,fd_nextsize
// 链表。因为大小相同的chunk只有一个会被串在nextsize链上
if (victim != last (bin) && chunksize_nomask (victim) == chunksize_nomask (victim->fd))
victim = victim->fd;
remainder_size = size - nb;
unlink (av, victim, bck, fwd);
/* Exhaust */
// 剩下的大小不足以当做一个块，设置标志，不切分
if (remainder_size < MINSIZE)
{
set_inuse_bit_at_offset (victim, size);
if (av != &main_arena)
set_non_main_arena (victim);
}
/* Split */
else // 剩下的大小还可以作为一个chunk，进行分割，剩余部分放入unsorted bin
{
remainder = chunk_at_offset (victim, nb);
/* We cannot assume the unsorted list is empty and therefore
have to perform a complete insert here. */
bck = unsorted_chunks (av);
fwd = bck->fd;
if (__glibc_unlikely (fwd->bk != bck))
{
errstr = "malloc(): corrupted unsorted chunks";
goto errout;
}
remainder->bk = bck;
remainder->fd = fwd;
bck->fd = remainder;
fwd->bk = remainder;
if (!in_smallbin_range (remainder_size))
{
remainder->fd_nextsize = NULL;
remainder->bk_nextsize = NULL;
}
set_head (victim, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0));
set_head (remainder, remainder_size | PREV_INUSE);
set_foot (remainder, remainder_size);
}
check_malloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
alloc_perturb (p, bytes);
return p;
}
}
2.5.9 大循环-寻找next largest bin
如果走到了这里，那说明对于用户所需的chunk，不能直接从其对应的bin中获取chunk，此时我们在下一个非空bin（bin数组中下一个索引）中获取chunk，切分后返还给用户。
【准备工作】
// 从下一个更大的bin开始扫描最小匹配的chunk。
// 利用bitmap可以避免扫描空的bin。
/*
Search for a chunk by scanning bins, starting with next largest
bin. This search is strictly by best-fit; i.e., the smallest
(with ties going to approximately the least recently used) chunk
that fits is selected.
The bitmap avoids needing to check that most blocks are nonempty.
The particular case of skipping all bins during warm-up phases
when no chunks have been returned yet is faster than it might look.
*/
// 获取下一个更大的bin
++idx;
bin = bin_at (av, idx);
//获取对应的bitmap标记
block = idx2block (idx);
map = av->binmap[block];
bit = idx2bit (idx);
for (;; )
{
//......
}
【找到合适的map--跳过位没有设置的block】
/* Skip rest of block if there are no more set bits in this block. */
//bit > map: bit中只有一个1，这说明map中这个1对应的高位均为0，
//为0表示对应的bin为空，则此block可以跳过
//bit <= map && bit == 0: bit = idx2bit (idx);
//bit初始化时不可能为0（有一个位置1），但是后面有左移位操作，如果刚好移到了最高位，则会为0
//bit为0时会进入下一个block，bit重新初始化为1
if (bit > map || bit == 0)
{
do
{
//跳过block
if (++block >= BINMAPSIZE) /* out of bins */
goto use_top;
}
while ((map = av->binmap[block]) == 0);//map为0表示此block所有的位均为0，跳过
bin = bin_at (av, (block << BINMAPSHIFT));
bit = 1;
}
【找到合适的bin】
//找到一个非空的bin
/* Advance to bin with set bit. There must be one. */
while ((bit & map) == 0)
{
bin = next_bin (bin);
bit <<= 1;
assert (bit != 0);
}
【检查bin是否为空】
为空时清空binMap中对应的标志位
/* Inspect the bin. It is likely to be non-empty */
victim = last (bin);
/* If a false alarm (empty bin), clear the bit. */
if (victim == bin)
{
av->binmap[block] = map &= ~bit; /* Write through */
bin = next_bin (bin);
bit <<= 1;
}
【unlink-切分-获取chunk】
这部分的代码和一般的切分代码流程差不多，在“大循环-large bin服务large chunk请求”中也可以看到；
else
{
size = chunksize (victim);
/* We know the first chunk in this bin is big enough to use. */
assert ((unsigned long) (size) >= (unsigned long) (nb));
remainder_size = size - nb;
/* unlink */
unlink (av, victim, bck, fwd);
/* Exhaust */
// 剩下的大小不足以当做一个块，设置标志，不切分
if (remainder_size < MINSIZE)
{
set_inuse_bit_at_offset (victim, size);
if (av != &main_arena)
set_non_main_arena (victim);
}
/* Split */
else // 剩下的大小还可以作为一个chunk，进行分割，剩余部分放入unsorted bin
{
remainder = chunk_at_offset (victim, nb);
/* We cannot assume the unsorted list is empty and therefore
have to perform a complete insert here. */
bck = unsorted_chunks (av);
fwd = bck->fd;
if (__glibc_unlikely (fwd->bk != bck))
{
errstr = "malloc(): corrupted unsorted chunks 2";
goto errout;
}
remainder->bk = bck;
remainder->fd = fwd;
bck->fd = remainder;
fwd->bk = remainder;
/* advertise as last remainder */
if (in_smallbin_range (nb))
av->last_remainder = remainder;
if (!in_smallbin_range (remainder_size))
{
remainder->fd_nextsize = NULL;
remainder->bk_nextsize = NULL;
}
set_head (victim, nb | PREV_INUSE | av != &main_arena ? NON_MAIN_ARENA : 0));
set_head (remainder, remainder_size | PREV_INUSE);
set_foot (remainder, remainder_size);
}
check_malloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
alloc_perturb (p, bytes);
return p;
}
2.5.10 大循环-使用top chunk
如果所有的 bin 中的 chunk 都没有办法直接满足要求（即不合并），或者说都没有空闲的 chunk。那么我们就只能使用 top chunk 了。
l Top chunk服务请求
l 合并fast bin，下次尝试
l 从系统请求内存
use_top:
/*
If large enough, split off the chunk bordering the end of memory
(held in av->top). Note that this is in accord with the best-fit
search rule. In effect, av->top is treated as larger (and thus
less well fitting) than any other available chunk since it can
be extended to be as large as necessary (up to system
limitations).
We require that av->top always exists (i.e., has size >=
MINSIZE) after initialization, so if it would otherwise be
exhausted by current request, it is replenished. (The main
reason for ensuring it exists is that we may need MINSIZE space
to put in fenceposts in sysmalloc.)
*/
victim = av->top;
size = chunksize (victim);
// 如果分割之后，top chunk 大小仍然满足 chunk 的最小大小，那么就可以直接进行分割
if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
{
remainder_size = size - nb;
remainder = chunk_at_offset (victim, nb);
av->top = remainder;
set_head (victim, nb | PREV_INUSE |(av != &main_arena ? NON_MAIN_ARENA : 0));
set_head (remainder, remainder_size | PREV_INUSE);
check_malloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
alloc_perturb (p, bytes);
return p;
}
/* When we are using atomic ops to free fast chunks we can get
here for all block sizes. */
// 否则，判断是否有 fast chunk
else if (have_fastchunks (av))
{
// 先执行一次fast bin的合并
malloc_consolidate (av);
// 判断需要的chunk是在small bin范围内还是large bin范围内
// 并计算对应的索引
// 等待下次再看看是否可以
/* restore original bin index */
if (in_smallbin_range (nb))
idx = smallbin_index (nb);
else
idx = largebin_index (nb);
}
/*
Otherwise, relay to handle system-dependent cases
*/
// 否则的话，我们就只能从系统中再次申请一点内存了
else
{
void *p = sysmalloc (nb, av);
if (p != NULL)
alloc_perturb (p, bytes);
return p;
}
2.6 Free
见注释，最终调用的是_int_free
/*
free(void* p)
Releases the chunk of memory pointed to by p, that had been previously
allocated using malloc or a related routine such as realloc.
It has no effect if p is null. It can have arbitrary (i.e., bad!)
effects if p has already been freed.
Unless disabled (using mallopt), freeing very large spaces will
when possible, automatically trigger operations that give
back unused memory to the system, thus reducing program footprint.
*/
// 除非使用mallopt关闭，释放非常大的空间会自动触发将不使用的内存归还给系统
void __libc_free(void*);
libc_hidden_proto (__libc_free)
strong_alias (__libc_free, __cfree) weak_alias (__libc_free, cfree)
strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
void
__libc_free (void *mem)
{
mstate ar_ptr;
mchunkptr p; /* chunk corresponding to mem */
void (*hook) (void *, const void *) = atomic_forced_read (__free_hook);
// 判断是否有钩子函数 __free_hook
if (__builtin_expect (hook != NULL, 0))
{
(*hook)(mem, RETURN_ADDRESS (0));
return;
}
if (mem == 0) /* free(0) has no effect */
return;
p = mem2chunk (mem);
// 如果该块内存是mmap得到的
if (chunk_is_mmapped (p)) /* release mmapped memory. */
{
/* See if the dynamic brk/mmap threshold needs adjusting.
Dumped fake mmapped chunks do not affect the threshold. */
if (!mp_.no_dyn_threshold
&& chunksize_nomask (p) > mp_.mmap_threshold
&& chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX
&& !DUMPED_MAIN_ARENA_CHUNK (p))
{
mp_.mmap_threshold = chunksize (p);
mp_.trim_threshold = 2 * mp_.mmap_threshold;
LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2,
mp_.mmap_threshold, mp_.trim_threshold);
}
munmap_chunk (p);
return;
}
ar_ptr = arena_for_chunk (p);
_int_free (ar_ptr, p, 0);
}
2.7 _int_free
static void
_int_free (mstate av, mchunkptr p, int have_lock)
功能：释放chunk，放入fast bin或合并后放入unsorted bin或合并到top chunk
具体过程如下：
如果属于fast bin，则插入fast bin链表头部；
否则如果不是mmap的内存，则进行合并；
后向合并-合并低地址 CHUNK
下一块不是top chunk-前向合并-合并高地址chunk
合并后把 chunk 放在 unsorted chunk 链表的头部
下一块是top chunk-合并到top chunk
如果合并后chunk大小大于64KB，则合并fast chunk，收缩top;
否则如果是mmap的内存，则释放mmap的chunk
2.7.1 简单的检查
检查大小与对齐
size = chunksize (p);
/* Little security check which won't hurt performance: the
allocator never wrapps around at the end of the address space.
Therefore we can exclude some size values which might appear
here by accident or by "design" from some intruder. */
//如果p+size绕过了地址空间的结尾，或者p没有对齐，则出错了
if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, 0)
|| __builtin_expect (misaligned_chunk (p), 0))
{
errstr = "free(): invalid pointer";
errout:
if (!have_lock && locked)
__libc_lock_unlock (av->mutex);
malloc_printerr (check_action, errstr, chunk2mem (p), av);
return;
}
/* We know that each chunk is at least MINSIZE bytes in size or a
multiple of MALLOC_ALIGNMENT. */
// 大小没有最小的chunk大，或者说，大小不是MALLOC_ALIGNMENT的整数倍，则有问题
if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size)))
{
errstr = "free(): invalid size";
goto errout;
}
// 检查该chunk是否处于使用状态，非调试状态下没有作用
check_inuse_chunk(av, p);
2.7.2 Fast bin
如果属于fast bin，则插入fast bin链表头部；
/*
If eligible, place chunk on a fastbin so it can be found
and used quickly in malloc.
*/
//大小属于fast bin
if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
#if TRIM_FASTBINS
/*
If TRIM_FASTBINS set, don't place chunks
bordering top into fastbins
*/
//默认 #define TRIM_FASTBINS 0，因此默认情况下下面的语句不会执行
//如果当前chunk是fast chunk，并且下一个chunk是top chunk，则不能插入
//此时由于定义了TRIM_FASTBINS，这个chunk会和top chunk合并
&& (chunk_at_offset(p, size) != av->top)
#endif
)
{
//检查下一个chunk的大小上限和下限
// 下一个chunk的大小不能小于两倍的SIZE_SZ,并且
// 下一个chunk的大小不能大于system_mem， 一般为132k
// 如果出现这样的情况，就报错
if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size))
<= 2 * SIZE_SZ, 0)
|| __builtin_expect (chunksize (chunk_at_offset (p, size))
>= av->system_mem, 0))
{
/* We might not have a lock at this point and concurrent modifications
of system_mem might have let to a false positive. Redo the test
after getting the lock. */
if (have_lock
|| ({ assert (locked == 0);
__libc_lock_lock (av->mutex);
locked = 1;
chunksize_nomask (chunk_at_offset (p, size)) <= 2 * SIZE_SZ
|| chunksize (chunk_at_offset (p, size)) >= av->system_mem;
}))
{
errstr = "free(): invalid next size (fast)";
goto errout;
}
if (! have_lock)
{
__libc_lock_unlock (av->mutex);
locked = 0;
}
}
// 将chunk的mem部分全部设置为perturb_byte
free_perturb (chunk2mem(p), size - 2 * SIZE_SZ);
// 设置fast chunk的标记位
set_fastchunks(av);
unsigned int idx = fastbin_index(size);
fb = &fastbin (av, idx);
/* Atomically link P to its fastbin: P->FD = *FB; *FB = P; */
//将P插入fast bin链表头部
mchunkptr old = *fb, old2;
unsigned int old_idx = ~0u;
do
{
/* Check that the top of the bin is not the record we are going to add
(i.e., double free). */
// 防止对 fast bin double free
if (__builtin_expect (old == p, 0))
{
errstr = "double free or corruption (fasttop)";
goto errout;
}
/* Check that size of fastbin chunk at the top is the same as
size of the chunk that we are adding. We can dereference OLD
only if we have the lock, otherwise it might have already been
deallocated. See use of OLD_IDX below for the actual check. */
if (have_lock && old != NULL)
old_idx = fastbin_index(chunksize(old));
//P插入fast bin链表头部
p->fd = old2 = old;
}
while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2)) != old2);
if (have_lock && old != NULL && __builtin_expect (old_idx != idx, 0))
{
errstr = "invalid fastbin entry (free)";
goto errout;
}
}
2.7.3 合并非 mmap 的空闲 chunk
合并其他非mmap的chunks。
/*
Consolidate other non-mmapped chunks as they arrive.
*/
else if (!chunk_is_mmapped(p)) {
if (! have_lock) {
__libc_lock_lock (av->mutex);
locked = 1;
}
nextchunk = chunk_at_offset(p, size);
检查是否有异常情况
/* Lightweight tests: check whether the block is already the
top block. */
// 当前free的chunk不能是top chunk
if (__glibc_unlikely (p == av->top))
{
errstr = "double free or corruption (top)";
goto errout;
}
/* Or whether the next chunk is beyond the boundaries of the arena. */
// 当前free的chunk的下一个chunk不能超过arena的边界
if (__builtin_expect (contiguous (av)
&& (char *) nextchunk
>= ((char *) av->top + chunksize(av->top)), 0))
{
errstr = "double free or corruption (out)";
goto errout;
}
/* Or whether the block is actually not marked used. */
// 当前要free的chunk的使用标记没有被标记，double free
if (__glibc_unlikely (!prev_inuse(nextchunk)))
{
errstr = "double free or corruption (!prev)";
goto errout;
}
//下一个chunk的大小是否超出范围
nextsize = chunksize(nextchunk);
if (__builtin_expect (chunksize_nomask (nextchunk) <= 2 * SIZE_SZ, 0)
|| __builtin_expect (nextsize >= av->system_mem, 0))
{
errstr = "free(): invalid next size (normal)";
goto errout;
}
填充
free_perturb (chunk2mem(p), size - 2 * SIZE_SZ);
后向合并-合并低地址 CHUNK
/* consolidate backward */
//如果低地址chunk空闲，则合并后unlink
if (!prev_inuse(p)) {
prevsize = prev_size (p);
size += prevsize;
p = chunk_at_offset(p, -((long) prevsize));
unlink(av, p, bck, fwd);
}
下一块不是top chunk-前向合并-合并高地址chunk
合并后把 chunk 放在 unsorted chunk 链表的头部
if (nextchunk != av->top) {
/* get and clear inuse bit */
nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
/* consolidate forward */
// 如果不在使用，合并，否则清空当前chunk的使用状态
if (!nextinuse) {
unlink(av, nextchunk, bck, fwd);
size += nextsize;
} else
clear_inuse_bit_at_offset(nextchunk, 0);
/*
Place the chunk in unsorted chunk list. Chunks are
not placed into regular bins until after they have
been given one chance to be used in malloc.
*/
// 把 chunk 放在 unsorted chunk 链表的头部
bck = unsorted_chunks(av);
fwd = bck->fd;
if (__glibc_unlikely (fwd->bk != bck))
{
errstr = "free(): corrupted unsorted chunks";
goto errout;
}
p->fd = fwd;
p->bk = bck;
if (!in_smallbin_range(size))
{
p->fd_nextsize = NULL;
p->bk_nextsize = NULL;
}
bck->fd = p;
fwd->bk = p;
set_head(p, size | PREV_INUSE);
set_foot(p, size);
check_free_chunk(av, p);
}
下一块是top chunk-合并到top chunk
/*
If the chunk borders the current high end of memory,
consolidate into top
*/
else {
size += nextsize;
set_head(p, size | PREV_INUSE);
av->top = p;
check_chunk(av, p);
}
向系统返还内存
如果合并后chunk大小大于64KB，则合并fast chunk，收缩top;
/*
If freeing a large space, consolidate possibly-surrounding
chunks. Then, if the total unused topmost memory exceeds trim
threshold, ask malloc_trim to reduce top.
Unless max_fast is 0, we don't know if there are fastbins
bordering top, so we cannot tell for sure whether threshold
has been reached unless fastbins are consolidated. But we
don't want to consolidate on each free. As a compromise,
consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
is reached.
*/
//如果释放大的空间，会合并周边的chunks。
//然后，如果top超过了trim的阈值，则使用malloc_trim来减少top
//在此之前，我们要先合并fast bin
//我们并不想每一次free都执行合并,作为一种折中，
//如果到达FASTBIN_CONSOLIDATION_THRESHOLD（64KB）才合并fast bin
//合并才执行top的收缩
if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
if (have_fastchunks(av))
malloc_consolidate(av);
if (av == &main_arena) {
#ifndef MORECORE_CANNOT_TRIM
if ((unsigned long)(chunksize(av->top)) >=(unsigned long)(mp_.trim_threshold))
systrim(mp_.top_pad, av);
#endif
} else {
/* Always try heap_trim(), even if the top chunk is not
large, because the corresponding heap might go away. */
heap_info *heap = heap_for_ptr(top(av));
assert(heap->ar_ptr == av);
heap_trim(heap, mp_.top_pad);
}
}
if (! have_lock) {
assert (locked);
__libc_lock_unlock (av->mutex);
}
2.7.4 释放mmap的chunk
/*
If the chunk was allocated via mmap, release via munmap().
*/
else {
munmap_chunk (p);
}
1.8 malloc_consolidate
功能：清除fast chunk，能合并则合并，放入unsorted bin或并入top chunk。
具体过程如下：
如果fast bin已经初始化
遍历fast bin
遍历fast chunk
如果低地址chunk空闲，合并之，unlink
如果高地址不是top chunk
如果chunk空闲，合并之，unlink
Fast chunk放入unsorted bin的头部
如果高地址是top chunk
合并到 top中
如果fast bin没有初始化
调用malloc_init_state初始化malloc_state
/*
------------------------- malloc_consolidate -------------------------
malloc_consolidate is a specialized version of free() that tears
down chunks held in fastbins. Free itself cannot be used for this
purpose since, among other things, it might place chunks back onto
fastbins. So, instead, we need to use a minor variant of the same
code.
Also, because this routine needs to be called the first time through
malloc anyway, it turns out to be the perfect place to trigger
initialization code.
*/
//合并fast bin或初始化malloc_state
static void malloc_consolidate(mstate av)
合并fast chunk
// fast bin已经初始化
if (get_max_fast () != 0) {
// 清除fastbin 标记
clear_fastchunks(av);
unsorted_bin = unsorted_chunks(av);
/*
Remove each chunk from fast bin and consolidate it, placing it
then in unsorted bin. Among other reasons for doing this,
placing in unsorted bin avoids needing to calculate actual bins
until malloc is sure that chunks aren't immediately going to be
reused anyway.
*/
//移除并合并fast bin中的每个chunk，将其放入unsorted bin
maxfb = &fastbin (av, NFASTBINS - 1); //最大的索引：9，实际上遍历的是fast bin 0~8
fb = &fastbin (av, 0);
do {
p = atomic_exchange_acq (fb, NULL);
if (p != 0) {
do {//遍历某个fast bin中的fast chunk
check_inuse_chunk(av, p);
nextp = p->fd;
/* Slightly streamlined version of consolidation code in free() */
//free()中的合并代码的简单流线型版本
size = chunksize (p);
nextchunk = chunk_at_offset(p, size);
nextsize = chunksize(nextchunk);
//如果低地址chunk空闲，则合并后unlink，后面会放入unsorted chunk
if (!prev_inuse(p)) {
prevsize = prev_size (p);
size += prevsize;
p = chunk_at_offset(p, -((long) prevsize));
unlink(av, p, bck, fwd);
}
//下一块不是top chunk-前向合并-合并高地址chunk
//合并后把 chunk 放在 unsorted chunk 链表的头部
if (nextchunk != av->top) {
nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
if (!nextinuse) {
size += nextsize;
unlink(av, nextchunk, bck, fwd);
} else
clear_inuse_bit_at_offset(nextchunk, 0);
first_unsorted = unsorted_bin->fd;
unsorted_bin->fd = p;
first_unsorted->bk = p;
if (!in_smallbin_range (size)) {
p->fd_nextsize = NULL;
p->bk_nextsize = NULL;
}
set_head(p, size | PREV_INUSE);
p->bk = unsorted_bin;
p->fd = first_unsorted;
set_foot(p, size);
}
else {//下一块是top chunk-合并到top chunk
size += nextsize;
set_head(p, size | PREV_INUSE);
av->top = p;
}
} while ( (p = nextp) != 0);
}
} while (fb++ != maxfb);
}
初始化malloc_state
else {
malloc_init_state(av);
check_malloc_state(av);
}
3 结论
【主要的API】
1. malloc_consolidate
功能：
清空fast bins，fast chunk能合并则合并，放入unsorted bin或并入top chunk。
具体过程如下：
如果fast bin已经初始化
遍历fast bin
遍历fast chunk
如果低地址chunk空闲，合并之，unlink
如果高地址不是top chunk
如果chunk空闲，合并之，unlink
Fast chunk放入unsorted bin的头部
如果高地址是top chunk
合并到 top中
如果fast bin没有初始化
调用malloc_init_state初始化malloc_state
2. Unlink
unlink 用来将一个双向 bin 链表中的一个 chunk 取出来
参数：
AV：arena header(malloc_state)
P ：将要unlink的chunk
BK：P后面的chunk <--
FD：P前面的chunk -->
3个chunk的顺序
-------->地址增加
BK P FD
具体过程如下：
将chunk从FD/BK链表中摘除；
如果是large chunk，则将chunk从fd_nextsize/bk_nextsize链表中摘除
3. _int_malloc
1．Fast bin服务请求
2．Small bin服务请求
3．如果是large chunk，则合并fast bin
4．循环处理
1) Unsorted bin服务请求
（遍历的unsorted chunk要么匹配用户请求被返回，要么放入对应的bin中）
2) Large bin服务请求
3) next largest bin服务请求
4) top chunk服务请求
5) 合并fast bin，下次尝试
6) 从系统请求内存
4. _int_free
功能：释放chunk，放入fast bin或合并后放入unsorted bin或合并到top chunk
具体过程如下：
如果属于fast bin，则插入fast bin链表头部；
否则如果不是mmap的内存，则进行合并；
后向合并-合并低地址 CHUNK
下一块不是top chunk-前向合并-合并高地址chunk
合并后把 chunk 放在 unsorted chunk 链表的头部
下一块是top chunk-合并到top chunk
如果合并后chunk大小大于64KB，则合并fast chunk，收缩top;
否则如果是mmap的内存，则释放mmap的chunk
3 参考文档
1. https://ctf-wiki.github.io/ctf-wiki/pwn/heap/heap_structure/
2. https://ctf-wiki.github.io/ctf-wiki/pwn/heap/heap_implementation_details/