BloomFilter布隆过滤器C，Python，Redis实现
Python实现：https://github.com/chfguo/bloomfilter
基本原理：
在进行布隆过滤器的介绍前，先说一下位数组（bit array）。所谓的位数组，主要是为了有效地利用内存空间而设计的一种存储数据的方式。在这种结构中一个整数在内存中用一位（1 bit）表示。这里所谓的表示就是如果整数存在，相应的二进制位就为 1，否则为 0。
布隆过滤器便利用了位数组的特性，它通过 hash 函数（为了降低 hash 函数的冲突效应，一般会选择多个随机的 hash 函数）将一个元素映射成一个位数组（Bit array）中的某些点上。在检索的时候，通过判断这些点是不是 1 来判读集合中有没有某个特定的元素。这就是布隆过滤器的基本思想。
举个例子，假设我们有 4 个元素 ‘cat’，‘dog’，‘pig’，‘bee’，选择 3 个 hash 函数，位数组长度为 10。为了方便说明，我们依次修改位数组的值。当插入第一个元素时 ‘cat’ 后，位数组变为：
当插入 ‘dog’ 和 ‘pig’ 后，位数组变为：
当我们查询 ‘bee’ 的时候，如果，‘bee’ 有一位或两位映射在 9，10 位上，则可以正确判断出元素 ‘bee’ 不在集合中，但是如果 ‘bee’ 被映射在 1-8 位上则会出现误判，并且随着元素增加，误判率会逐渐增加。
布隆过滤器的判断有以下特点：
- 如果判断一个元素在集合中，那么这个元素有一定概率不在集合中（会出现误判）
- 如果判断一个元素不在集合中，那么这个元素一定不在集合中（不会出现误判）
上面的例子说明了布隆过滤器的插入过程和误判率的问题，在这个例子中涉及三个重要数值：
- 需要存放的集合中元素的个数 n n n
- 位数组的长度 m m m
- hash 函数的个数 k k k
通过一定的计算(参考这里)，可以计算出最优的 m m m 和 k k k:
m = − n ln ϵ ( ln 2 ) 2 , k = ln 2 m n m=-\frac{n\ln\epsilon}{(\ln2)^2}, k=\ln2\frac{m}{n} m=−(ln2)2nlnϵ,k=ln2nm
C语言实现
位数组的实现
我们可以利用 char 数组实现位数组 char *bit_arr，对于一个数字，比如 26，它应该存放在这个 char 数组的第 4 个 char 中，即 bit_arr[3] 中， 如果按照高位在前，低位在后的顺序，它应该在 (26 & 0x7 + 1) 的位置， & 0x7 可以取得一个数字的二进制的最后三位，相当于取除 8 的余数，最后利用 1<< 将该位设置为 1。这样的获得的位数组是：7 6 5 4 3 2 1 0 – 15 14 13 12 11 10 9 8 – 23 22 21 20 19 18 17 16 – 31 30 29 28 27 26 25 24–…，如果要改正这个顺序，将 26 & 0x7 改为 7 - （26 & 0x7）
具体实现：
//插入函数
void add_to_bitarray(char *bitarr, uint64_t num){ /* num代表要插进数组中的数 */
// SHIFT = 3， num >> SHIFT 相当于 num/8
// MASK = 0x7， num & MASK 相当于取余数
bitarr[num >> SHIFT] |= (1 << (num & MASK));
}
//查询函数
int is_in_bitarray(char *bitarr, int num){
return bitarr[num >> SHIFT] & (1 << (num & MASK));
}
Hash 函数
Hash 函数选用 MurmurHash。MurmurHash 是一种非加密型哈希函数，适用于一般的哈希检索操作。已经被应用到很多开源项目如Redis，Memcached，Cassandra，HBase，Lucene等。与其它流行的哈希函数相比，对于规律性较强的key，MurmurHash的随机分布特征表现更良好。
具体实现：
uint64_t MurmurHash64A (const void * key, uint32_t len, uint64_t seed ){
const uint64_t m = 0xc6a4a7935bd1e995;
const int r = 47;
uint64_t h = seed ^ (len * m);
const uint64_t * data = (const uint64_t *)key;
const uint64_t * end = data + (len/8);
while(data != end)
{
uint64_t k = *data++;
k *= m;
k ^= k >> r;
k *= m;
h ^= k;
h *= m;
}
const unsigned char * data2 = (const unsigned char*)data;
switch(len & 7){
case 7: h ^= (uint64_t)(data2[6]) << 48;
case 6: h ^= (uint64_t)(data2[5]) << 40;
case 5: h ^= (uint64_t)(data2[4]) << 32;
case 4: h ^= (uint64_t)(data2[3]) << 24;
case 3: h ^= (uint64_t)(data2[2]) << 16;
case 2: h ^= (uint64_t)(data2[1]) << 8;
case 1: h ^= (uint64_t)(data2[0]);
h *= m;
};
h ^= h >> r;
h *= m;
h ^= h >> r;
return h;
}
包装下 MurmurHash64A，使获得的散列值在指定的长度内：
uint64_t hash(const char* data, uint32_t len, uint64_t seed, uint64_t bits) {
return MurmurHash64A(data, len, seed) % bits;
}
BloomFilter
利用一个结构体存放布隆过滤器的一些设置参数：
typedef struct {
uint32_t capacity; //需要存放的元素个数
uint32_t hashes; //hash 函数的个数
uint64_t bits; //位数组的长度
float error; //误报率
uint64_t *seeds; //hash 函数的seed
} bloomfilterctxt;
bloomfilterctxt 的初始化和释放内存：
void init_bloomfilter(bloomfilterctxt * ctxt, uint32_t capacity, float error, int prime_length) { //prime_length是否将 bits 设置为素数长度
uint32_t i;
ctxt->capacity = capacity;
ctxt->bits = (uint64_t)(-(log(error) * capacity) / (log(2) * log(2)));
if (prime_length)
ctxt->bits = next_prime_number(ctxt->bits);
ctxt->hashes = (uint32_t)(ceil(log(2) * ctxt->bits / capacity));
ctxt->error = error;
ctxt->seeds = (uint64_t *)(malloc(ctxt->hashes * sizeof(uint64_t)));
uint32_t a = 1664525;
uint32_t c = 1013904223;
uint32_t x = 314159265;
for (i = 0; i < ctxt->hashes; ++i) {
ctxt->seeds[i] = x;
x = a * x + c;
}
}
void free_bloomfilter(bloomfilterctxt * ctxt) {
if (ctxt->seeds) {
free(ctxt->seeds);
}
}
int is_prime_number(uint64_t num){
if (num == 2) return 1;
if (num%2 == 0) return 0;
uint32_t i;
uint32_t len = (uint32_t)(sqrt((double)num)) + 1;
for (i = 3; i < len ; i+=2)
if (num%i == 0)
return 0;
return 1;
}
uint64_t next_prime_number(uint64_t num){
uint64_t i;
for (i = num; i < 2*num; i++)
if (is_prime_number(i))
return i;
return num;
}
实现插入和查询函数：
void add(char *bitmap, const char *data, uint32_t len, uint64_t *seeds, uint32_t hashes, uint64_t bits){
uint32_t i;
uint64_t position;
for(i=0; i<hashes; i++){
position = hash(data,len,seeds[i],bits);
bitmap[position >> SHIFT] |= (1 << (position & MASK));
}
}
int is_contain(char *bitmap, const char *data, uint32_t len, uint64_t *seeds, uint32_t hashes, uint64_t bits){
uint32_t i;
uint64_t position = hash(data,len,seeds[0],bits);
int result = bitmap[position >> SHIFT] & (1 << (position & MASK));
for(i=1; i<hashes; i++){
position = hash(data,len,seeds[i],bits);
result = result && (bitmap[position >> SHIFT] & (1 << (position & MASK)));
if(result == 0)
break;
}
return result;
}
//包装一下方便调用
void bf_add(bloomfilterctxt* ctxt, char *bitmap, const char *data, uint32_t len){
add(bitmap,data, len, ctxt->seeds, ctxt->hashes, ctxt->bits);
}
int bf_is_contain(bloomfilterctxt* ctxt,char *bitmap, const char *data, uint32_t len){
return is_contain(bitmap,data,len,ctxt->seeds, ctxt->hashes, ctxt->bits);
}
测试一下
main.c函数
bloomfilterctxt ctxt;
int i;
//假设每个人推送1000个
uint32_t capacity = 1000;
float error = 0.05;
init_bloomfilter(&ctxt,capacity,error,1);
char *bitmap = (char*)calloc(ctxt.bits/8+1,sizeof(char));
char *data1 = "apple";
bf_add(&ctxt,bitmap,data1,5);
if(bf_is_contain(&ctxt,bitmap,data1,5))
printf("%s in bitmap\n",data1);
else
printf("%s not in bitmap\n",data1);
char *data2 = "banana";
bf_add(&ctxt,bitmap,data1,6);
if(bf_is_contain(&ctxt,bitmap,data2,6))
printf("%s in bitmap\n",data2);
else
printf("%s not in bitmap\n",data2);
结果：
apple in bitmap
banana not in bitmap
Python包装
首先编辑一个 bloom.pyx 文件, 该文件中有个BloomFilter类，该类实现对相关C函数的调用。bitmap数组由 bytes 函数获的。
cdef extern from "bloomfilter.h":
ctypedef unsigned long int uint64_t
ctypedef unsigned int uint32_t
ctypedef struct bloomfilterctxt:
uint32_t capacity;
uint32_t hashes;
uint64_t bits;
float error;
uint64_t *seeds;
void clean_bitmap(char *bitmap, uint32_t len)
void free_bloomfilter(bloomfilterctxt * ctxt)
void init_bloomfilter(bloomfilterctxt * ctxt, uint32_t capacity, float error,bint prime_length)
void bf_add(bloomfilterctxt* ctxt,char *bitmap, const char *data, uint32_t len);
bint bf_is_contain(bloomfilterctxt* ctxt,char *bitmap, const char *data, uint32_t len);
uint64_t hash(const char* data, uint32_t len, uint64_t seed, uint64_t bits);
cdef class BloomFilter(object):
cdef bloomfilterctxt context
property bits:
def __get__(self):
return self.context.bits
property hashes:
def __get__(self):
return self.context.hashes
def __cinit__(self, capacity, error, prime_length = True):
init_bloomfilter(&self.context,capacity,error,prime_length)
def __dealloc__(self):
free_bloomfilter(&self.context)
def add(self,bitmap,data):
key = data.encode()
bf_add(&self.context, bitmap,key,len(key))
def is_contain(self,bitmap,data):
key = data.encode()
return bf_is_contain(&self.context, bitmap,key,len(key))
def clean_bitmap(self,bitmap):
clean_bitmap(bitmap, len(bitmap))
def hash(self,data):
key = data.encode()
offset = []
for i in range(self.hashes):
seed = self.context.seeds[i]
offset.append(hash(key,len(key),seed,self.context.bits))
return offset
包装一个 bloomfilter.py 文件，方便调用。
from BloomFilter import BloomFilter
class LocalBloomFilter():
def __init__(self,capacity,error,prime_length = True):
self.bf = BloomFilter(capacity,error,prime_length)
self.bitmap = bytes(int(self.bf.bits/8)+1)
def add(self,data):
if isinstance(data,(list,tuple)):
for v in data:
assert isinstance(v,str),'add() arg must be a str or list/tuple of strings'
self.bf.add(self.bitmap, v)
else:
assert isinstance(data, str), 'add() arg must be a str or list/tuple of strings'
self.bf.add(self.bitmap,data)
def is_contain(self,data):
if isinstance(data,(list,tuple)):
for v in data:
assert isinstance(v,str),'is_contain() arg must be a str or list/tuple of strings'
return [self.bf.is_contain(self.bitmap, v) for v in data]
else:
assert isinstance(data, str), 'is_contain() arg must be a str or list/tuple of strings'
return self.bf.is_contain(self.bitmap, data)
def clean(self):
self.bf.clean_bitmap(self.bitmap)
class RedisBloomFilter():
def __init__(self, capacity, error, redis_conn, prime_length=True):
self.bf = BloomFilter(capacity, error, prime_length)
self.redis_conn = redis_conn
def add(self, key, data):
if isinstance(data, (list, tuple)):
offset = []
for v in data:
assert isinstance(v, str), 'add() arg must be a str or list/tuple of strings'
offset += self.bf.hash(v)
with self.redis_conn.pipeline() as pipe:
for o in offset:
pipe.setbit(key,o,1)
pipe.execute()
else:
assert isinstance(data, str), 'add() arg must be a str or list/tuple of strings'
offset = self.bf.hash(data)
with self.redis_conn.pipeline() as pipe:
for o in offset:
pipe.setbit(key,o,1)
pipe.execute()
def is_contain(self, key, data):
if isinstance(data, (list, tuple)):
offset = []
for v in data:
assert isinstance(v, str), 'is_contain() arg must be a str or list/tuple of strings'
offset += self.bf.hash(v)
with self.redis_conn.pipeline() as pipe:
for o in offset:
pipe.getbit(key, o)
result_bits = pipe.execute()
result = []
for i in range(len(data)):
result_bit = result_bits[i*self.bf.hashes:(i+1)*self.bf.hashes]
if sum(result_bit) == self.bf.hashes:
result.append(True)
else:
result.append(False)
return result
else:
assert isinstance(data, str), 'is_contain() arg must be a str or list/tuple of strings'
offset = self.bf.hash(data)
with self.redis_conn.pipeline() as pipe:
for o in offset:
pipe.getbit(key, o)
results = pipe.execute()
if sum(results) == self.bf.hashes:
return True
return False
def clean(self,key):
self.redis_conn.delete(key)
setup文件
创建 setup.py 文件安装：
from distutils.core import setup
ext_files = ['bloomfilter.c']
kwargs = {}
from Cython.Distutils import build_ext
from Cython.Distutils import Extension
print('Building from Cython')
ext_files.append('bloom.pyx')
kwargs['cmdclass'] = {'build_ext': build_ext}
ext_modules = [Extension("BloomFilter",ext_files)]
setup(
name = 'bloomfilter',
version = '0.0.1',
author = '*********',
author_email = '***********',
license = 'MIT License',
ext_modules = ext_modules,
py_modules = ['bloomfilter'],
classifiers = [
'Intended Audience :: Developers',
'License :: OSI Approved :: MIT License',
'Programming Language :: C',
'Programming Language :: Cython',
'Topic :: Software Development :: Libraries :: Python Modules',
],
**kwargs
)
测试
BloomFilter测试
import random
import string
from bloomfilter import BloomFilter
items = []
for i in range(20000):
random_string = ''.join(random.sample(string.ascii_letters + string.digits, 5))
items.append(random_string)
bf = BloomFilter(10000,0.05,True)
bitmap = bytes(int(bf.bits/8)+1)
for i in range(10000):
item = items[i]
bf.add(bitmap,item)
error = 0
for i in range(10000,20000):
item = items[i]
if bf.is_contain(bitmap,item):
error += 1
print('%s errors in 10000 samples'%error)
LocalBloomFilter测试
from bloomfilter import LocalBloomFilter
lbf = LocalBloomFilter(1000,0.05)
lbf.add(['apple','banana','C','China','china'])
lbf.is_contain(['apple','Python','Cython','China','china'])
lbf.clean() # set bit array to zero
RedisBloomFilter测试
from bloomfilter import RedisBloomFilter
from redis import StrictRedis
redis_conn = StrictRedis(host = 'localhost',port = 6379, db = 0)
rbf = RedisBloomFilter(1000,0.05,redis_conn)
rbf.add('bloom','Apple') # arg 1 is key, arg 2 is item
rbf.is_contain('bloom','Apple')
rbf.add('bloom',['apple','China'])
rbf.is_contain('bloom',['apple','china'])