url: http://thrillerzw.iteye.com/blog/2055487
2014-04-25 23:27
memcached的分布式主要体现在client端,server之间没有关系。
一致性hash算法:cache不能命中的问题仍然存在,但是只存在于2个节点之间的位置。相对于取模的算法,一致性hash算法除了计算key的hash值外,还会计算每个server对应的hash值,然后将这些hash值映射到一个有限的值域上(比如0~2^32)。通过寻找hash值等于大于hash(key)的最小server作为存储该key数据的目标server。如果找不到,则直接把具有最小hash值的server作为目标server。
源码版本:Memcached-Java-Client-release_2.6.1
1、计算每个server对应的hash值,将值存入treemap 排序模拟圆,主要方法代码入下。
private void populateConsistentBuckets() { // store buckets in tree map consistentBuckets = new TreeMap<Long, String>(); MessageDigest md5 = MD5.get(); if (this.totalWeight <= 0 && this.weights != null) { for (int i = 0; i < this.weights.length; i++) this.totalWeight += (this.weights[i] == null) ? 1 : this.weights[i]; } else if (this.weights == null) { this.totalWeight = this.servers.length; } for (int i = 0; i < servers.length; i++) { int thisWeight = 1; if (this.weights != null && this.weights[i] != null) thisWeight = this.weights[i]; double factor = Math.floor(((double) (40 * this.servers.length * thisWeight)) / (double) this.totalWeight); //根据权重控制虚拟节点数量 (factor * 4)。 for (long j = 0; j < factor; j++) { byte[] d = md5.digest((servers[i] + "-" + j).getBytes()); //192.168.211.240:11212-0 。。。 192.168.211.240:11212-39 // System.out.println("server key="+servers[i] + "-" + j); for (int h = 0; h < 4; h++) { Long k = ((long) (d[3 + h * 4] & 0xFF) << 24) | ((long) (d[2 + h * 4] & 0xFF) << 16) | ((long) (d[1 + h * 4] & 0xFF) << 8) | ((long) (d[0 + h * 4] & 0xFF)); //按Long值从小到大排序 consistentBuckets.put(k, servers[i]); } } // Create a socket pool for each host // Create an object pool to contain our active connections GenericObjectPool gop; SchoonerSockIOFactory factory; if (authInfo != null) { factory = new AuthSchoonerSockIOFactory(servers[i], isTcp, bufferSize, socketTO, socketConnectTO, nagle, authInfo); } else { factory = new SchoonerSockIOFactory(servers[i], isTcp, bufferSize, socketTO, socketConnectTO, nagle); } gop = new GenericObjectPool(factory, maxConn, GenericObjectPool.WHEN_EXHAUSTED_BLOCK, maxIdle, maxConn); factory.setSockets(gop); socketPool.put(servers[i], gop); } System.out.println("consistentBuckets="+consistentBuckets); }
输出:
consistentBuckets.get(bucket) //计算客户端key hash值 private static long md5HashingAlg(String key) { MessageDigest md5 = MD5.get(); md5.reset(); md5.update(key.getBytes()); byte[] bKey = md5.digest(); long res = ((long) (bKey[3] & 0xFF) << 24) | ((long) (bKey[2] & 0xFF) << 16) | ((long) (bKey[1] & 0xFF) << 8) | (long) (bKey[0] & 0xFF); System.out.println("key="+key+" hash result="+res); return res; } 输出: key=1 hash result=943901380 key=2 hash result=2373066440 //查找目标server private final Long findPointFor(Long hv) { // this works in java 6, but still want to release support for java5 // Long k = this.consistentBuckets.ceilingKey( hv ); // return ( k == null ) ? this.consistentBuckets.firstKey() : k; //hash值等于大于hash(key)的最小server作为存储该key数据的目标server。 SortedMap<Long, String> tmap = this.consistentBuckets.tailMap(hv); System.out.println("hash result="+hv+" tailMap="+tmap); //如果找不到,则直接把具有最小hash值的server作为目标server。 return (tmap.isEmpty()) ? this.consistentBuckets.firstKey() : tmap.firstKey(); } 输出: hash result=943901380 tailMap={948021698=192.168.211.240:11212, 973571166=192.168.211.240:11212, 。。。4294784513=192.168.211.240:11212}