redis client高并发量死锁问题

在线上服务器跑一段时间之后发现大量如下错误,初步分析是redis实现的JedisPool的的配置有问题。

yunpeng@yunpeng-duitang:/work/xp$ cat aa.log | grep "java.lang.Object.wait(Native Method)" | wc -l
254
[quote]

"btpool0-1692" prio=10 tid=0x00002aaab54d8000 nid=0x2bf9 in Object.wait() [0x000000005bcf4000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:485)
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
- locked <0x00000000c1ce6488> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at redis.clients.util.Pool.getResource(Pool.java:20)
at org.springframework.data.redis.connection.jedis.JedisConnectionFactory.fetchJedisConnector(JedisConnectionFactory.java:94)

[/quote]
问题原因GenericObjectPool默认的maxActive只有8，参考：
http://commons.apache.org/pool/api-1.6/org/apache/commons/pool/impl/GenericObjectPool.html

spring 的代码结构：

class JedisConnectionFactory{

	private JedisPool pool = null;
	private JedisPoolConfig poolConfig = new JedisPoolConfig();

	public void afterPropertiesSet() {

		pool = new JedisPool(poolConfig, shardInfo.getHost(), shardInfo.getPort(), shardInfo.getTimeout(),shardInfo.getPassword());
	}

	protected Jedis fetchJedisConnector() {
		return pool.getResource();
	}

}

class Pool{  

    private final GenericObjectPool internalPool;  

    public Pool(final GenericObjectPool.Config poolConfig,  
            PoolableObjectFactory factory) {  

	 this.internalPool = new GenericObjectPool(factory, poolConfig);  
    }  

    pool.getResource(){  
         internalPool.borrowObject();  
    }  

}  

class JedisPool extends Pool{

    public JedisPool(final Config poolConfig, final String host, int port, int timeout, final String password,final int database) {
        super(poolConfig, new JedisFactory(host, port, timeout, password, database));
    }

}

写了一个测试代码重现了这个问题：

import org.apache.commons.pool.BasePoolableObjectFactory;
import org.apache.commons.pool.PoolableObjectFactory;
import org.apache.commons.pool.impl.GenericObjectPool;
import org.apache.commons.pool.impl.GenericObjectPool.Config;

public class GenericObjectPoolTest {

	public static void main(String[] args) throws Exception {
		Config config = new Config();
//		config.setTestWhileIdle(true);
//		config.setMinEvictableIdleTimeMillis(60000);
//		config.setTimeBetweenEvictionRunsMillis(30000);
//		config.setNumTestsPerEvictionRun(-1);
		final GenericObjectPool internalPool = new GenericObjectPool(new ConnectionFactory(),config);
		for (int x = 0;x<1000;++x){
			Thread t  = new Thread(new Runnable() {

				@Override
				public void run() {
					while(true){
						try {
							Object str = internalPool.borrowObject();
							System.out.println(str);
						} catch (Exception e) {
							e.printStackTrace();
						}
					}
				}
			});
			t.start();
		}

	}

	static class ConnectionFactory extends BasePoolableObjectFactory implements PoolableObjectFactory{

		@Override
		public Object makeObject() throws Exception {
			System.out.println(Thread.currentThread().getName()+" make....");
			return "connection";
		}



	}
}

输出，只有8个object：

Thread-0 make....
Thread-1 make....
Thread-2 make....
Thread-3 make....
connection
connection
connection
Thread-2 make....
connection
Thread-2 make....
connection
Thread-2 make....
connection
Thread-1 make....
connection
connection


"Thread-999" prio=10 tid=0x00007f87dc6c0000 nid=0x213d in Object.wait() [0x00007f879a264000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e19f3a18> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at java.lang.Object.wait(Object.java:485)
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
- locked <0x00000000e19f3a18> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at GenericObjectPoolTest$1.run(GenericObjectPoolTest.java:22)
at java.lang.Thread.run(Thread.java:662)

"Thread-998" prio=10 tid=0x00007f87dc6be000 nid=0x213c in Object.wait() [0x00007f879a365000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e19f39e8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at java.lang.Object.wait(Object.java:485)
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
- locked <0x00000000e19f39e8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at GenericObjectPoolTest$1.run(GenericObjectPoolTest.java:22)
at java.lang.Thread.run(Thread.java:662)

"Thread-997" prio=10 tid=0x00007f87dc6bc000 nid=0x213b in Object.wait() [0x00007f879a466000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e19f39b8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at java.lang.Object.wait(Object.java:485)
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
- locked <0x00000000e19f39b8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at GenericObjectPoolTest$1.run(GenericObjectPoolTest.java:22)
at java.lang.Thread.run(Thread.java:662)