Zookeeper分布式队列的实现
摘要：本文要通过zookeeper实现一个简单可靠的分布式队列
本文源码请在这里下载：https://github.com/appleappleapple/DistributeLearning
一、队列
Zookeeper可以处理两种类型的队列：
（1）同步队列
当一个队列的成员都聚齐时，这个队列才可用，否则一直等待所有成员到达。例如一个班去旅游，看是否所有人都到齐了，到齐了就发车。例如有个大任务分解为多个子任务，要所有子任务都完成了才能进入到下一流程。
（2）先进先出队列
按照FIFO方式进行入队和出队
例如实现生产者和消费者模型
二、实现思路
（1）同步队列
在zookeeper中先创建一个根目录 queue_sync，做为队列队列的消费者监视/queue/start节点，刚开始还没有这个节点，所以什么都不会做。入队操作就是在queue_sync下创建子节点，然后计算子节点的总数，看是否和队列的目标数量相同。如果相同，创建/queue_sync/start节点，由于/queue_sync/start这个节点有了状态变化，zookeeper就会通知监视者：队员已经到齐了，监视者得到通知后进行自己的后续流程
实现代码
package com.github.distribute.queue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import org.I0Itec.zkclient.ExceptionUtil;
import org.I0Itec.zkclient.exception.ZkNoNodeException;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.Watcher.Event.EventType;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 分布式队列，同步队列的实现
*
* @author linbingwen
*
* @param
*/
public class DistributedQueue {
private static Logger logger = LoggerFactory.getLogger(DistributedQueue.class);
protected final ZooKeeper zooKeeper;// 用于操作zookeeper集群
protected final String root;// 代表根节点
private int queueSize;
private String startPath = "/queue/start";
protected static final String Node_NAME = "n_";// 顺序节点的名称
public DistributedQueue(ZooKeeper zooKeeper, String root, int queueSize) {
this.zooKeeper = zooKeeper;
this.root = root;
this.queueSize = queueSize;
init();
}
/**
* 初始化根目录
*/
private void init() {
try {
Stat stat = zooKeeper.exists(root, false);// 判断一下根目录是否存在
if (stat == null) {
zooKeeper.create(root, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
zooKeeper.delete(startPath, -1); // 删除队列满的标志
} catch (Exception e) {
logger.error("create rootPath error", e);
}
}
/**
* 获取队列的大小
*
* @return
* @throws Exception
*/
public int size() throws Exception {
return zooKeeper.getChildren(root, false).size();
}
/**
* 判断队列是否为空
*
* @return
* @throws Exception
*/
public boolean isEmpty() throws Exception {
return zooKeeper.getChildren(root, false).size() == 0;
}
/**
* bytes 转object
*
* @param bytes
* @return
*/
private Object ByteToObject(byte[] bytes) {
Object obj = null;
try {
// bytearray to object
ByteArrayInputStream bi = new ByteArrayInputStream(bytes);
ObjectInputStream oi = new ObjectInputStream(bi);
obj = oi.readObject();
bi.close();
oi.close();
} catch (Exception e) {
logger.error("translation" + e.getMessage());
e.printStackTrace();
}
return obj;
}
/**
* Object 转byte
*
* @param obj
* @return
*/
private byte[] ObjectToByte(java.lang.Object obj) {
byte[] bytes = null;
try {
// object to bytearray
ByteArrayOutputStream bo = new ByteArrayOutputStream();
ObjectOutputStream oo = new ObjectOutputStream(bo);
oo.writeObject(obj);
bytes = bo.toByteArray();
bo.close();
oo.close();
} catch (Exception e) {
logger.error("translation" + e.getMessage());
e.printStackTrace();
}
return bytes;
}
/**
* 向队列提供数据,队列满的话会阻塞等待直到start标志位清除
*
* @param element
* @return
* @throws Exception
*/
public boolean offer(T element) throws Exception {
// 构建数据节点的完整路径
String nodeFullPath = root.concat("/").concat(Node_NAME);
try {
if (queueSize > size()) {
// 创建持久的节点，写入数据
zooKeeper.create(nodeFullPath, ObjectToByte(element), ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
// 再判断一下队列是否满
if (queueSize > size()) {
zooKeeper.delete(startPath, -1); // 确保不存在
} else {
zooKeeper.create(startPath, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
} else {
// 创建队列满的标记
if (zooKeeper.exists(startPath, false) != null) {
zooKeeper.create(startPath, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
final CountDownLatch latch = new CountDownLatch(1);
final Watcher previousListener = new Watcher() {
public void process(WatchedEvent event) {
if (event.getType() == EventType.NodeDeleted) {
latch.countDown();
}
}
};
// 如果节点不存在会出现异常
zooKeeper.exists(startPath, previousListener);
latch.await();
offer(element);
}
} catch (ZkNoNodeException e) {
logger.error("", e);
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
return true;
}
/**
* 从队列取数据,当有start标志位时，开始取数据，全部取完数据后才删除start标志
*
* @return
* @throws Exception
*/
@SuppressWarnings("unchecked")
public T poll() throws Exception {
try {
// 队列还没满
if (zooKeeper.exists(startPath, false) == null) {
final CountDownLatch latch = new CountDownLatch(1);
final Watcher previousListener = new Watcher() {
public void process(WatchedEvent event) {
if (event.getType() == EventType.NodeCreated) {
latch.countDown();
}
}
};
// 如果节点不存在会出现异常
zooKeeper.exists(startPath, previousListener);
// 如果节点不存在会出现异常
latch.await();
}
List list = zooKeeper.getChildren(root, false);
if (list.size() == 0) {
return null;
}
// 将队列按照由小到大的顺序排序
Collections.sort(list, new Comparator() {
public int compare(String lhs, String rhs) {
return getNodeNumber(lhs, Node_NAME).compareTo(getNodeNumber(rhs, Node_NAME));
}
});
/**
* 将队列中的元素做循环，然后构建完整的路径，在通过这个路径去读取数据
*/
for (String nodeName : list) {
String nodeFullPath = root.concat("/").concat(nodeName);
try {
T node = (T) ByteToObject(zooKeeper.getData(nodeFullPath, false, null));
zooKeeper.delete(nodeFullPath, -1);
return node;
} catch (ZkNoNodeException e) {
logger.error("", e);
}
}
return null;
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
}
/**
* 截取节点的数字的方法
*
* @param str
* @param nodeName
* @return
*/
private String getNodeNumber(String str, String nodeName) {
int index = str.lastIndexOf(nodeName);
if (index >= 0) {
index += Node_NAME.length();
return index <= str.length() ? str.substring(index) : "";
}
return str;
}
}
代码还没验证，可能会有问题！
（2）先进先出队列
在zookeeper中先创建一个根目录 queue_fifo，做为队列。入队操作就是在queue_fifo下创建自增序的子节点，并把数据放入节点内。出队操作就是先找到queue_fifo下序号最下的那个节点，取出数据，然后删除此节点。
实现代码：
package com.github.distribute.queue;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.I0Itec.zkclient.ExceptionUtil;
import org.I0Itec.zkclient.ZkClient;
import org.I0Itec.zkclient.exception.ZkNoNodeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.distribute.lock.zookeeper.BaseDistributedLock;
/**
* 分布式队列，生产者，消费者的实现
* @author linbingwen
*
* @param
*/
public class DistributedSimpleQueue {
private static Logger logger = LoggerFactory.getLogger(BaseDistributedLock.class);
protected final ZkClient zkClient;//用于操作zookeeper集群
protected final String root;//代表根节点
protected static final String Node_NAME = "n_";//顺序节点的名称
public DistributedSimpleQueue(ZkClient zkClient, String root) {
this.zkClient = zkClient;
this.root = root;
}
//获取队列的大小
public int size() {
/**
* 通过获取根节点下的子节点列表
*/
return zkClient.getChildren(root).size();
}
//判断队列是否为空
public boolean isEmpty() {
return zkClient.getChildren(root).size() == 0;
}
/**
* 向队列提供数据
* @param element
* @return
* @throws Exception
*/
public boolean offer(T element) throws Exception{
//构建数据节点的完整路径
String nodeFullPath = root .concat( "/" ).concat( Node_NAME );
try {
//创建持久的节点，写入数据
zkClient.createPersistentSequential(nodeFullPath , element);
}catch (ZkNoNodeException e) {
zkClient.createPersistent(root);
offer(element);
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
return true;
}
//从队列取数据
@SuppressWarnings("unchecked")
public T poll() throws Exception {
try {
List list = zkClient.getChildren(root);
if (list.size() == 0) {
return null;
}
//将队列按照由小到大的顺序排序
Collections.sort(list, new Comparator() {
public int compare(String lhs, String rhs) {
return getNodeNumber(lhs, Node_NAME).compareTo(getNodeNumber(rhs, Node_NAME));
}
});
/**
* 将队列中的元素做循环，然后构建完整的路径，在通过这个路径去读取数据
*/
for ( String nodeName : list ){
String nodeFullPath = root.concat("/").concat(nodeName);
try {
T node = (T) zkClient.readData(nodeFullPath);
zkClient.delete(nodeFullPath);
return node;
} catch (ZkNoNodeException e) {
logger.error("",e);
}
}
return null;
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
}
private String getNodeNumber(String str, String nodeName) {
int index = str.lastIndexOf(nodeName);
if (index >= 0) {
index += Node_NAME.length();
return index <= str.length() ? str.substring(index) : "";
}
return str;
}
}
测试一下：
package com.github.distribute.queue;
import java.io.Serializable;
import org.I0Itec.zkclient.ZkClient;
import org.I0Itec.zkclient.serialize.SerializableSerializer;
public class DistributedQueueTest {
public static void main(String[] args) {
ZkClient zkClient = new ZkClient("127.0.0.1:2181", 5000, 5000, new SerializableSerializer());
DistributedSimpleQueue queue = new DistributedSimpleQueue(zkClient, "/Queue");
new Thread(new ConsumerThread(queue)).start();
new Thread(new ProducerThread(queue)).start();
}
}
class ConsumerThread implements Runnable {
private DistributedSimpleQueue queue;
public ConsumerThread(DistributedSimpleQueue queue) {
this.queue = queue;
}
public void run() {
for (int i = 0; i < 10000; i++) {
try {
Thread.sleep((int) (Math.random() * 5000));// 随机睡眠一下
SendObject sendObject = (SendObject) queue.poll();
System.out.println("消费一条消息成功：" + sendObject);
} catch (Exception e) {
}
}
}
}
class ProducerThread implements Runnable {
private DistributedSimpleQueue queue;
public ProducerThread(DistributedSimpleQueue queue) {
this.queue = queue;
}
public void run() {
for (int i = 0; i < 10000; i++) {
try {
Thread.sleep((int) (Math.random() * 5000));// 随机睡眠一下
SendObject sendObject = new SendObject(String.valueOf(i), "content" + i);
queue.offer(sendObject);
System.out.println("发送一条消息成功：" + sendObject);
} catch (Exception e) {
}
}
}
}
class SendObject implements Serializable {
private static final long serialVersionUID = 1L;
public SendObject(String id, String content) {
this.id = id;
this.content = content;
}
private String id;
private String content;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
@Override
public String toString() {
return "SendObject [id=" + id + ", content=" + content + "]";
}
}
输出结果：
本文源码请在这里下载：https://github.com/appleappleapple/DistributeLearning
更多技术请关注笔者微信技术公众号"单例模式"
