Zookeeper 是一个开源的分布式协调服务，由知名互联网公司雅虎公司创建，是Google Chubby 开源实现。 他致力于提供一个高性能，高可用，且具有严格的顺序访问控制能力的分布式协调服务。分布式应用可以基于诸如数据发布/订阅、负载均衡、命名服务、分布式协调/通知，集群管理、Master选举、分布式锁和分布式队列。
Master 选举
架构图
Master 选举
程序流程图
Master 选举流程图
当出现网络抖动，master 节点掉线，需要进行 master 重新选举
网络抖动
master 节点何时会删除
- master 会主动放权
- master 节点down机
- master 节点网络斗动，会发生新一轮选举
- 如果master 和上一轮选举的master 不是同一个节点，不一致时发生资源迁移
优化：让上一轮选举出的master 再新一轮选举中优先采用 master 节点
代码示例
public class LeaderSelectorZkClient {
//启动的服务个数
private static final int CLIENT_QTY = 10;
//zookeeper服务器的地址
private static final String ZOOKEEPER_SERVER = "127.o.o.1:2181,127.o.o.1:2182,127.o.o.1:2183";
public static void main(String[] args) throws Exception {
//保存所有zkClient的列表
List clients = new ArrayList<>();
//保存所有服务的列表
List workServers = new ArrayList<>();
try {
for (int i = 0; i < CLIENT_QTY; ++i) {
//创建zkClient
ZkClient client = new ZkClient(ZOOKEEPER_SERVER, 15000, 105000, new SerializableSerializer());
clients.add(client);
//创建serverData
RunningData runningData = new RunningData();
runningData.setCid(Long.valueOf(i));
runningData.setName("Client #" + i);
//创建服务
WorkServer workServer = new WorkServer(runningData);
workServer.setZkClient(client);
workServers.add(workServer);
workServer.start();
}
System.out.println("敲回车键退出！\n");
new BufferedReader(new InputStreamReader(System.in)).readLine();
} finally {
System.out.println("Shutting down...");
for (WorkServer workServer : workServers) {
try {
workServer.stop();
} catch (Exception e) {
e.printStackTrace();
}
}
for (ZkClient client : clients) {
try {
client.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
@Data
public class RunningData implements Serializable {
private Long cid;
private String name;
}
public class WorkServer {
private volatile boolean running = false;
private ZkClient zkClient;
private static final String MASTER_PATH = "/master";
private IZkDataListener dataListener;
private RunningData serverData;
private RunningData masterData;
private ScheduledExecutorService delayExecutor = Executors.newScheduledThreadPool(1);
private int delayTime = 5;
public WorkServer(RunningData rd) {
this.serverData = rd;
this.dataListener = new IZkDataListener() {
public void handleDataDeleted(String dataPath) throws Exception {
//takeMaster();
if (masterData != null && masterData.getName().equals(serverData.getName())) {
takeMaster();
} else {
delayExecutor.schedule(() -> takeMaster(), delayTime, TimeUnit.SECONDS);
}
}
public void handleDataChange(String dataPath, Object data) throws Exception {
}
};
}
public void setZkClient(ZkClient zkClient) {
this.zkClient = zkClient;
}
public void start() throws Exception {
if (running) {
throw new Exception("server has startup...");
}
running = true;
zkClient.subscribeDataChanges(MASTER_PATH, dataListener);
takeMaster();
}
public void stop() throws Exception {
if (!running) {
throw new Exception("server has stopped");
}
running = false;
delayExecutor.shutdown();
zkClient.unsubscribeDataChanges(MASTER_PATH, dataListener);
releaseMaster();
}
private void takeMaster() {
if (!running)
return;
try {
zkClient.create(MASTER_PATH, serverData, CreateMode.EPHEMERAL);
masterData = serverData;
System.out.println(serverData.getName() + " is master");
delayExecutor.schedule(() -> {
if (checkMaster()) {
releaseMaster();
}
}, 5, TimeUnit.SECONDS);
} catch (ZkNodeExistsException e) {
RunningData runningData = zkClient.readData(MASTER_PATH, true);
if (runningData == null) {
takeMaster();
} else {
masterData = runningData;
}
} catch (Exception e) {
// ignore;
}
}
private void releaseMaster() {
if (checkMaster()) {
zkClient.delete(MASTER_PATH);
}
}
private boolean checkMaster() {
try {
RunningData eventData = zkClient.readData(MASTER_PATH);
masterData = eventData;
if (masterData.getName().equals(serverData.getName())) {
return true;
}
return false;
} catch (ZkNoNodeException e) {
return false;
} catch (ZkInterruptedException e) {
return checkMaster();
} catch (ZkException e) {
return false;
}
}
}
发布/订阅
架构图
架构图
Manager Server 流程图
Manager Server 流程图
WorkServer 流程图
WorkServer 流程图
代码示例
服务器信息
@Data
public class ServerData {
private String address;
private Integer id;
private String name;
}
服务器中的配置数据
@Data
public class ServerConfig {
private String dbUrl;
private String dbPwd;
private String dbUser;
}
WorkServer
@Slf4j
public class WorkServer {
private ZkClient zkClient;
private String configPath;
private String serversPath;
private ServerData serverData;
private ServerConfig serverConfig;
private IZkDataListener dataListener;
public WorkServer(String configPath, String serversPath,
ServerData serverData, ZkClient zkClient, ServerConfig initConfig) {
this.zkClient = zkClient;
this.serversPath = serversPath;
this.configPath = configPath;
this.serverConfig = initConfig;
this.serverData = serverData;
// 当 /config 目录下的数据变化时，更新自己的配置信息
this.dataListener = new IZkDataListener() {
public void handleDataDeleted(String dataPath) {
}
public void handleDataChange(String dataPath, Object data) {
String retJson = new String((byte[])data);
ServerConfig serverConfigLocal = JSON.parseObject(retJson, ServerConfig.class);
updateConfig(serverConfigLocal);
log.info("new Work server config is:" + serverConfig.toString());
}
};
}
/**
* 启动 WorkServer，在 zk 的 /servers 节点下注册自己的 ip 地址，并写入自己的 serverData 信息
* 并且监听了 /config 节点的数据变化
*/
public void start() {
log.info("work server start...");
initRunning();
}
public void stop() {
log.info("work server stop...");
zkClient.unsubscribeDataChanges(configPath, dataListener);
}
private void initRunning() {
registerMe();
zkClient.subscribeDataChanges(configPath, dataListener);
}
private void registerMe() {
String mePath = serversPath.concat("/").concat(serverData.getAddress());
try {
zkClient.createEphemeral(mePath, JSON.toJSONString(serverData).getBytes());
} catch (ZkNoNodeException e) {
zkClient.createPersistent(serversPath, true);
registerMe();
}
}
private void updateConfig(ServerConfig serverConfig) {
this.serverConfig = serverConfig;
}
}
ManagerServer
@Slf4j
public class ManageServer {
private String serversPath;
private String commandPath;
private String configPath;
private ZkClient zkClient;
private ServerConfig config;
private IZkChildListener childListener;
private IZkDataListener dataListener;
private List workServerList;
public ManageServer(String serversPath, String commandPath,
String configPath, ZkClient zkClient, ServerConfig config) {
this.serversPath = serversPath;
this.commandPath = commandPath;
this.zkClient = zkClient;
this.config = config;
this.configPath = configPath;
// 如果 /servers 节点变化(WorkServer 宕机或新增)
this.childListener = (parentPath, currentChildren) -> {
workServerList = currentChildren;
log.info("work server list changed, new list is ");
execList();
};
// 如果 /command 节点变化根据 /command 内容执行相关命令
this.dataListener = new IZkDataListener() {
public void handleDataDeleted(String dataPath) {
// ignore;
}
public void handleDataChange(String dataPath, Object data) {
String cmd = new String((byte[]) data);
log.info("cmd:" + cmd);
exeCmd(cmd);
}
};
}
/**
* 订阅 /command、/servers 节点
*/
private void initRunning() {
zkClient.subscribeDataChanges(commandPath, dataListener);
zkClient.subscribeChildChanges(serversPath, childListener);
}
/*
* 1: list 2: create 3: modify
*/
private void exeCmd(String cmdType) {
if ("list".equals(cmdType)) {
execList();
} else if ("create".equals(cmdType)) {
execCreate();
} else if ("modify".equals(cmdType)) {
execModify();
} else {
log.info("error command!" + cmdType);
}
}
private void execList() {
log.info(workServerList.toString());
}
private void execCreate() {
if (!zkClient.exists(configPath)) {
try {
zkClient.createPersistent(configPath, JSON.toJSONString(config).getBytes());
} catch (ZkNodeExistsException e) {
zkClient.writeData(configPath, JSON.toJSONString(config).getBytes());
} catch (ZkNoNodeException e) {
String parentDir = configPath.substring(0, configPath.lastIndexOf('/'));
zkClient.createPersistent(parentDir, true);
execCreate();
}
}
}
private void execModify() {
config.setDbUser(config.getDbUser() + "_modify");
try {
zkClient.writeData(configPath, JSON.toJSONString(config).getBytes());
} catch (ZkNoNodeException e) {
execCreate();
}
}
public void start() {
initRunning();
}
public void stop() {
zkClient.unsubscribeChildChanges(serversPath, childListener);
zkClient.unsubscribeDataChanges(commandPath, dataListener);
}
}
启动类
@Slf4j
public class SubscribeZkClient {
private static final int CLIENT_QTY = 5;
private static final String ZOOKEEPER_SERVER = "39.106.111.160:2181,39.106.111.160:2182,39.106.111.160:2183";
private static final String CONFIG_PATH = "/config";
private static final String COMMAND_PATH = "/command";
private static final String SERVERS_PATH = "/servers";
public static void main(String[] args) throws Exception {
List clients = new ArrayList<>();
List workServers = new ArrayList<>();
ManageServer manageServer;
try {
// 初始化的配置信息
ServerConfig initConfig = new ServerConfig();
initConfig.setDbPwd("123456");
initConfig.setDbUrl("jdbc:mysql://localhost:3306/mydb");
initConfig.setDbUser("root");
ZkClient clientManage = new ZkClient(ZOOKEEPER_SERVER, 5000, 5000, new BytesPushThroughSerializer());
manageServer = new ManageServer(SERVERS_PATH, COMMAND_PATH, CONFIG_PATH, clientManage, initConfig);
manageServer.start();
for (int i = 0; i < CLIENT_QTY; ++i) {
ZkClient client = new ZkClient(ZOOKEEPER_SERVER, 5000, 5000, new BytesPushThroughSerializer());
clients.add(client);
ServerData serverData = new ServerData();
serverData.setId(i);
serverData.setName("WorkServer#" + i);
serverData.setAddress("192.168.1." + i);
WorkServer workServer = new WorkServer(CONFIG_PATH, SERVERS_PATH, serverData, client, initConfig);
workServers.add(workServer);
workServer.start();
}
log.info("敲回车键退出！");
new BufferedReader(new InputStreamReader(System.in)).readLine();
} finally {
log.info("Shutting down...");
for (WorkServer workServer : workServers) {
try {
workServer.stop();
} catch (Exception e) {
e.printStackTrace();
}
}
for (ZkClient client : clients) {
try {
client.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
负载均衡
架构图
架构图
客户端流程
客户端流程
服务端主体流程图
服务端主体流程图
命名服务
架构图
架构图
核心算法
核心算法
代码示例
@Slf4j
public class IdMaker {
private ZkClient client = null;
private final String server;
private final String root;
private final String nodeName;
private volatile boolean running = false;
private ExecutorService cleanExecutor = null;
public enum RemoveMethod{
NONE, IMMEDIATELY, DELAY
}
public IdMaker(String zkServer,String root, String nodeName){
this.root = root;
this.server = zkServer;
this.nodeName = nodeName;
}
public void start() throws Exception {
if (running)
throw new Exception("server has stated...");
running = true;
init();
}
public void stop() throws Exception {
if (!running)
throw new Exception("server has stopped...");
running = false;
freeResource();
}
private void init(){
client = new ZkClient(server,5000,5000,new BytesPushThroughSerializer());
cleanExecutor = Executors.newFixedThreadPool(10);
try{
client.createPersistent(root,true);
}catch (ZkNodeExistsException e){
//ignore;
}
}
private void freeResource(){
cleanExecutor.shutdown();
try{
cleanExecutor.awaitTermination(2, TimeUnit.SECONDS);
}catch(InterruptedException e){
e.printStackTrace();
}finally{
cleanExecutor = null;
}
if (client!=null){
client.close();
client=null;
}
}
private void checkRunning() throws Exception {
if (!running)
throw new Exception("请先调用start");
}
private String ExtractId(String str){
int index = str.lastIndexOf(nodeName);
if (index >= 0){
index += nodeName.length();
return index <= str.length() ? str.substring(index): "";
}
return str;
}
public String generateId(RemoveMethod removeMethod) throws Exception{
checkRunning();
final String fullNodePath = root.concat("/").concat(nodeName);
final String ourPath = client.createPersistentSequential(fullNodePath, null);
if (removeMethod.equals(RemoveMethod.IMMEDIATELY)){
client.delete(ourPath);
}else if (removeMethod.equals(RemoveMethod.DELAY)){
cleanExecutor.execute(() -> client.delete(ourPath));
}
//node-0000000000, node-0000000001
return ExtractId(ourPath);
}
}
@Slf4j
public class TestIdMaker {
public static void main(String[] args) throws Exception {
IdMaker idMaker = new IdMaker("localhost:2181",
"/NameService/IdGen", "ID");
idMaker.start();
try {
for (int i = 0; i < 10; i++) {
String id = idMaker.generateId(IdMaker.RemoveMethod.DELAY);
log.info(id);
}
} finally {
idMaker.stop();
}
}
}
分布式队列
架构图
架构图
offer 核心算法
offer 核心算法
poll 核心算法
poll 核心算法
核心代码
简单的队列
@Slf4j
public class DistributedSimpleQueue {
protected final ZkClient zkClient;
protected final String root;
protected static final String Node_NAME = "n_";
public DistributedSimpleQueue(ZkClient zkClient, String root) {
this.zkClient = zkClient;
this.root = root;
}
public int size() {
return zkClient.getChildren(root).size();
}
public boolean isEmpty() {
return zkClient.getChildren(root).size() == 0;
}
public boolean offer(T element) {
String nodeFullPath = root.concat("/").concat(Node_NAME);
try {
zkClient.createPersistentSequential(nodeFullPath, element);
} catch (ZkNoNodeException e) {
zkClient.createPersistent(root);
offer(element);
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
return true;
}
@SuppressWarnings("unchecked")
public T poll() throws Exception {
try {
List list = zkClient.getChildren(root);
if (list.size() == 0) {
return null;
}
Collections.sort(list, Comparator.comparing((s) -> getNodeNumber(s, Node_NAME)));
for (String nodeName : list) {
String nodeFullPath = root.concat("/").concat(nodeName);
try {
T node = zkClient.readData(nodeFullPath);
zkClient.delete(nodeFullPath);
return node;
} catch (ZkNoNodeException e) {
// ignore
}
}
return null;
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
}
private String getNodeNumber(String str, String nodeName) {
int index = str.lastIndexOf(nodeName);
if (index >= 0) {
index += Node_NAME.length();
return index <= str.length() ? str.substring(index) : "";
}
return str;
}
}
扩展队列
public class DistributedBlockingQueue extends DistributedSimpleQueue {
public DistributedBlockingQueue(ZkClient zkClient, String root) {
super(zkClient, root);
}
@Override
public T poll() throws Exception {
while (true) {
final CountDownLatch latch = new CountDownLatch(1);
final IZkChildListener childListener = (parentPath, currentChilds) -> latch.countDown();
zkClient.subscribeChildChanges(root, childListener);
try {
T node = super.poll();
if (node != null) {
return node;
} else {
latch.await();
}
} finally {
zkClient.unsubscribeChildChanges(root, childListener);
}
}
}
}
测试类
@Slf4j
public class TestDistributedBlockingQueue {
public static void main(String[] args) {
ScheduledExecutorService delayExecutor = Executors.newScheduledThreadPool(1);
int delayTime = 5;
ZkClient zkClient = new ZkClient("localhost:2181", 5000, 5000, new SerializableSerializer());
final DistributedBlockingQueue queue = new DistributedBlockingQueue<>(zkClient, "/Queue");
final User user1 = new User();
user1.setId("1");
user1.setName("xiao wang");
final User user2 = new User();
user2.setId("2");
user2.setName("xiao wang");
try {
delayExecutor.schedule(() -> {
try {
queue.offer(user1);
queue.offer(user2);
} catch (Exception e) {
e.printStackTrace();
}
}, delayTime, TimeUnit.SECONDS);
log.info("ready poll!");
User u1 = queue.poll();
User u2 = queue.poll();
if (user1.getId().equals(u1.getId()) && user2.getId().equals(u2.getId())) {
log.info("Success!");
}
} catch (Exception e) {
e.printStackTrace();
} finally {
delayExecutor.shutdown();
try {
delayExecutor.awaitTermination(2, TimeUnit.SECONDS);
} catch (InterruptedException e) {
}
}
}
}
分布式锁
架构图
架构图
核心算法流程图
核心算法流程图