zookeeper C API 完整运行实例
介绍下每个组件代码流摘要
Master:
1、获取leadership前2、尝试创建主znode
3、如果通过，则获取leadership
4、连接丢失后，需要检查znode是否存在以及谁拥有它
5、一旦确定他人拥有它，Watch主znode
6、获取leadership后
7、Get workers
8、在worker list上设置一个watcher
9、检查挂掉的worker并重新分配任务
10、遍历各个挂掉的worker
11、获取分配的任务
12、获取任务数据
13、将任务移至未分配的任务列表
14、删除分配
15、恢复任务（分配给死亡工人的任务）
16、获取未分配的任务并分配它们
17、遍历各个未分配的任务
18、获取任务数据
19、选择worker
20、分配给worker
21、从未分配的列表中删除任务
Worker:
1、创建/分配/ worker-xxx znode2、创建/ workers / worker-xxx znode
3、Watches /assign/worker-xxx znode
4、获得任务
5、对于每个任务，获取任务数据
6、执行任务数据
7、创建状态
8、删除分配
Client
1、创建任务2、Watch for status znode
3、在收到状态znode的通知后，获取状态数据
4、删除状态znode
编译：
编译需要依赖zookeeper的多线程c库zookeeper_mt,所以在编译前需要先安装其C API
进入./zookeeper/src/c目录
./configure
make
make install
然后编译当前代码：
gcc master.c -I/usr/local/zookeeper-3.4.8/src/c/include -I/usr/local/zookeeper-3.4.8/src/c/generated -DTHREADED -L/usr/local/lib -l zookeeper_mt
运行
先开启zookeeper：
bin/zkServer.sh start
运行程序：
./a.out 127.0.0.1:2181
master.h
#ifndef _master_h
#define _master_h
#include
#include
#include
#include
#include
#include
#include
#include
#include
static const char *hostPort;
static zhandle_t *zh;
static int connected = 0;
static int expired = 0;
static int server_id;
static struct String_vector* workers = NULL;
static struct String_vector* tasks = NULL;
/*
* 函数定义
*/
void create_parent();
void run_for_master();
void check_master();
void master_exists();
void get_workers();
void get_tasks();
void task_assignment();
void get_task_data();
void delete_pending_task();
/*
* Master states
*/
enum master_states {
RUNNING,
ELECTED,
NOTELECTED
};
static enum master_states state;
enum master_states get_state () {
return state;
}
/*
* 将此结构的实例分配给worker时使用
*/
struct task_info {
char * name;
char * value;
int value_len;
char * worker;
};
/*
* 以下两种方法分别把事件类型和返回码转换为字符串
*/
static const char * type2string(int type){
if (type == ZOO_CREATED_EVENT)
return "CREATED_EVENT";
if (type == ZOO_DELETED_EVENT)
return "DELETED_EVENT";
if (type == ZOO_CHANGED_EVENT)
return "CHANGED_EVENT";
if (type == ZOO_CHILD_EVENT)
return "CHILD_EVENT";
if (type == ZOO_SESSION_EVENT)
return "SESSION_EVENT";
if (type == ZOO_NOTWATCHING_EVENT)
return "NOTWATCHING_EVENT";
return "UNKNOWN_EVENT_TYPE";
}
static const char * rc2string(int rc){
if (rc == ZOK) {
return "OK";
}
if (rc == ZSYSTEMERROR) {
return "System error";
}
if (rc == ZRUNTIMEINCONSISTENCY) {
return "Runtime inconsistency";
}
if (rc == ZDATAINCONSISTENCY) {
return "Data inconsistency";
}
if (rc == ZCONNECTIONLOSS) {
return "Connection to the server has been lost";
}
if (rc == ZMARSHALLINGERROR) {
return "Error while marshalling or unmarshalling data ";
}
if (rc == ZUNIMPLEMENTED) {
return "Operation not implemented";
}
if (rc == ZOPERATIONTIMEOUT) {
return "Operation timeout";
}
if (rc == ZBADARGUMENTS) {
return "Invalid argument";
}
if (rc == ZINVALIDSTATE) {
return "Invalid zhandle state";
}
if (rc == ZAPIERROR) {
return "API error";
}
if (rc == ZNONODE) {
return "Znode does not exist";
}
if (rc == ZNOAUTH) {
return "Not authenticated";
}
if (rc == ZBADVERSION) {
return "Version conflict";
}
if (rc == ZNOCHILDRENFOREPHEMERALS) {
return "Ephemeral nodes may not have children";
}
if (rc == ZNODEEXISTS) {
return "Znode already exists";
}
if (rc == ZNOTEMPTY) {
return "The znode has children";
}
if (rc == ZSESSIONEXPIRED) {
return "The session has been expired by the server";
}
if (rc == ZINVALIDCALLBACK) {
return "Invalid callback specified";
}
if (rc == ZINVALIDACL) {
return "Invalid ACL specified";
}
if (rc == ZAUTHFAILED) {
return "Client authentication failed";
}
if (rc == ZCLOSING) {
return "ZooKeeper session is closing";
}
if (rc == ZNOTHING) {
return "No response from server";
}
if (rc == ZSESSIONMOVED) {
return "Session moved to a different server";
}
return "UNKNOWN_EVENT_TYPE";
}
#endif
master.c
#include "master.h"
/*
* 辅助函数
*/
char * make_path(int num, ...) {
const char * tmp_string;
va_list arguments;
va_start ( arguments, num );
int total_length = 0;
int x;
for ( x = 0; x < num; x++ ) {
tmp_string = va_arg ( arguments, const char * );
if(tmp_string != NULL) {
LOG_DEBUG(("Counting path with this path %s (%d)", tmp_string, num));
total_length += strlen(tmp_string);
}
}
va_end ( arguments );
char * path = malloc(total_length * sizeof(char) + 1);
path[0] = '\0';
va_start ( arguments, num );
for ( x = 0; x < num; x++ ) {
tmp_string = va_arg ( arguments, const char * );
if(tmp_string != NULL) {
LOG_DEBUG(("Counting path with this path %s",
tmp_string));
strcat(path, tmp_string);
}
}
return path;
}
struct String_vector* make_copy( const struct String_vector* vector ) {
struct String_vector* tmp_vector = malloc(sizeof(struct String_vector));
tmp_vector->data = malloc(vector->count * sizeof(const char *));
tmp_vector->count = vector->count;
int i;
for( i = 0; i < vector->count; i++) {
tmp_vector->data[i] = strdup(vector->data[i]);
}
return tmp_vector;
}
/*
* 分配String_vector，从zookeeper.jute.c复制
*/
int allocate_vector(struct String_vector *v, int32_t len) {
if (!len) {
v->count = 0;
v->data = 0;
} else {
v->count = len;
v->data = calloc(sizeof(*v->data), len);
}
return 0;
}
/*
* 释放内存的函数
*/
void free_vector(struct String_vector* vector) {
int i;
// Free each string
for(i = 0; i < vector->count; i++) {
free(vector->data[i]);
}
// Free data
free(vector -> data);
// Free the struct
free(vector);
}
void free_task_info(struct task_info* task) {
free(task->name);
free(task->value);
free(task->worker);
free(task);
}
/*
* 处理workers和任务缓存的函数
*/
int contains(const char * child, const struct String_vector* children) {
int i;
for(i = 0; i < children->count; i++) {
if(!strcmp(child, children->data[i])) {
return 1;
}
}
return 0;
}
/*
* 此函数返回当前与之前相比是新的元素，并更新之前的元素
*/
struct String_vector* added_and_set(const struct String_vector* current,
struct String_vector** previous) {
struct String_vector* diff = malloc(sizeof(struct String_vector));
int count = 0;
int i;
for(i = 0; i < current->count; i++) {
if (!contains(current->data[i], (*previous))) {
count++;
}
}
allocate_vector(diff, count);
int prev_count = count;
count = 0;
for(i = 0; i < current->count; i++) {
if (!contains(current->data[i], (* previous))) {
diff->data[count] = malloc(sizeof(char) * strlen(current->data[i]) + 1);
memcpy(diff->data[count++],
current->data[i],
strlen(current->data[i]));
}
}
assert(prev_count == count);
free_vector((struct String_vector*) *previous);
(*previous) = make_copy(current);
return diff;
}
/*
* 此函数返回与之前想比较已被删除的元素，并更新之前的元素
*/
struct String_vector* removed_and_set(const struct String_vector* current,
struct String_vector** previous) {
struct String_vector* diff = malloc(sizeof(struct String_vector));
int count = 0;
int i;
for(i = 0; i < (* previous)->count; i++) {
if (!contains((* previous)->data[i], current)) {
count++;
}
}
allocate_vector(diff, count);
int prev_count = count;
count = 0;
for(i = 0; i < (* previous)->count; i++) {
if (!contains((* previous)->data[i], current)) {
diff->data[count] = malloc(sizeof(char) * strlen((* previous)->data[i]));
strcpy(diff->data[count++], (* previous)->data[i]);
}
}
assert(prev_count == count);
free_vector((struct String_vector*) *previous);
(*previous) = make_copy(current);
return diff;
}
/*
* 辅助函数结束, 以上都与zookeeper相关
*/
/**
* 我们使用Watcher来处理会话事件。特别是，当它收到一个ZOO_CONNECTED_STATE事件时
* 我们设置连接变量，以便知道连接已建立
*/
void main_watcher (zhandle_t *zkh,
int type,
int state,
const char *path,
void* context)
{
/*
* zookeeper_init 可能没有返回，所以我们使用zkh代替
*/
if (type == ZOO_SESSION_EVENT) {
if (state == ZOO_CONNECTED_STATE) {
connected = 1;
LOG_DEBUG(("Received a connected event."));
} else if (state == ZOO_CONNECTING_STATE) {
if(connected == 1) {
LOG_WARN(("Disconnected."));
}
connected = 0;
} else if (state == ZOO_EXPIRED_SESSION_STATE) {
expired = 1;
connected = 0;
zookeeper_close(zkh);
}
}
LOG_DEBUG(("Event: %s, %d", type2string(type), state));
}
int is_connected() {
return connected;
}
int is_expired() {
return expired;
}
/**
*
* 分配任务，但首先读取任务数据。在这个简单的实现中，
* 在znode中没有真正的任务数据，但是我们包含了获取数据的例子
*/
void assign_tasks(const struct String_vector *strings) {
/*
* For each task, assign it to a worker.
*/
LOG_DEBUG(("Task count: %d", strings->count));
int i;
for( i = 0; i < strings->count; i++) {
LOG_DEBUG(("Assigning task %s",
(char *) strings->data[i]));
get_task_data( strings->data[i] );
}
}
/**
*
* 在调用获取任务列表的时候调用完成函数
*
*/
void tasks_completion (int rc,
const struct String_vector *strings,
const void *data) {
switch (rc) {
case ZCONNECTIONLOSS:
case ZOPERATIONTIMEOUT:
get_tasks();
break;
case ZOK:
LOG_DEBUG(("Assigning tasks"));
struct String_vector *tmp_tasks = added_and_set(strings, &tasks);
assign_tasks(tmp_tasks);
free_vector(tmp_tasks);
break;
default:
LOG_ERROR(("Something went wrong when checking tasks: %s", rc2string(rc)));
break;
}
}
/**
*
* 当任务列表更改时，调用watcher函数
*
*/
void tasks_watcher (zhandle_t *zh,
int type,
int state,
const char *path,
void *watcherCtx) {
LOG_DEBUG(("Tasks watcher triggered %s %d", path, state));
if( type == ZOO_CHILD_EVENT) {
assert( !strcmp(path, "/tasks") );
get_tasks();
} else {
LOG_INFO(("Watched event: %s", type2string(type)));
}
LOG_DEBUG(("Tasks watcher done"));
}
static int task_count = 0;
/**
*
* 获取任务列表
*
*/
void get_tasks () {
LOG_DEBUG(("Getting tasks"));
zoo_awget_children(zh,
"/tasks",
tasks_watcher,
NULL,
tasks_completion,
NULL);
}
/**
*
* 在删除任务znode的请求返回时调用完成函数
*
*/
void delete_task_completion(int rc, const void *data) {
switch (rc) {
case ZCONNECTIONLOSS:
case ZOPERATIONTIMEOUT:
delete_pending_task((const char *) data);
break;
case ZOK:
LOG_DEBUG(("Deleted task: %s", (char *) data));
free((char *) data);
break;
default:
LOG_ERROR(("Something went wrong when deleting task: %s",
rc2string(rc)));
break;
}
}
/**
*
* 一旦分配完成，删除暂挂任务
*
*/
void delete_pending_task (const char * path) {
if(path == NULL) return;
char * tmp_path = strdup(path);
zoo_adelete(zh,
tmp_path,
-1,
delete_task_completion,
(const void*) tmp_path);
}
void workers_completion (int rc,
const struct String_vector *strings,
const void *data) {
switch (rc) {
case ZCONNECTIONLOSS:
case ZOPERATIONTIMEOUT:
get_workers();
break;
case ZOK:
if(strings->count == 1) {
LOG_DEBUG(("Got %d worker", strings->count));
} else {
LOG_DEBUG(("Got %d workers", strings->count));
}
struct String_vector *tmp_workers = removed_and_set(strings, &workers);
free_vector(tmp_workers);
get_tasks();
break;
default:
LOG_ERROR(("Something went wrong when checking workers: %s", rc2string(rc)));
break;
}
}
void workers_watcher (zhandle_t *zh, int type, int state, const char *path,void *watcherCtx) {
if( type == ZOO_CHILD_EVENT) {
assert( !strcmp(path, "/workers") );
get_workers();
} else {
LOG_DEBUG(("Watched event: ", type2string(type)));
}
}
void get_workers() {
zoo_awget_children(zh,
"/workers",
workers_watcher,
NULL,
workers_completion,
NULL);
}
void take_leadership() {
get_workers();
}
/*
* 在尝试 创建master lock时发生错误，
* 我们需要检查znode是否存在并确认其内容
*/
void master_check_completion (int rc, const char *value, int value_len,
const struct Stat *stat, const void *data) {
int master_id;
switch (rc) {
case ZCONNECTIONLOSS:
case ZOPERATIONTIMEOUT:
check_master();
break;
case ZOK:
sscanf(value, "%x", &master_id );
if(master_id == server_id) {
take_leadership();
LOG_DEBUG(("Elected primary master"));
} else {
master_exists();
LOG_DEBUG(("The primary is some other process"));
}
break;
case ZNONODE:
run_for_master();
break;
default:
LOG_ERROR(("Something went wrong when checking the master lock: %s", rc2string(rc)));
break;
}
}
void check_master () {
zoo_aget(zh,
"/master",
0,
master_check_completion,
NULL);
}
void task_assignment_completion (int rc, const char *value, const void *data) {
switch (rc) {
case ZCONNECTIONLOSS:
case ZOPERATIONTIMEOUT:
task_assignment((struct task_info*) data);
break;
case ZOK:
if(data != NULL) {
/*
* 从挂起列表中删除任务
*/
LOG_DEBUG(("Deleting pending task %s",
((struct task_info*) data)->name));
char * del_path = "";
del_path = make_path(2, "/tasks/", ((struct task_info*) data)->name);
if(del_path != NULL) {
delete_pending_task(del_path);
}
free(del_path);
free_task_info((struct task_info*) data);
}
break;
case ZNODEEXISTS:
LOG_DEBUG(("Assignment has already been created: %s", value));
break;
default:
LOG_ERROR(("Something went wrong when checking assignment completion: %s", rc2string(rc)));
break;
}
}
void task_assignment(struct task_info *task){
//将任务添加到工作列表
char * path = make_path(4, "/assign/" , task->worker, "/", task->name);
zoo_acreate(zh,
path,
task->value,
task->value_len,
&ZOO_OPEN_ACL_UNSAFE,
0,
task_assignment_completion,
(const void*) task);
free(path);
}
void get_task_data_completion(int rc, const char *value, int value_len,
const struct Stat *stat, const void *data) {
int worker_index;
switch (rc) {
case ZCONNECTIONLOSS:
case ZOPERATIONTIMEOUT:
get_task_data((const char *) data);
break;
case ZOK:
LOG_DEBUG(("Choosing worker for task %s", (const char *) data));
if(workers != NULL) {
/*
* Choose worker
*/
worker_index = (rand() % workers->count);
LOG_DEBUG(("Chosen worker %d %d",
worker_index, workers->count));
/*
*分配任务给worker
*/
struct task_info *new_task;
new_task = (struct task_info*) malloc(sizeof(struct task_info));
new_task->name = (char *) data;
new_task->value = strndup(value, value_len);
new_task->value_len = value_len;
const char * worker_string = workers->data[worker_index];
new_task->worker = strdup(worker_string);
LOG_DEBUG(("Ready to assign it %d, %s",
worker_index,
workers->data[worker_index]));
task_assignment(new_task);
}
break;
default:
LOG_ERROR(("Something went wrong when checking the master lock: %s",
rc2string(rc)));
break;
}
}
void get_task_data(const char *task) {
if(task == NULL) return;
LOG_DEBUG(("Task path: %s",
task));
char * tmp_task = strndup(task, 15);
char * path = make_path(2, "/tasks/", tmp_task);
LOG_DEBUG(("Getting task data %s",
tmp_task));
zoo_aget(zh,
path,
0,
get_task_data_completion,
(const void *) tmp_task);
free(path);
}
/*
* Run for master.
*/
void run_for_master();
void master_exists_watcher (zhandle_t *zh,
int type,
int state,
const char *path,
void *watcherCtx) {
if( type == ZOO_DELETED_EVENT) {
assert( !strcmp(path, "/master") );
run_for_master();
} else {
LOG_DEBUG(("Watched event: ", type2string(type)));
}
}
void master_exists_completion (int rc, const struct Stat *stat, const void *data) {
switch (rc) {
case ZCONNECTIONLOSS:
case ZOPERATIONTIMEOUT:
master_exists();
break;
case ZOK:
break;
case ZNONODE:
LOG_INFO(("Previous master is gone, running for master"));
run_for_master();
break;
default:
LOG_WARN(("Something went wrong when executing exists: %s", rc2string(rc)));
break;
}
}
void master_exists() {
zoo_awexists(zh,
"/master",
master_exists_watcher,
NULL,
master_exists_completion,
NULL);
}
void master_create_completion (int rc, const char *value, const void *data) {
switch (rc) {
case ZCONNECTIONLOSS:
check_master();
break;
case ZOK:
take_leadership();
break;
case ZNODEEXISTS:
master_exists();
break;
default:
LOG_ERROR(("Something went wrong when running for master."));
break;
}
}
void run_for_master() {
if(!connected) {
LOG_WARN(("Client not connected to ZooKeeper"));
return;
}
char server_id_string[9];
snprintf(server_id_string, 9, "%x", server_id);
zoo_acreate(zh,
"/master",
(const char *) server_id_string,
strlen(server_id_string) + 1,
&ZOO_OPEN_ACL_UNSAFE,
ZOO_EPHEMERAL,
master_create_completion,
NULL);
}
/*
* 创建父节点
*/
void create_parent_completion (int rc, const char * value, const void * data) {
switch (rc) {
case ZCONNECTIONLOSS:
create_parent(value, (const char *) data);
break;
case ZOK:
LOG_INFO(("Created parent node", value));
break;
case ZNODEEXISTS:
LOG_WARN(("Node already exists"));
break;
default:
LOG_ERROR(("Something went wrong when running for master: %s, %s", value, rc2string(rc)));
break;
}
}
void create_parent(const char * path, const char * value) {
zoo_acreate(zh,
path,
value,
0,
&ZOO_OPEN_ACL_UNSAFE,
0,
create_parent_completion,
NULL);
}
void bootstrap() {
if(!connected) {
LOG_WARN(("Client not connected to ZooKeeper"));
return;
}
create_parent("/workers", "");
create_parent("/assign", "");
create_parent("/tasks", "");
create_parent("/status", "");
// Initialize tasks
tasks = malloc(sizeof(struct String_vector));
allocate_vector(tasks, 0);
workers = malloc(sizeof(struct String_vector));
allocate_vector(workers, 0);
}
int init (char * hostPort) {
srand(time(NULL));
server_id = rand();
zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
zh = zookeeper_init(hostPort,
main_watcher,
15000,
0,
0,
0);
return errno;
}
int main (int argc, char * argv[]) {
LOG_DEBUG(("THREADED defined"));
if (argc != 2) {
fprintf(stderr, "USAGE: %s host:port\n", argv[0]);
exit(1);
}
/*
* 初始化zookeeper session
*/
if(init(argv[1])){
LOG_ERROR(("Error while initializing the master: ", errno));
}
#ifdef THREADED
/*
* Wait until connected
*/
while(!is_connected()) {
sleep(1);
}
LOG_DEBUG(("Connected, going to bootstrap and run for master"));
/*
* Create parent znodes
*/
bootstrap();
/*
* Run for master
*/
run_for_master();
/*
* 运行直到会话过期
*/
while(!is_expired()) {
sleep(1);
}
#else
int run = 0;
fd_set rfds, wfds, efds;
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
while (!is_expired()) {
int fd = -1;
int interest = 0;
int events ;
struct timeval tv;
int rc;
zookeeper_interest(zh, &fd, &interest, &tv);
if (fd != -1) {
if (interest&ZOOKEEPER_READ) {
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if (interest&ZOOKEEPER_WRITE) {
FD_SET(fd, &wfds);
} else {
FD_CLR(fd, &wfds);
}
} else {
fd = 0;
}
/*
* 下一个if块包含调用bootstrap、运行master
* 只有当客户端已建立好连接并且is_connected为true
* 的情况下才会进入
*/
if(is_connected() && !run) {
LOG_DEBUG(("Connected, going to bootstrap and run for master"));
/*
* Create parent znodes
*/
bootstrap();
/*
* Run for master
*/
run_for_master();
run = 1;
}
rc = select(fd+1, &rfds, &wfds, &efds, &tv);
events = 0;
if (rc > 0) {
if (FD_ISSET(fd, &rfds)) {
events |= ZOOKEEPER_READ;
}
if (FD_ISSET(fd, &wfds)) {
events |= ZOOKEEPER_WRITE;
}
}
zookeeper_process(zh, events);
}
#endif
return 0;
}