haproxy无缝热加载的辅助进程multibinder的C语言实现版本

  本模块用epoll模型来实现了一个multibinder，供haproxy无缝热重启来使用，需要另外再做一个haproxy_wrapper来实现haproxy配置文件的生成和进程的加载功能。
   本模块也可以作为入门epoll开发和signalfd开发的学习材料。

haproxy的无缝热重启的实现原理

功能：

• 创建一个listen socket
• 关闭一个listen socket
• 获取一个listen socket
• 列出正在监听中的listen socket列表

编译：

   g++ multibinder.cpp -o multibinder

运行：

   ./multibinder
  或者 /multibinder [unix socket path]

退出：

   kill -SIGINT pgrep multibinder

源码：

packet.h

#pragma once
#include 


enum REQ_TYPE{
    REQ_TYPE_CREATE = 0,
    REQ_TYPE_GET    = 1,
    REQ_TYPE_CLOSE  = 2,
    REQ_TYPE_LIST   = 3
};

enum RESP_STATUS {
    RESP_STATUS_OK       = 0,
    RESP_STATUS_ERR      = 1
};

struct req_packet_t {
    uint16_t  size;
    uint8_t   type;
    char      ip[1];
};

struct resp_packet_t {
    uint16_t size;
    uint8_t  status;
    char     msg[1];
};

multibinder.cpp

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

#include "packet.h"

using namespace std;

#define MAX_EVENTS 10
#define SOCKET_PATH "./multibind_socket"
string socket_path;


struct event_ctx_t {
    char          *buf;
    uint16_t       data_end;
    uint16_t       size;
    req_packet_t  *req;
    resp_packet_t *reply;
    int sockfd;

    event_ctx_t() {
        buf = NULL;
        req = NULL;
        sockfd = -1;
        data_end = 0;
        size = 0;
    }

    ~event_ctx_t() {
        if (buf) {
            delete []buf;
            buf = NULL;
        }
        if (req) {
            req = NULL;
        }
    }
};

map<string, int> bind_ip_map;


void reply_error(struct event_ctx_t *ctx, const char * msg, ...) 
{
    va_list args;
    va_start(args, msg);

    ctx->reply = (resp_packet_t*)ctx->buf;
    ctx->reply->status = RESP_STATUS_ERR;
    ctx->reply->size = vsnprintf(ctx->reply->msg, ctx->size - 1, msg, args) + offsetof(resp_packet_t, msg);
    printf("REPLY: %s\n", ctx->reply->msg);
    
    ctx->data_end = ctx->reply->size;

    if (send(ctx->sockfd, ctx->buf, ctx->data_end, 0) == -1) {
        perror("failed to send");
    }
}

void reply_error_with_status(struct event_ctx_t *ctx, uint8_t status, const char * msg, ...) 
{
    va_list args;
    va_start(args, msg);

    ctx->reply = (resp_packet_t*)ctx->buf;
    ctx->reply->status = status;
    ctx->reply->size = vsnprintf(ctx->reply->msg, ctx->size - 1, msg, args) + offsetof(resp_packet_t, msg);
    printf("REPLY: %s\n", ctx->reply->msg);

    ctx->data_end = ctx->reply->size;

    if (send(ctx->sockfd, ctx->buf, ctx->data_end, 0) == -1) {
        perror("failed to send");
    }
}

void reply_ok(struct event_ctx_t *ctx, int sockfd)
{
    ctx->reply = (resp_packet_t*)ctx->buf;
    ctx->reply->status = 0;
    sprintf(ctx->reply->msg, "OK, socket: %d", sockfd);
    ctx->reply->size = strlen(ctx->reply->msg) + offsetof(resp_packet_t, msg);
    ctx->data_end = ctx->reply->size;
    printf("REPLY: %s\n\n", ctx->reply->msg);

    struct msghdr msg;
    memset(&msg, 0, sizeof(msg));

    // 控制信息辅助数据缓冲区
    char control_buf[CMSG_SPACE(sizeof(int))];
    msg.msg_control = control_buf;
    msg.msg_controllen = sizeof(control_buf);

    // 指向辅助数据的指针
    struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
    cmsg->cmsg_len = CMSG_LEN(sizeof(int));
    cmsg->cmsg_level = SOL_SOCKET;
    cmsg->cmsg_type = SCM_RIGHTS;

    // 将文件句柄复制到辅助数据
    memcpy(CMSG_DATA(cmsg), &sockfd, sizeof(int));

    msg.msg_name = NULL;
    msg.msg_namelen = 0;

    struct iovec iov;
    iov.iov_base = (void *)ctx->buf;
    iov.iov_len = ctx->data_end;
 
    msg.msg_iov = &iov;
    msg.msg_iovlen = 1;

    if (sendmsg(ctx->sockfd, &msg, 0) == -1) {
        perror("failed to send");
    }

}


int create_listen_socket(bool ipv6, const char* ip, int port)
{
    int sockfd = -1;
    if (ipv6) {
        sockfd = socket(AF_INET6, SOCK_STREAM, 0);
        if (sockfd == -1) {
            perror("create socket");
            return -1;
        }
        
        // 设置 SO_REUSEADDR 选项，以便多个进程可以同时监听同一IP端口
        int reuse = 1;
        if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)) == -1) {
            perror("setsockopt");
            exit(EXIT_FAILURE);
        }
        
        sockaddr_in6 addr6; 
        memset(&addr6, 0, sizeof(addr6));
        addr6.sin6_family = AF_INET6;
        addr6.sin6_port = htons(port);
        inet_pton(AF_INET6, ip, &addr6.sin6_addr);
        
        if (bind(sockfd, (struct sockaddr *)&addr6, sizeof(addr6)) == -1) {
            perror("bind failed");
            return -1;
        }

        if (listen(sockfd, 5) == -1) {
            perror("listen failed");
            return -1;
        }

    }
    else {
        sockfd = socket(AF_INET, SOCK_STREAM, 0);
        if (sockfd == -1) {
            perror("create socket");
            return -1;
        }

        sockaddr_in addr; 
        memset(&addr, 0, sizeof(addr));
        addr.sin_family = AF_INET;
        addr.sin_port = htons(port);
        inet_pton(AF_INET, ip, &addr.sin_addr);
        
        if (bind(sockfd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
            perror("bind failed");
            return -1;
        }

        if (listen(sockfd, 5) == -1) {
            perror("listen failed");
            return -1;
        }
    }

    return sockfd;
}

const char* get_request_type_name(uint8_t type)
{
    switch (type) {
    case REQ_TYPE_CREATE:
        return "create";
    case REQ_TYPE_GET:
        return "get";
    case REQ_TYPE_CLOSE:
        return "close";
    case REQ_TYPE_LIST:
        return "list";
    default:
        return "unknown";
    }
}

bool deal_request(struct event_ctx_t *ctx)
{
    req_packet_t* req = ctx->req;

    if (req->type != REQ_TYPE_CREATE && req->type != REQ_TYPE_GET 
        && req->type != REQ_TYPE_CLOSE && req->type != REQ_TYPE_LIST) {
        reply_error(ctx, "Invalid request type"); 
        return false;
    }

    bool ipv6 = false;
    bool ipv6_ok = false;

    char *p = NULL;

    printf("->Request is %s : %s\n", get_request_type_name(req->type), req->ip);

    if (req->type == REQ_TYPE_LIST) {
        string out;
        if (!bind_ip_map.empty()) {
            for(auto iter = bind_ip_map.begin(); iter != bind_ip_map.end(); iter++) {
                out += iter->first + "->" + std::to_string(iter->second) + "\n";
            }
        }
        else{
            out = "[EMPTY]\n";
        }

        reply_error_with_status(ctx, RESP_STATUS_OK, out.c_str());
        return true;
    }

    if (req->ip[0] == '[') {
        ipv6 = true;

        for(int i = 1; i < req->size - offsetof(req_packet_t, ip); i++) {
            if (req->ip[i] == ']') {
                ipv6_ok = true;
                p = &req->ip[i] + 1;
                break;
            }
        }

        if (!ipv6_ok) {
            reply_error(ctx, "Invalid IP address");
            return false;
        }
        
        if (*p != ':') {
            reply_error(ctx, "Invalid ip:port in the request");
            return false;
        }
    }
    else{
        p = strchr(req->ip, ':');
        if (p == NULL) {
            reply_error(ctx, "Invalid ip:port in the request");
            return false;
        }
    }
    

    string ip(req->ip, p);
    string port(++p);

    string addr = ip + ":" + port;

    auto iter = bind_ip_map.find(addr);

    int sockfd = -1;
    if (iter == bind_ip_map.end()) {
        if (req->type == REQ_TYPE_CREATE) {
            int p = strtol(port.c_str(), NULL, 10);
            if (p == 0 || p >= 65535) {
                reply_error(ctx, "Invalid ip:port in the request");
                return false;
            }

            sockfd = create_listen_socket(ipv6, ip.c_str(), p);
            if (sockfd != -1) {
                bind_ip_map.emplace(addr, sockfd);
                printf("socket %d created for %s\n", sockfd, addr.c_str());
            }
            else {
                reply_error(ctx, "Failed to create socket, %s", strerror(errno));
                return false;
            }
        }
        else {
            reply_error(ctx, "Cannot find the ip:port in the map");
            return false;
        }
    }
    else {
        sockfd = iter->second;
        if (req->type == REQ_TYPE_GET || req->type == REQ_TYPE_CREATE) {
            printf("socket %d found for %s\n", sockfd, addr.c_str());
        }
        else if(req->type == REQ_TYPE_CLOSE) {
            bind_ip_map.erase(iter);
            close(sockfd);
            printf("socket %d removed for %s\n", sockfd, addr.c_str());
            reply_error_with_status(ctx, RESP_STATUS_OK, "OK, socket %d removed", sockfd);
        }
    }

    reply_ok(ctx, sockfd);

    return true;

}


void free_client(struct event_ctx_t *ctx, int epoll_fd)
{
    if (epoll_ctl(epoll_fd, EPOLL_CTL_DEL, ctx->sockfd, NULL) == -1) {
        perror("epoll_ctl");
    }

    close(ctx->sockfd);
    ctx->sockfd = -1;
    delete ctx;
}



int main(int argc, char *argv[]) {


    int server_fd, client_fd, epoll_fd;
    struct sockaddr_un server_addr, client_addr;
    socklen_t client_len;

    if (argc > 2) {
        socket_path = argv[1];
    }
    else {
        socket_path = SOCKET_PATH;
    }

    unlink(socket_path.c_str());

    // 创建UNIX域套接字
    server_fd = socket(AF_UNIX, SOCK_STREAM, 0);
    if (server_fd == -1) {
        perror("socket");
        exit(EXIT_FAILURE);
    }

    // 设置服务器地址
    server_addr.sun_family = AF_UNIX;
    strncpy(server_addr.sun_path, socket_path.c_str(), sizeof(server_addr.sun_path) - 1);

    // 将套接字绑定到服务器地址
    if (bind(server_fd, (struct sockaddr *)&server_addr, sizeof(server_addr)) == -1) {
        perror("bind");
        exit(EXIT_FAILURE);
    }

    // 开始监听连接
    if (listen(server_fd, 5) == -1) {
        perror("listen");
        exit(EXIT_FAILURE);
    }

    // 设置signal消息处理
    signal(SIGPIPE, SIG_IGN);
    sigset_t mask;
    sigemptyset(&mask);
    sigaddset(&mask, SIGINT);
    sigaddset(&mask, SIGTERM);

    if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) {
        perror("sigprocmask");
        exit(EXIT_FAILURE);
    }

    int sfd = signalfd(-1, &mask, 0);
    if (sfd == -1) {
        perror("signalfd");
        exit(EXIT_FAILURE);
    }


    // 创建epoll实例
    epoll_fd = epoll_create1(0);
    if (epoll_fd == -1) {
        perror("epoll_create1");
        exit(EXIT_FAILURE);
    }

    // 添加服务器套接字到epoll事件集合中
    struct epoll_event event, events[MAX_EVENTS];
    event.events = EPOLLIN;
    event.data.fd = server_fd;
    if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, server_fd, &event) == -1) {
        perror("epoll_ctl");
        exit(EXIT_FAILURE);
    }


    // 将 signalfd 添加到 epoll 实例中
    event.events = EPOLLIN;
    event.data.fd = sfd;
    if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sfd, &event) == -1) {
        perror("epoll_ctl");
        exit(EXIT_FAILURE);
    }

    printf("Waiting for incoming connections on UNIX socket [%s]\n", socket_path.c_str());

    bool stop = false;

    while (!stop) {
        // 等待事件发生
        int num_events = epoll_wait(epoll_fd, events, MAX_EVENTS, -1);
        if (num_events == -1) {
            if (errno == EINTR) {
                // epoll_wait被中断，重新调用
                continue;
            }

            perror("epoll_wait");
            exit(EXIT_FAILURE);
        }

        // 处理所有就绪的事件
        for (int i = 0; i < num_events; i++) {
            // 如果是服务器套接字就绪，表示有新的客户端连接
            if (events[i].data.fd == server_fd) {
                // 接受客户端连接
                client_len = sizeof(client_addr);
                client_fd = accept(server_fd, (struct sockaddr *)&client_addr, &client_len);
                if (client_fd == -1) {
                    perror("accept");
                    exit(EXIT_FAILURE);
                }

                // 将客户端套接字添加到epoll事件集合中
                event_ctx_t *ctx = new event_ctx_t();
                ctx->data_end = 0;
                ctx->size = 1023;
                ctx->buf = new char[ctx->size];
                ctx->req = (req_packet_t*)ctx->buf;
                ctx->sockfd = client_fd;

                event.events = EPOLLIN;
                event.data.ptr = ctx;
                if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &event) == -1) {
                    perror("epoll_ctl");
                    exit(EXIT_FAILURE);
                }
            }
            else if (events[i].data.fd == sfd) {
                // 读取 signalfd 中的数据来处理信号事件
                struct signalfd_siginfo fdsi;
                ssize_t s = read(sfd, &fdsi, sizeof(struct signalfd_siginfo));
                if (s != sizeof(struct signalfd_siginfo)) {
                    perror("read");
                    exit(EXIT_FAILURE);
                }

                // 处理信号事件
                if (fdsi.ssi_signo == SIGINT) {
                    printf("Received SIGINT, exiting...\n");
                    stop = true;

                } else if (fdsi.ssi_signo == SIGTERM) {
                    printf("Received SIGTERM, exiting...\n");
                    stop = true;
                }
            }            
            // 否则，是客户端套接字就绪，表示有数据可读
            else {
                int bytes_read;
                 event_ctx_t *ctx = (event_ctx_t*)events[i].data.ptr;

                // 读取客户端发送的数据
                bytes_read = read(ctx->sockfd, ctx->buf + ctx->data_end, ctx->size - ctx->data_end);
                
                if (bytes_read == -1) {
                    perror("read");

                    free_client(ctx, epoll_fd);                    
                    exit(EXIT_FAILURE);
                } else if (bytes_read == 0) {
                    // 客户端关闭了连接
                    printf("Client disconnected.\n");

                    // 关闭客户端套接字并从epoll事件集合中移除
                    free_client(ctx, epoll_fd);

                } else {
                    uint16_t pkt_size = 0;

                    ctx->data_end += bytes_read;
                    if (ctx->data_end > 2) {
                        pkt_size = ctx->req->size;    
                    }

                    if (pkt_size > ctx->size) {
                        char* old = ctx->buf;
                        ctx->buf = new char[pkt_size];
                        ctx->size = pkt_size;
                        ctx->req = (req_packet_t*)ctx->buf;
                        memcpy(ctx->buf, old, ctx->data_end);
                        delete [] old;
                    }

                    if (pkt_size > 0 && pkt_size <= ctx->data_end) {
                        // 处理接收到的数据
                        ctx->buf[ctx->data_end] = '\0';
                        deal_request(ctx);
                        free_client(ctx, epoll_fd);
                    }
                }
            }
        }
    }

	// 退出释放相关资源
    close(server_fd);
    close(epoll_fd);

    unlink(socket_path.c_str());

    for (auto iter = bind_ip_map.begin(); iter!= bind_ip_map.end(); iter) {
        close(iter->second);
    }

    return 0;
}
```