Linux服务器多进程模型

http://blog.csdn.net/winlinvip/article/details/7764526

Linux多进程服务器真的很给力，赞一个！

Linux多进程一般是master负责侦听，worker接受和伺服client。

一个使用了以下技术的多进程模型：

1. sigset：安全信号，信号屏蔽和接受。

2. epoll：异步io模型。

master进程使用信号模型，侦听用户信号和程序信号，并和worker交流。它的主循环是sigsuspend。

worker进程使用事件模型，使用epoll_wait等待事件，同时他也接受信号（信号会中断epoll_wait）。

 // mpserver —— multiple processes server
 #include <stdio.h>
 #include <stdlib.h>
 #include <iostream>
 using namespace std;

 #include <unistd.h>
 #include <signal.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <sys/wait.h>
 #include <sys/epoll.h>
 #include <string.h>
 #include <errno.h>

 #define err_exit(msg) cout << "[error] " << msg << endl; exit(1)

 struct UserOptions{
     int port;
     int num_processes;
 };

 void discovery_user_options(int argc, char** argv, UserOptions& options){
     if(argc <= 2){
         cout << "Usage: " << argv[0] << " <port> <num_processes>" << endl
             << "port: the port to listen" << endl
             << "num_processes: the num to start processes. if 0, use single process mode" << endl;
         exit(1);
     }

     options.port = atoi(argv[1]);
     options.num_processes = atoi(argv[2]);
 }

 int listen_server_socket(UserOptions& options){
     int serverfd = socket(AF_INET, SOCK_STREAM, 0);

     if(serverfd == -1){
         err_exit("init socket error!");
     }
     cout << "init socket success! #" << serverfd << endl;

     int reuse_socket = 1;
     if(setsockopt(serverfd, SOL_SOCKET, SO_REUSEADDR, &reuse_socket, sizeof(int)) == -1){
         err_exit("setsockopt reuse-addr error!");
     }
     cout << "setsockopt reuse-addr success!" << endl;

     sockaddr_in addr;
     addr.sin_family = AF_INET;
     addr.sin_port = htons(options.port);
     addr.sin_addr.s_addr = INADDR_ANY;
     if(bind(serverfd, (const sockaddr*)&addr, sizeof(sockaddr_in)) == -1){
         err_exit("bind socket error!");
     }
     cout << "bind socket success!" << endl;

     if(listen(serverfd, 10) == -1){
         err_exit("listen socket error!");
     }
     cout << "listen socket success! " << options.port << endl;

     return serverfd;
 }

 void signal_handler(int signo){
     cout << "#" << getpid() << " get a signal:";

     bool do_exit = false;
     switch(signo){
         case SIGCHLD:
             cout << "SIGCHLD";
             int status;
             while(waitpid(0, &status, WNOHANG) > 0){
             }
             break;
         case SIGALRM:
             cout << "SIGALRM";
             break;
         case SIGIO:
             cout << "SIGIO";
             break;
         case SIGINT:
             cout << "SIGINT";
             do_exit = true;
             break;
         case SIGHUP:
             cout << "SIGHUP";
             do_exit = true;
             break;
         case SIGTERM:
             cout << "SIGTERM";
             do_exit = true;
             break;
         case SIGQUIT:
             cout << "SIGQUIT";
             do_exit = true;
             break;
         case SIGUSR1:
             cout << "SIGUSR1";
             break;
         case SIGUSR2:
             cout << "SIGUSR2";
             break;
     }

     cout << "!" << endl;
     if(do_exit){
         exit(0);
     }
 }

 void register_signal_handler_imp(int signum, void (*handler)(int)){
     struct sigaction action;

     action.sa_handler = handler;
     sigemptyset(&action.sa_mask);
     //action.sa_flags = 0;

     if(sigaction(signum, &action, NULL) == -1){
         err_exit("register signal handler failed!");
     }
 }
 void register_signal_handler(){
     register_signal_handler_imp(SIGCHLD, signal_handler);
     register_signal_handler_imp(SIGALRM, signal_handler);
     register_signal_handler_imp(SIGIO, signal_handler);
     register_signal_handler_imp(SIGINT, signal_handler);
     register_signal_handler_imp(SIGHUP, signal_handler);
     register_signal_handler_imp(SIGTERM, signal_handler);
     register_signal_handler_imp(SIGQUIT, signal_handler);
     register_signal_handler_imp(SIGUSR1, signal_handler);
     register_signal_handler_imp(SIGUSR2, signal_handler);
     cout << "register signal handler success!" << endl;
 }

 void block_specified_signals(){
     sigset_t set;

     sigemptyset(&set);

     sigaddset(&set, SIGCHLD);
     sigaddset(&set, SIGALRM);
     sigaddset(&set, SIGIO);
     sigaddset(&set, SIGINT);
     sigaddset(&set, SIGHUP);
     sigaddset(&set, SIGTERM);
     sigaddset(&set, SIGQUIT);
     sigaddset(&set, SIGUSR1);
     sigaddset(&set, SIGUSR2);

     if(sigprocmask(SIG_BLOCK, &set, NULL) == -1){
         err_exit("sigprocmask block signal failed!");
     }
     cout << "sigprocmask block signal success!" << endl;
 }

 void unblock_all_signals(){
     sigset_t set;
     sigemptyset(&set);

     // the master process block all signals, we unblock all for we use epoll to wait events.
     if(sigprocmask(SIG_SETMASK, &set, NULL) == -1){
         err_exit("sigprocmask block signal failed!");
     }
     cout << "sigprocmask block signal success!" << endl;
 }

 void epoll_add_event(int ep, int fd){
     epoll_event ee;
     ee.events = EPOLLIN;
     ee.data.fd = fd;
     if(epoll_ctl(ep, EPOLL_CTL_ADD, fd, &ee) == -1){
         err_exit("epoll add event failed!");
     }
     cout << "epoll add fd success: #" << fd << endl;
 }
 void epoll_remove_event(int ep, int fd){
     if(epoll_ctl(ep, EPOLL_CTL_DEL, fd, NULL) == -1){
         err_exit("epoll remove event failed! fd=" << fd << ", errno=0x" << hex << errno << dec << " " << strerror(errno));
     }
     cout << "epoll remove fd success: #" << fd << endl;
 }

 void close_client(int ep, int fd){
     cout << "the client dead, remove it: #" << fd << endl;
     epoll_remove_event(ep, fd);
     close(fd);
 }

 void worker_get_event(int ep, int serverfd, epoll_event& active){
     int fd = active.data.fd;

     // server listening socket event.
     if(fd == serverfd){
         int client = accept(serverfd, NULL, 0);
         if(client == -1){
             err_exit("accept client socket error!");
         }
         cout << "get a client: #" << client << endl;
         epoll_add_event(ep, client);
         return;
     }

     // client event.
     if((active.events & EPOLLHUP) == EPOLLHUP || (active.events & EPOLLERR) == EPOLLERR){
         cout << "get a EPOLLHUP or EPOLLERR event from client #" << fd << endl;
         close_client(ep, fd);
         return;
     }
     if((active.events & EPOLLIN) == EPOLLIN){
         if(true){
             cout << "get a EPOLLIN event from client #" << fd << endl;
             char buf[1024];
             memset(buf, 0, sizeof(buf));
             if(recv(fd, buf, sizeof(buf), 0) <= 0){
                 close_client(ep, fd);
                 return;
             }
             cout << "recv from client: " << buf << endl;
         }
         if(true){
             char msg[] = "hello, server ping!";
             if(send(fd, msg, sizeof(msg), 0) <= 0){
                 close_client(ep, fd);
                 return;
             }
         }
         return;
     }
 }

 void worker_process_cycle(int serverfd){
     cout << "start worker process cycle" << endl;
     unblock_all_signals();

     int ep = epoll_create(1024);
     if(ep == -1){
         err_exit("epoll_create failed!");
     }
     cout << "epoll create success!" << endl;
     epoll_add_event(ep, serverfd);

     for(;;){
         epoll_event events[1024];
         int incoming = epoll_wait(ep, events, 1024, -1);
         if(incoming == -1){
             break;
         }
         for(int i = 0; i < incoming; i++){
             worker_get_event(ep, serverfd, events[i]);
         }
     }

     cout << "worker process exited" << endl;
     close(ep);
 }

 void start_worker_process(int serverfd){
     pid_t pid = fork();

     if(pid == -1){
         err_exit("fork process failed");
     }

     if(pid == 0){
         worker_process_cycle(serverfd);
         exit(0);
     }

     cout << "fork process success: #" << pid << endl;
 }

 int main(int argc, char** argv){
     register_signal_handler();
     block_specified_signals();

     //sleep(3);
     UserOptions options;
     discovery_user_options(argc, argv, options);
     int serverfd = listen_server_socket(options);

     for(int i = 0; i < options.num_processes; i++){
         start_worker_process(serverfd);
     }
     if(options.num_processes == 0){
         worker_process_cycle(serverfd);
     }

     sigset_t set;
     sigemptyset(&set);
     for(;;){
         sigsuspend(&set);
     }

     return 0;
 }

 // mpclient —— multiple processes client
 #include <stdio.h>
 #include <stdlib.h>
 #include <iostream>
 using namespace std;

 #include <unistd.h>
 #include <signal.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <string.h>

 #define err_exit(msg) cout << "[error] " << msg << endl; exit(1)

 struct UserOptions{
     char* server_ip;
     int port;
     int num_processes;
 };

 void discovery_user_options(int argc, char** argv, UserOptions& options){
     if(argc <= 2){
         cout << "Usage: " << argv[0] << " <server_ip> <port> <num_processes>" << endl
             << "server_ip: the ip address of server" << endl
             << "port: the port to connect at" << endl
             << "num_processes: the num to start processes. if 0, use single process mode" << endl;
         exit(1);
     }

     options.server_ip = argv[1];
     options.port = atoi(argv[2]);
     options.num_processes = atoi(argv[3]);
 }

 int connect_server_socket(UserOptions& options){
     int clientfd = socket(AF_INET, SOCK_STREAM, 0);

     if(clientfd == -1){
         err_exit("init socket error!");
     }
     cout << "init socket success!" << endl;

     sockaddr_in addr;
     addr.sin_family = AF_INET;
     addr.sin_port = htons(options.port);
     addr.sin_addr.s_addr = inet_addr(options.server_ip);
     if(connect(clientfd, (const sockaddr*)&addr, sizeof(sockaddr_in)) == -1){
         err_exit("connect socket error!");
     }
     cout << "connect socket success!" << endl;

     return clientfd;
 }

 int main(int argc, char** argv){
     UserOptions options;
     discovery_user_options(argc, argv, options);
     int* fds = new int[options.num_processes];
     for(int i = 0; i < options.num_processes; i++){
         fds[i] = connect_server_socket(options);
     }

     bool do_loop = true;
     while(do_loop){
         for(int i = 0; i < options.num_processes; i++){
             int fd = fds[i];
             if(true){
                 char msg[] = "hello, client ping!";
                 if(send(fd, msg, sizeof(msg), 0) <= 0){
                     close(fd);
                     do_loop = false;
                     break;
                 }
             }
             if(true){
                 char buf[1024];
                 memset(buf, 0, sizeof(buf));
                 if(recv(fd, buf, sizeof(buf), 0) <= 0){
                     close(fd);
                     do_loop = false;
                     break;
                 }
                 cout << "recv from server: " << buf << endl;
             }

             sleep(3);
         }
     }

     delete[] fds;

     return 0;
 }

master启动的主要过程：

1. 注册信号处理函数：sigaction。

2. 屏蔽所有关心的信号，让内核暂时保留信号，不要发给程序，因为这时候程序不系统被打断。

3. 侦听端口。

4. 启动worker进程。

5. 等待信号的到来：sigsuspend。

worker启动的主要过程：

1. 取消信号的屏蔽。因为这时候worker拷贝的是master的内存结构，所以信号屏蔽也拷贝过来了。若不取消，则无法收到退出消息，只能重启服务器或它自己退出才能杀这个进程了。

2. 创建epoll。

3. 将侦听的server_socket_fd添加到epoll。

4. 等待epoll事件的到来：epoll_wait。或者信号也能中断这个epoll_wait：返回-1.

典型的多进程服务器模型如下图所示：

考虑一个RTMP服务器，假设和FMS类似以vhost为边界启动worker，但是client应该连接到指定的worker。所以这个典型的模型就需要修改了。

有一种方式是master接受连接，handshake，然后接收到clieint的connect请求后，知道连接的vhost是哪个，再把这个client fd转发给worker。如下图所示：

但这种方式master过于复杂，它实际上需要做worker要做的事情。多进程稳定的一个原因是进程的自然边界，worker进程崩溃不会影响其他worker和master，如果中心master崩溃，那么系统就完蛋了。所以master要尽量简单不能有过多的逻辑。另外，单进程和多进程切换的一个条件，就是worker能用一个函数表达，单进程时直接调用这个函数，而多进程时fork后调用这个函数；若master要混合worker的功能，单进程时也需要过多交互。

改进如下图：

master变得简单，worker功能也单一了。worker发现不是自己的client时，将client发给给master，由master分发给特定的worker。单进程时没有分发这一步了；多进程时master也只需要分发fd，而不需要处理协议的逻辑。

在master和worker之间传递fd不是一个好办法，因为过多频繁的SIGIO可能导致“I/O possible”；google上找了一下说是由于信号SIGIO没有处理引起，但明显已经侦听了handler；有人说是signal queue overflow，恩，这个很像！

socketpaire有时还会引起“[write_channel] sendmsg error, ret=-1, errno=0xb Resource temporarily unavailable”错误。管道错误。一般发生在worker进程中，如果多个woker进程都向master发送数据，而master处理一个数据后sleep或者做其他的事情去了，就会造成这种情况（master的socketpaire缓冲区满了）。

master若需要和worker使用socketpair通信，master使用信号来实现异步，当worker的socketpair有SIGIO信号时，说明worker在发数据，如何判断是哪个worker发的呢？可以用select，当然不太好；可以使用F_SETSIG来更改默认的SIGIO，改为SIGRTMIN+fd，当worker有数据时，收到的信号就包含了fd了。

获取SIGIO的fd的另外一个方法，是使用siginfo_t取得fd。使用F_SETSIG可以设置的SIG较少，SIGRTMAX为64，所以最多可用32个自定义实时信号，一般用来对SIGIO分类。必须将SIGIO定义为实时信号，然后在sigaction时设置SA_SIGINFO和使用void (*handler)(int signo, siginfo_t* info, void* context)，info->si_fd就是SIGIO的fd。

以下是一个糅合了多进程、信号、socketpair、passing fd的一个原型，较稳定的一个原型：

 // mpserver2 —— multiple processes server
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <iostream>
 #include <vector>
 using namespace std;

 #include <unistd.h>
 #include <signal.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <sys/wait.h>
 #include <sys/epoll.h>
 #include <string.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/ioctl.h>

 #define info(msg) cout << "#" << getpid() << " " << msg
 #define err_exit(msg) info("[error] " << msg << ", errno=0x" << hex << errno << dec << " " << strerror(errno) << endl); exit(-1)

 // define the SIGIO to RTSig, the SIGIO now means rt-sig queue full.
 #define SIG_SOCKET_IO SIGRTMIN+1

 bool global_terminate = false;
 bool global_channel_msg = false;
 // when global_channel_msg is true, the current_active_io_fd set to the fd.
 int current_active_io_fd = -1;

 int current_client_num = 0;
 bool accepting = true;

 // the num of client for a proecess to serve.
 // 2+N
 // 2: the serverfd, the channelfd
 // N: clients.
 #define max_clients_per_process 2+1000
 // the physical max clients, if exceed this value, error!
 #define physical_max_clients 1024

 #define max_workers 100

 struct UserOptions{
     int port;
     int num_processes;
 };

 void discovery_user_options(int argc, char** argv, UserOptions& options){
     if(argc <= 2){
         cout << "Usage: " << argv[0] << " <port> <num_processes>" << endl
             << "port: the port to listen" << endl
             << "num_processes: the num to start processes. if 0, use single process mode" << endl
             << "for example: " << argv[0] << " 1990 20" << endl;
         exit(1);
     }

     options.port = atoi(argv[1]);
     options.num_processes = atoi(argv[2]);
 }

 #define CMD_FD 100
 #define CMD_INFO 200
 struct channel_msg{
     int command;
     int fd; // the fd, set to -1 if no fd.
     int total_client; // the total clients number of specified process.
     int param; // the param.
 };
 void write_channel(int sock, channel_msg* data, int size)
 {
     msghdr msg;

     // init msg_control
     if(data->fd == -1){
         msg.msg_control = NULL;
         msg.msg_controllen = 0;
     }
     else{
         union {
             struct cmsghdr cm;
             char space[CMSG_SPACE(sizeof(int))];
         } cmsg;
         memset(&cmsg, 0, sizeof(cmsg));

         cmsg.cm.cmsg_level = SOL_SOCKET;
         cmsg.cm.cmsg_type = SCM_RIGHTS; // we are sending fd.
         cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int));

         msg.msg_control = (cmsghdr*)&cmsg;
         msg.msg_controllen = sizeof(cmsg);
         *(int *)CMSG_DATA(&cmsg.cm) = data->fd;
     }

     // init msg_iov
     iovec iov[1];
     iov[0].iov_base = data;
     iov[0].iov_len  = size;

     msg.msg_iov = iov;
     msg.msg_iovlen = 1;

     // init msg_name
     msg.msg_name = NULL;
     msg.msg_namelen = 0;

     info("[write_channel][info] data fd=" << data->fd << ", command="
         << data->command << ", total_client=" << data->total_client
         << ", param=" << data->param << endl);

     while(true){
         ssize_t ret;
         if ((ret = sendmsg(sock, &msg, 0)) <= 0){
             // errno = 0xb Resource temporarily unavailable
             // donot retry, for it's no use, error infinite loop.
             err_exit("[write_channel] sendmsg error, ret=" << ret);
         }
         break;
     }
 }
 void read_channel(int sock, channel_msg* data, int size)
 {
     msghdr msg;

     // msg_iov
     iovec iov[1];
     iov[0].iov_base = data;
     iov[0].iov_len = size;

     msg.msg_iov = iov;
     msg.msg_iovlen  = 1;

     // msg_name
     msg.msg_name = NULL;
     msg.msg_namelen = 0;

     // msg_control
     union { // union to create a 8B aligned memory.
         struct cmsghdr cm; // 16B = 8+4+4
         char space[CMSG_SPACE(sizeof(int))]; // 24B = 16+4+4
     } cmsg;
     memset(&cmsg, 0, sizeof(cmsg));

     msg.msg_control = (cmsghdr*)&cmsg;
     msg.msg_controllen = sizeof(cmsg);

     ssize_t ret;
     if ((ret = recvmsg(sock, &msg, 0)) == -1) {
         err_exit("[read_channel] recvmsg error");
     }
     if(ret == 0){
         info("[read_channel] connection closed" << endl);
         exit(0);
     }

     info("[read_channel][info] data fd=" << data->fd << ", command="
         << data->command << ", total_client=" << data->total_client
         << ", param=" << data->param << endl);
     if(data->command == CMD_FD){
         int fd = *(int *)CMSG_DATA(&cmsg.cm);
         if(fd == 0 || fd == 1){
             err_exit("[read_channel] recvmsg invalid fd=" << fd
                 << ", data->fd=" << data->fd << ", command=" << data->command
                 << ", ret=" << ret);
         }
         data->fd = fd;
     }
     else{
         data->fd = -1;
     }
 }

 void set_noblocking(int fd){
     int flag = 1;
     if(ioctl(fd, FIONBIO, &flag) == -1){
         err_exit("[master] ioctl failed!");
     }
     info("[master] server socket no-block flag set success" << endl);
 }

 int listen_server_socket(UserOptions& options){
     int serverfd = socket(AF_INET, SOCK_STREAM, 0);

     if(serverfd == -1){
         err_exit("[master] init socket error!");
     }
     info("[master] init socket success! #" << serverfd << endl);

     int reuse_socket = 1;
     if(setsockopt(serverfd, SOL_SOCKET, SO_REUSEADDR, &reuse_socket, sizeof(int)) == -1){
         err_exit("[master] setsockopt reuse-addr error!");
     }
     info("[master] setsockopt reuse-addr success!" << endl);

     set_noblocking(serverfd);

     sockaddr_in addr;
     addr.sin_family = AF_INET;
     addr.sin_port = htons(options.port);
     addr.sin_addr.s_addr = INADDR_ANY;
     if(bind(serverfd, (const sockaddr*)&addr, sizeof(sockaddr_in)) == -1){
         err_exit("[master] bind socket error!");
     }
     info("[master] bind socket success!" << endl);

     if(listen(serverfd, 10) == -1){
         err_exit("[master] listen socket error!");
     }
     info("[master] listen socket success! " << options.port << endl);

     return serverfd;
 }

 // both master and worker will invoke this handler.
 void signal_handler(int signo, siginfo_t* info, void* context){
     info("#" << getpid() << " get a signal:");

     if(signo == SIG_SOCKET_IO){
         cout << "SIG_SOCKET_IO!" << endl;
         global_channel_msg = true;
         current_active_io_fd = info->si_fd;
         info("[master] get SIGIO: fd=" << info->si_fd << endl);
         return;
     }

     switch(signo){
         case SIGCHLD:
             cout << "SIGCHLD";
             break;
         case SIGALRM:
             cout << "SIGALRM";
             break;
         case SIGIO:
             cout << "SIGIO";
             info("[master][warning] ignore the SIGIO: rt-sig queue is full!" << endl);
             break;
         case SIGINT:
             cout << "SIGINT";
             global_terminate = true;
             break;
         case SIGHUP:
             cout << "SIGHUP";
             global_terminate = true;
             break;
         case SIGTERM:
             cout << "SIGTERM";
             global_terminate = true;
             break;
         case SIGQUIT:
             cout << "SIGQUIT";
             global_terminate = true;
             break;
         case SIGUSR1:
             cout << "SIGUSR1";
             break;
         case SIGUSR2:
             cout << "SIGUSR2";
             break;
     }

     cout << "!" << endl;
 }

 void register_signal_handler_imp(int signum, void (*handler)(int, siginfo_t*, void*)){
     struct sigaction action;

     action.sa_flags = SA_SIGINFO;
     action.sa_sigaction = handler;
     sigemptyset(&action.sa_mask);
     //action.sa_flags = 0;

     if(sigaction(signum, &action, NULL) == -1){
         err_exit("[master] register signal handler failed!");
     }
 }
 void register_signal_handler(){
     register_signal_handler_imp(SIGCHLD, signal_handler);
     register_signal_handler_imp(SIGALRM, signal_handler);
     register_signal_handler_imp(SIGIO, signal_handler);
     register_signal_handler_imp(SIG_SOCKET_IO, signal_handler);
     register_signal_handler_imp(SIGINT, signal_handler);
     register_signal_handler_imp(SIGHUP, signal_handler);
     register_signal_handler_imp(SIGTERM, signal_handler);
     register_signal_handler_imp(SIGQUIT, signal_handler);
     register_signal_handler_imp(SIGUSR1, signal_handler);
     register_signal_handler_imp(SIGUSR2, signal_handler);
     info("[master] register signal handler success!" << endl);
 }

 void block_specified_signals(){
     sigset_t set;

     sigemptyset(&set);

     sigaddset(&set, SIGCHLD);
     sigaddset(&set, SIGALRM);
     sigaddset(&set, SIGIO);
     sigaddset(&set, SIG_SOCKET_IO);
     sigaddset(&set, SIGINT);
     sigaddset(&set, SIGHUP);
     sigaddset(&set, SIGTERM);
     sigaddset(&set, SIGQUIT);
     sigaddset(&set, SIGUSR1);
     sigaddset(&set, SIGUSR2);

     if(sigprocmask(SIG_BLOCK, &set, NULL) == -1){
         err_exit("[master] sigprocmask block signal failed!");
     }
     info("[master] sigprocmask block signal success!" << endl);
 }

 void unblock_all_signals(){
     sigset_t set;
     sigemptyset(&set);

     // the master process block all signals, we unblock all for we use epoll to wait events.
     if(sigprocmask(SIG_SETMASK, &set, NULL) == -1){
         err_exit("[worker] sigprocmask unblock signal failed!");
     }
     info("[worker] sigprocmask unblock signal success!" << endl);
 }

 void epoll_add_event(int ep, int fd){
     epoll_event ee;
     ee.events = EPOLLIN;
     ee.data.fd = fd;
     if(epoll_ctl(ep, EPOLL_CTL_ADD, fd, &ee) == -1){
         err_exit("[worker] epoll add event failed!");
     }
     current_client_num ++;
     info("[worker] epoll add fd success: #" << fd << ", current_client_num=" << current_client_num << endl);
 }
 void epoll_remove_event(int ep, int fd){
     if(epoll_ctl(ep, EPOLL_CTL_DEL, fd, NULL) == -1){
         err_exit("[worker] epoll remove event failed! fd=" << fd << ", errno=0x" << hex << errno << dec << " " << strerror(errno));
     }
     current_client_num --;
     info("[worker] epoll remove fd success: #" << fd << ", current_client_num=" << current_client_num << endl);
 }

 void close_client(int ep, int fd, int serverfd, int worker_channel, bool do_report = true){
     info("[worker] the client dead, remove it: #" << fd << endl);
     epoll_remove_event(ep, fd);
     close(fd);

     if(serverfd == fd){
         info("[worker] warning to close the server fd" << endl);
     }

     // start accept again.
     if(!accepting && current_client_num < max_clients_per_process){
         epoll_add_event(ep, serverfd);
         accepting = true;
     }

     // notice the master: worker can take more.
     if(do_report){
         channel_msg msg;
         msg.command = CMD_INFO;
         msg.total_client = current_client_num;
         msg.fd = -1;
         msg.param = 0;
         write_channel(worker_channel, &msg, sizeof(channel_msg));
     }
 }

 void worker_get_event(int ep, int serverfd, epoll_event& active, int worker_channel){
     info("[worker] process #" << getpid() << " current_client_num=" << current_client_num << ", worker_channel=" << worker_channel << endl);
     int fd = active.data.fd;

     // server listening socket event.
     if(fd == serverfd){
         if(current_client_num < physical_max_clients){
             int client = accept(serverfd, NULL, 0);
             if(client == -1){
                 // thundering herd
                 // http://www.citi.umich.edu/projects/linux-scalability/reports/accept.html
                 if(errno == EAGAIN || errno == EWOULDBLOCK){
                     info("[worker][warning] get a thundering herd at #" << getpid() << endl);
                     return;
                 }
                 err_exit("[worker] accept client socket error!");
             }

             int reuse_socket = 1;
             if(setsockopt(client, SOL_SOCKET, SO_REUSEADDR, &reuse_socket, sizeof(int)) == -1){
                 err_exit("[worker] setsockopt reuse-addr error!");
             }
             info("[worker] setsockopt reuse-addr success!" << endl);

             info("[worker] process #" << getpid() << " get a client: #" << client << endl);
             epoll_add_event(ep, client);
         }
         else{
             epoll_remove_event(ep, fd); // donot accept.
             accepting = false;
             info("[worker] warning, the worker #" << getpid()
                 << " exceed the max clients, current_client_num=" << current_client_num << endl);
         }
         return;
     }

     // channel event.
     if(fd == worker_channel){
         channel_msg msg;
         read_channel(worker_channel, &msg, sizeof(channel_msg));
         info("[worker] get a channel message. fd=" << msg.fd << ", command=" << msg.command
             << ", current_client_num=" << current_client_num << ", max_clients_per_process=" << max_clients_per_process
             << ", physical_max_clients=" << physical_max_clients << endl);
         if(msg.command == CMD_FD && msg.fd != -1){
             if(msg.fd == 0 || msg.fd == 1){
                 info("[worker][warning] ignore the invalid passing fd=" << msg.fd << endl);
                 return;
             }
             // accept all client from master, util the read overloaded.
             if(current_client_num < physical_max_clients){
                 epoll_add_event(ep, msg.fd);
                 info("[worker] get a dispatched client. #" << msg.fd << endl);
             }
             else{
                 info("[worker][warning] worker overloaded, close the fd: " << msg.fd << endl);
                 close(msg.fd);
             }
         }
         else{
             err_exit("[worker] get a invalid channel");
         }
         return;
     }

     // client event.
     if((active.events & EPOLLHUP) == EPOLLHUP || (active.events & EPOLLERR) == EPOLLERR){
         info("[worker] get a EPOLLHUP or EPOLLERR event from client #" << fd << endl);
         close_client(ep, fd, serverfd, worker_channel);
         return;
     }
     if((active.events & EPOLLIN) == EPOLLIN){
         if(true){
             info("[worker] get a EPOLLIN event from client #" << fd << endl);
             char ch_control; // we MUST not read more bytes, for the fd maybe passing to other process!
             if(recv(fd, &ch_control, sizeof(char), 0) <= 0){
                 close_client(ep, fd, serverfd, worker_channel);
                 return;
             }
             // check the first byte, if 'M' it's message, if 'C' it's control message.
             if(ch_control == 'C'){
                 int client_required_id; //worker_channel
                 if(recv(fd, &client_required_id, sizeof(int), 0) <= 0){
                     close_client(ep, fd, serverfd, worker_channel);
                     return;
                 }

                 pid_t pid = getpid();
                 info("[worker] get client control message: client_required_id=" << client_required_id << endl);
                 if((client_required_id % pid) == 0){
                     info("[worker][verified] client exactly required this server" << endl);
                 }
                 else{
                     // never send the follows:
                     if(fd == 0 || fd == 1 || fd == serverfd || fd == worker_channel){
                         info("[worker][warning] ignore invalid fd to passing: " << fd << endl);
                         close_client(ep, fd, serverfd, worker_channel);
                         return;
                     }
                     info("[worker] send to master to dispatch it" << endl);
                     channel_msg msg;
                     msg.fd = fd;
                     msg.command = CMD_FD;
                     msg.total_client = current_client_num;
                     msg.param = client_required_id;
                     write_channel(worker_channel, &msg, sizeof(channel_msg));
                     // donot report to master, for the CMD_FD message has reported.
                     close_client(ep, fd, serverfd, worker_channel, false);
                     return;
                 }
             }
             else{
                 char buf[1024];
                  if(recv(fd, buf, sizeof(buf), 0) <= 0){
                     close_client(ep, fd, serverfd, worker_channel);
                     return;
                 }
                info("[worker] get client data message: " << buf << endl);
             }
         }
         if(true){
             char msg[] = "hello, server ping!";
             if(send(fd, msg, sizeof(msg), 0) <= 0){
                 close_client(ep, fd, serverfd, worker_channel);
                 return;
             }
         }
         return;
     }
 }

 void worker_process_cycle(int serverfd, int worker_channel){
     cout << "[worker] start worker process cycle. serverfd=" << serverfd
         << ", worker_channel=" << worker_channel
         << ((worker_channel == -1)? "(single process)" : "(multiple processes)")<< endl;
     unblock_all_signals();

     int ep = epoll_create(1024);
     if(ep == -1){
         err_exit("[worker] epoll_create failed!");
     }
     info("[worker] epoll create success!" << endl);
     epoll_add_event(ep, serverfd);
     epoll_add_event(ep, worker_channel);

     // worker use epoll event, the signal handler will break the epoll_wait to return -1.
     for(;;){
         epoll_event events[1024];
         int incoming = epoll_wait(ep, events, 1024, -1);

         // get a event or error.
         if(incoming == -1){
             if(global_terminate){
                 close(worker_channel);
                 close(serverfd);
                 info("[worker] worker process exit" << endl);
                 exit(0);
             }
             break;
         }

         for(int i = 0; i < incoming; i++){
             worker_get_event(ep, serverfd, events[i], worker_channel);
         }
     }

     info("[worker] worker process exited" << endl);
     close(ep);
 }

 struct WorkerProcess{
     pid_t pid;
     int channel;
     int num_clients;
 };

 void start_worker_process(int serverfd, WorkerProcess& worker_process){
     // master/worker use domain socket to communicate.
     int fds[2];
     if(socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1){
         err_exit("[master] sockpair create domain socket failed");
     }
     info("[master] sockpair create domain socket success! [" << fds[0] << ", " << fds[1] << "]" << endl);

     // master process, use the fds[0].
     worker_process.num_clients = 0;
     worker_process.channel = fds[0];

     /**
     * SIGIO (http://www.lindevdoc.org/wiki/SIGIO)
     * The SIGIO signal is sent to a process to notify it about some event on a file descriptor, namely:
     *   the file descriptor is ready to receive data (previous write has finished).
     *   the file descriptor has new data to read
     *   an error has occurred
     * This signal is not sent by default; it must be enabled by specifying the O_ASYNC flag to the descriptor
     *
     * O_ASYNC (http://www.lindevdoc.org/wiki/O_ASYNC)
     * The O_ASYNC flag specifies that the process that owns a data stream (by default the process that opened it,
     * but can be changed with fcntl() with the F_SETOWN command) will receive a signal (SIGIO by default, but can
     * be changed with fcntl() with the F_SETSIG command) when the file is ready for reading or writing.
     */
     // set the socket owner to master, to get a SIGIO signal.
     if(fcntl(worker_process.channel, F_SETOWN, getpid()) == -1){
         err_exit("[master] fcntl F_SETOWN to master failed");
     }
     info("[master] fcntl F_SETOWN to master success" << endl);
     // set socket to async to singal SIGIO when data coming.
     int on = 1;
     if(ioctl(worker_process.channel, FIOASYNC, &on) == -1){
         err_exit("[master] ioctl set FIOASYNC failed");
     }
     info("[master] ioctl set FIOASYNC success" << endl);
     /** F_SETSIG
     * if donot use set_sig, we must use select to find which fd is read.
     * http://ajaxxx.livejournal.com/62378.html
     * Linux has the additional feature of fcntl(F_SETSIG), which lets you get the file descriptor that generated the
     * signal back as part of the signal information. So I implemented this, with the obvious semantics: receiving a
     * SIGIO would call the handler for just that file descriptor and then return.
     * remark: the SIGRTMIN=34, SIGRTMAX=62, so only 28 fd cannbe discoveried. so it's used for category signal,
     *       and we use SA_SIGINFO for sigaction to get the fd of SIGIO.
     */
     if(fcntl(worker_process.channel, F_SETSIG, SIG_SOCKET_IO) == -1){
         err_exit("[master] fcntl set rtsig failed: F_SETSIG.");
     }
     info("[master] fcntl set rtsig success" << endl);
     // set to unblocking.
     set_noblocking(worker_process.channel);
     set_noblocking(fds[1]);

     // start process by fork.
     pid_t pid = fork();

     if(pid == -1){
         err_exit("[master] fork process failed");
     }

     // worker process, use the fds[1].
     if(pid == 0){
         close(fds[0]);
         worker_process_cycle(serverfd, fds[1]);
         exit(0);
     }
     // master process.
     worker_process.pid = pid;
     close(fds[1]);

     info("[master] fork process success: #" << pid << ", channel=" << worker_process.channel << endl);
 }

 int find_the_required_worker(int serverfd, vector<WorkerProcess>& workers, int client_required_id, int pre_pid){
     // find the exactly worker.
     WorkerProcess* target = NULL;
     int channel = -1;
     for(vector<WorkerProcess>::iterator ite = workers.begin(); ite != workers.end(); ++ite){
         WorkerProcess& worker = *ite;
         if((client_required_id % worker.pid) == 0){
             target = &worker;
             break;
         }
     }
     // find the most idling worker.
     int pre_num = 0;
     for(vector<WorkerProcess>::iterator ite = workers.begin(); ite != workers.end(); ++ite){
         WorkerProcess& worker = *ite;
         if(worker.num_clients < max_clients_per_process && worker.pid != pre_pid){
             if(pre_num == 0 || worker.num_clients < pre_num){
                 target = &worker;
                 pre_num = worker.num_clients;
             }
         }
     }

     if(target == NULL){
         if(workers.size() < max_workers){
             // start a new worker
             WorkerProcess worker;
             start_worker_process(serverfd, worker);
             workers.push_back(worker);

             return find_the_required_worker(serverfd, workers, client_required_id, pre_pid);
         }
         else{
             err_exit("[master] all " << workers.size() << " workers overloaded!");
         }
     }
     info("[master] find a target to passing fd, pid=" << target->pid << ", num_clients=" << target->num_clients << endl);
     target->num_clients ++;

     return target->channel;
 }
 void on_channel_message(int serverfd, WorkerProcess& host, vector<WorkerProcess>& workers){
     int sock = host.channel;

     channel_msg msg;
     read_channel(sock, &msg, sizeof(channel_msg));
     host.num_clients = msg.total_client;

     cout << "[master] get a channel msg, pid=#" << host.pid << ", fd=" << msg.fd
         << ", total_client=" << msg.total_client
         << ", param=" << msg.param << ", command=" << msg.command << endl;
     if(msg.command == CMD_FD){
         // dispatch and re-dispatch fd
         int channel = find_the_required_worker(serverfd, workers, msg.param, host.pid);
         msg.command = CMD_FD;
         if(msg.fd == 0 || msg.fd == 1){
             info("[master][warning] ignore invalid fd to passing: " << msg.fd << endl);
             return;
         }
         write_channel(channel, &msg, sizeof(channel_msg));
         // direct close the fd, the target worker MUST take it or close it.
         close(msg.fd);
         host.num_clients--;
     }
     else if(msg.command == CMD_INFO){
         return;
     }
     else{
         err_exit("[master] invalid response!");
     }
 }

 int main(int argc, char** argv){
     info("[master] SIGRTMIN=0x" << hex << SIGRTMIN
         << ", SIGRTMAX=0x" << hex << SIGRTMAX << dec
         << ", SIGIO=0x" << hex << SIGIO << dec
         << ", SIGPOLL=0x" << hex << SIGPOLL << dec
         << endl);
     register_signal_handler();
     block_specified_signals();

     //sleep(3);
     UserOptions options;
     discovery_user_options(argc, argv, options);
     int serverfd = listen_server_socket(options);

     // single process mode
     if(options.num_processes == 0){
         worker_process_cycle(serverfd, -1);
     }
     // multiple processes mode
     else{
         vector<WorkerProcess> workers;
         for(int i = 0; i < options.num_processes; i++){
             WorkerProcess worker;
             start_worker_process(serverfd, worker);
             workers.push_back(worker);
         }

         // the empty set for sigsuspend
         sigset_t set;
         sigemptyset(&set);
         // master process use signal only.
         for(;;){
             sigsuspend(&set);

             if(global_terminate){
                 // kill all workers.
                 for(vector<WorkerProcess>::iterator ite = workers.begin(); ite != workers.end(); ++ite){
                     WorkerProcess& worker = *ite;
                     kill(worker.pid, SIGTERM);
                 }
                 // wait for workers to exit.
                 for(vector<WorkerProcess>::iterator ite = workers.begin(); ite != workers.end(); ++ite){
                     WorkerProcess& worker = *ite;
                     int status;
                     waitpid(worker.pid, &status, 0);
                 }
                 info("[master] all worker terminated, master exit" << endl);
                 exit(0);
             }
             if(global_channel_msg){
                 int active_fd = current_active_io_fd;
                 current_active_io_fd = -1;
                 global_channel_msg = false;

                 // read each channel.
                 for(vector<WorkerProcess>::iterator ite = workers.begin(); ite != workers.end(); ++ite){
                     WorkerProcess& worker = *ite;
                     if(worker.channel == active_fd){
                         on_channel_message(serverfd, worker, workers);
                         break;
                     }
                 }
             }
         }
     }

     return 0;
 }

 // mpclient2 —— multiple processes client
 #include <stdio.h>
 #include <stdlib.h>
 #include <iostream>
 using namespace std;

 #include <unistd.h>
 #include <signal.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <string.h>

 #define err_exit(msg) cout << "[error] " << msg << endl; exit(1)

 struct UserOptions{
     char* server_ip;
     int port;
     int num_clients;
     int sleep_ms;
 };

 void discovery_user_options(int argc, char** argv, UserOptions& options){
     if(argc <= 4){
         cout << "Usage: " << argv[0] << " <server_ip> <port> <num_clients> <sleep_ms>" << endl
             << "server_ip: the ip address of server" << endl
             << "port: the port to connect at" << endl
             << "num_clients: the num to start client." << endl
             << "sleep_ms: the time to sleep in ms." << endl
             << "for example: " << argv[0] << " 127.0.0.1 1990 1000 500" << endl;
         exit(1);
     }

     options.server_ip = argv[1];
     options.port = atoi(argv[2]);
     options.num_clients = atoi(argv[3]);
     options.sleep_ms = atoi(argv[4]);
     if(options.num_clients <= 0){
         err_exit("num_clients must > 0.");
     }
 }

 int connect_server_socket(UserOptions& options){
     int clientfd = socket(AF_INET, SOCK_STREAM, 0);

     if(clientfd == -1){
         err_exit("init socket error!");
     }
     cout << "init socket success!" << endl;

     sockaddr_in addr;
     addr.sin_family = AF_INET;
     addr.sin_port = htons(options.port);
     addr.sin_addr.s_addr = inet_addr(options.server_ip);
     if(connect(clientfd, (const sockaddr*)&addr, sizeof(sockaddr_in)) == -1){
         err_exit("connect socket error!");
     }
     cout << "connect socket success!" << endl;

     return clientfd;
 }

 int main(int argc, char** argv){
     srand(0);
     UserOptions options;
     discovery_user_options(argc, argv, options);
     int* fds = new int[options.num_clients];
     for(int i = 0; i < options.num_clients; i++){
         fds[i] = connect_server_socket(options);
         usleep(10 * 1000); // to prevent the SIGIO error: maybe signal queue overflow!
     }

     for(int i = 0; i < options.num_clients; i++){
         int fd = fds[i];
         if(true){
             char c = 'C'; // control message
             if(send(fd, &c, sizeof(char), 0) <= 0){
                 close(fd);
                 err_exit("send message failed!");
             }
             int required_id = rand();
             if(send(fd, &required_id, sizeof(int), 0) <= 0){
                 close(fd);
                 err_exit("send message failed!");
             }
         }
     }

     bool do_loop = true;
     while(do_loop){
         int ret = 0;
         for(int i = 0; i < options.num_clients; i++){
             int fd = fds[i];
             if(true){
                 char c = 'M'; // data message
                 ret = send(fd, &c, sizeof(char), 0);
                 if(ret <= 0){
                     cout << "send control message to server error" << endl;
                     close(fd);
                     do_loop = false;
                     break;
                 }
                 char msg[] = "client, ping message!";
                 ret = send(fd, msg, sizeof(msg), 0);
                 if(ret <= 0){
                     cout << "send control value to server error" << endl;
                     close(fd);
                     do_loop = false;
                     break;
                 }
                 memset(&msg, 0, sizeof(msg));
             }
             if(true){
                 char buf[1024];
                 memset(buf, 0, sizeof(buf));
                 ret = recv(fd, buf, sizeof(buf), 0);
                 if(ret <= 0){
                     cout << "recv from server error" << endl;
                     close(fd);
                     do_loop = false;
                     break;
                 }
                 cout << "recv from server: " << buf << endl;
             }

             usleep(options.sleep_ms * 1000);
         }
     }

     delete[] fds;

     return 0;
 }