Nginx网络epoll多进程系列：tcp服务器仿nginx多进程和多路IO的实现

http://blog.csdn.net/ypbsyy/article/details/38046267

1、服务端代码，开启8个工作进程

#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include

#define WORKER_MAX        1024
#define EVENT_LIST_MAX    128
#define EVENT_MAX         12
#define WORK_REAL         8
#define SERVER_PORT       8080

static int workers[WORKER_MAX];
static int icEpollFd = -1;
static int cur_pid;

typedef int (*PFCALLBACL)(struct epoll_event *);
typedef struct EPOLL_DATA_S
{
    int iEpoll_Fd;
    int iEvent_Fd;
    PFCALLBACL pfCallBack;
}Epoll_Data_S;

/* 互斥量 */
static pthread_mutex_t *mutex;

/* 创建共享的mutex */
static void initMutex(void)
{
    pthread_mutexattr_t attr;
    int ret;

    //设置互斥量为进程间共享
    mutex=(pthread_mutex_t*)mmap(NULL, sizeof(pthread_mutex_t), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANON, -1, 0);
    if( MAP_FAILED==mutex) {
        perror("mutex mmap failed");
        return;
    }

    //设置attr的属性
    pthread_mutexattr_init(&attr);
    ret = pthread_mutexattr_setpshared(&attr,PTHREAD_PROCESS_SHARED);
    if(ret != 0) {
        fprintf(stderr, "mutex set shared failed");
        return;
    }
    pthread_mutex_init(mutex, &attr);

    return;
}

static int startup(unsigned short port)
{
    struct sockaddr_in servAddr;
    unsigned value = 1;
    int listenFd;

    memset(&servAddr, 0, sizeof(servAddr));

    //协议域（ip地址和端口）
    servAddr.sin_family = AF_INET;

    //绑定默认网卡
    servAddr.sin_addr.s_addr = htonl(INADDR_ANY);

    //端口
    servAddr.sin_port = htons(port);

    //创建套接字
    if ((listenFd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        printf("create socket error: %s(errno: %d)\n",strerror(errno),errno);
        return 0;
    }

    setsockopt(listenFd, SOL_SOCKET, SO_REUSEADDR, &value, sizeof(value));

    //绑定套接字
    if (bind(listenFd, (struct sockaddr *)&servAddr, sizeof(servAddr))) {
        printf("bind socket error: %s(errno: %d)\n",strerror(errno),errno);
        return 0;
    }
    //开始监听，设置最大连接请求
    if (listen(listenFd, 10) == -1) {
        printf("listen socket error: %s(errno: %d)\n",strerror(errno),errno);
        return 0;
    }

    return listenFd;
}

static int create_workers(unsigned int worker_num)
{
    unsigned int i;
    unsigned int real_num = worker_num;
    int pid;

    if (real_num > WORKER_MAX)
    {
        real_num = WORKER_MAX;
    }

    for (i = 0; i < real_num; i++)
    {
        pid = fork();
        if (0 == pid)
        {
            return 0;
        }
        else if (0 < pid)
        {
            workers[i] = pid;
            continue;
        }
        else
        {
            printf("fork error\r\n");
            return 0;
        }
    }

    return 1;
}

/* 创建epoll */
static int create_epoll(unsigned int event_num)
{
    int epoll_fd;

    epoll_fd = epoll_create(event_num);
    if (-1 == epoll_fd)
    {
        printf("epoll create failed\r\n");
        return -1;
    }

    return epoll_fd;
}

/* 将事件加到epoll */
static void add_event_epoll(int iEpoll_Fd, int iEvent_Fd, PFCALLBACL pfCallBack)
{
    int                  op = EPOLL_CTL_ADD;
    struct epoll_event   ee;
    Epoll_Data_S *data;

    data = malloc(sizeof(Epoll_Data_S));
    if (NULL == data)
    {
        return;
    }

    /* 设置私有数据 */
    data->iEpoll_Fd = iEpoll_Fd;
    data->iEvent_Fd = iEvent_Fd;
    data->pfCallBack = pfCallBack;

    ee.events = EPOLLIN | EPOLLOUT | EPOLLHUP;
    //ee.events = EPOLLIN|EPOLLET;
    ee.data.ptr = (voidvoid *)data;

    if (epoll_ctl(iEpoll_Fd, op, iEvent_Fd, &ee) == -1)
    {
        printf("epoll_ctl(%d, %d) failed", op, iEvent_Fd);
        return;
    }

    return;
}

/* 从epoll删除事件 */
static void del_event_epoll(int iEpoll_Fd, int iEvent_Fd)
{
    int op = EPOLL_CTL_DEL;

    if (epoll_ctl(iEpoll_Fd, op, iEvent_Fd, NULL) == -1)
    {
        printf("epoll_ctl(%d, %d) failed", op, iEvent_Fd);
    }

    return;
}

static int make_socket_non_blocking(int fd)
{
  int flags, s;

  flags = fcntl (fd, F_GETFL, 0);
  if (flags == -1)
    {
      perror ("fcntl");
      return -1;
    }

  flags |= O_NONBLOCK;
  s = fcntl (fd, F_SETFL, flags);
  if (s == -1)
    {
      perror ("fcntl");
      return -1;
    }

  return 0;
}

/* 处理Receive事件 */
static int proc_receive(struct epoll_event *pstEvent)
{
    char buff[4096];   /* 缓存 */
    int len;
    Epoll_Data_S *data = (Epoll_Data_S *)(pstEvent->data.ptr);
    int iEpoll_Fd = data->iEpoll_Fd;
    int iEvent_Fd = data->iEvent_Fd;

    if (pstEvent->events & EPOLLIN)
    {
        while(1)
        {
            /* 读取数据 */
            len = (int)recv(iEvent_Fd, buff, sizeof(buff), 0);
            //printf("proc_receive iEvent_Fd = %d pid = %d len = %d\r\n", iEvent_Fd, getpid(), len);
            if (len <= 0)
            {
                if (errno == EINTR)
                {
                   continue;
                }

                del_event_epoll(iEpoll_Fd, iEvent_Fd);
                close(iEvent_Fd);
                free(data);
            }
            else if (len > 0)
            {
                buff[len] = '\0';
                printf("pid %d receive data: %s\r\n", cur_pid, buff);
                //usleep(10000);
            }

            break;
        }
    }
    else if (pstEvent->events & EPOLLHUP)
    {
        printf("receive EPOLLHUP or EPOLLOUT\r\n");

        del_event_epoll(iEpoll_Fd, iEvent_Fd);
        close(iEvent_Fd);
        free(data);
    }
    else
    {
       // printf("receive others pstEvent->events=%d\r\n", pstEvent->events);
    }

    return 0;
}

/* 处理Accept事件 */
static int proc_accept(struct epoll_event *pstEvent)
{
    int newFd;
    Epoll_Data_S *data = (Epoll_Data_S *)(pstEvent->data.ptr);
    int iEpoll_Fd = data->iEpoll_Fd;
    int iEvent_Fd = data->iEvent_Fd;

    if (pthread_mutex_trylock(mutex)==0)
    {
        while(-1 != (newFd = accept(iEvent_Fd, (struct sockaddr *)NULL, NULL)))
        {
            make_socket_non_blocking(newFd);
            //printf("accept pid = %d\r\n", getpid());
            add_event_epoll(icEpollFd, newFd, proc_receive);
        }

        pthread_mutex_unlock(mutex);
    }

    return 0;
}

static void handleterm(int sig)
{
    int i;

    for (i = 0; i < WORK_REAL; i++)
    {
        /* 杀掉子进程 */
        kill(workers[i], SIGTERM);
    }

    return;
}

static void proc_epoll(int iEpollFd, int timeout)
{
    int iEventNum;
    int i;
    struct epoll_event events[EVENT_LIST_MAX];

    iEventNum = epoll_wait(iEpollFd, events, EVENT_LIST_MAX, timeout);
    for (i = 0; i < iEventNum; i++)
    {
        Epoll_Data_S *data = (Epoll_Data_S *)(events[i].data.ptr);
        data->pfCallBack(&(events[i]));
    }

    return;
}

int main()
{
    int iServerFd = -1;
    int bParent;
    int iEpollFd = -1;

    /* 初始化互斥量 */
    initMutex();

    /* 初始化，创建监听端口 */
    iServerFd = startup(SERVER_PORT);
    if (-1 == iServerFd)
    {
        return 0;
    }

    make_socket_non_blocking(iServerFd);

    /* 父进程创建epoll */
    iEpollFd = create_epoll(EVENT_MAX);
    if (-1 == iEpollFd)
    {
        close(iServerFd);
        return 0;
    }

    /* 将监听端口加到epoll */
    add_event_epoll(iEpollFd, iServerFd, proc_accept);

    /* 创建子进程  */
    bParent = create_workers(WORK_REAL);

    /* 主进程 */
    if (bParent)
    {
        while(1)
        {
            signal(SIGTERM, handleterm);
            pause();
        }
    }
    else
    {
          /* 子进程创建epoll */
          icEpollFd = create_epoll(EVENT_MAX);
          if (-1 == icEpollFd)
          {
             close(iServerFd);
             return 0;
          }

          cur_pid = getpid();

          while (1)
          {
            /* 处理父epoll消息 */
            proc_epoll(iEpollFd, 50);

            /* 处理子epoll消息 */
            proc_epoll(icEpollFd, 50);
        }
    }

    return 0;
}

2、客户端代码，发起高并发连接，暂定连接数为2000，可以自己调

#include
#include
#include
#include
#include
#include
#include
#include
#include
#include

const int MAXLINE = 5;
int count = 1;

static int make_socket_non_blocking(int fd)
{
  int flags, s;

  flags = fcntl (fd, F_GETFL, 0);
  if (flags == -1)
    {
      perror ("fcntl");
      return -1;
    }

  flags |= O_NONBLOCK;
  s = fcntl (fd, F_SETFL, flags);
  if (s == -1)
    {
      perror ("fcntl");
      return -1;
    }

  return 0;
}

void sockconn()
{
    int sockfd;
    struct sockaddr_in server_addr;
    struct hostent *host;
    char buf[100];
    unsigned int value = 1;

    host = gethostbyname("127.0.0.1");
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd == -1) {
        perror("socket error\r\n");
        return;
    }

    //setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &value, sizeof(value));

    //make_socket_non_blocking(sockfd);

    bzero(&server_addr, sizeof(server_addr));

    server_addr.sin_family = AF_INET;
    server_addr.sin_port = htons(8080);
    server_addr.sin_addr = *((struct in_addr*) host->h_addr);

    int cn = connect(sockfd, (struct sockaddr *) &server_addr,
            sizeof(server_addr));
    if (cn == -1) {
        printf("connect error errno=%d\r\n", errno);
        return;

    }
//  char *buf = "h";
    sprintf(buf, "%d", count);
    count++;
    write(sockfd, buf, strlen(buf));
    close(sockfd);

    printf("client send %s\r\n", buf);

    return;
}

int main(void) {

 int i;
 for (i = 0; i < 2000; i++)
 {
      sockconn();
 }

 return 0;
}

3、测试结果

1秒多时间内，处理了2000个连接

4、需要注意的问题连接太多时，open files too many问题

localhost:/share/code # ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 7900
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) 865300
open files                      (-n) 1024
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 7900
virtual memory          (kbytes, -v) 2491600
file locks                      (-x) unlimited
localhost:/share/code #

解决方法：

localhost:/share/code # ulimit -n 100000
localhost:/share/code # ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 7900
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) 865300
open files                      (-n) 100000
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 7900
virtual memory          (kbytes, -v) 2491600
file locks                      (-x) unlimited
localhost:/share/code #

2）客户端connect错误

原因：处于TIME-WAIT状态的socket太多，socket不够用

解决：减少TIME-WAIT时间

localhost:/share/code # echo "1" > /proc/sys/net/ipv4/tcp_tw_recycle
localhost:/share/code # echo "1" > /proc/sys/net/ipv4/tcp_tw_reuse
localhost:/share/code #