单进程支持高并发

学习nginx，很好奇如何做到单个进程支持几万并发（当时用3万client连接过去，都能保持连接）。

在linux服务器上：

1. 不使用select，因为fd_set限制为1024，只能有1024个连接，而且慢。
   使用epoll，man epoll可以看到资料。
2. 尽管使用了epoll，但是超过1024连接时，还是会有too many open files的错误，每个连接都是一个文件句柄，所以要修改这个限制。
   以root运行程序前，修改这个限制：
   g++ server2.cpp -o nginx.winlin
   sudo su
   ulimit -n 65535
   ./nginx.winlin 1985 65535 > /dev/null 2>&1
3. 模拟客户端：
   g++ client.cpp -o client
   sudo su
   ulimit -n 65535
   ./client 10000 10.33.0.190 1985 > /dev/null 2>&1 &
   
4. 统计连接：
   sudo bash status.sh 10 1985
   nginx-origin:0 master:23429 worker:23430 listen-1985:(24584/nginx.winlin) all:24836 established:23531
   
   

这样可以打开几万个连接：

clients:20001 established:20000

演示代码如下：

#include<sys/types.h>
#include<sys/socket.h>
#include<errno.h>
#include<string.h>
#include<sys/ioctl.h>
#include<netinet/in.h>
#include<sys/epoll.h>

#include<iostream>
using namespace std;

int main(int argc, char** argv)
{
    if(argc <= 2){
        cout << "Usage: " << argv[0] << " <port> <max_clients>" << endl
            << "  port: the listening port." << endl
            << "  max_clients: the max accept clients." << endl;
        return -1;
    }
    
    int port = atoi(argv[1]);
    int max_clients = atoi(argv[2]);
    cout << "port:" << port << " max_clients:" << max_clients << " pid:" << getpid() << endl;
    
    int server_socket = socket(PF_INET, SOCK_STREAM, 0);
    if(server_socket == -1){
        cout << "init socket error: " << strerror(errno) << endl;
        exit(-1);
    }
    cout << "init socket success: fd=" << server_socket << "!" << endl;
    
    int flag;
    if(ioctl(server_socket, FIONBIO, &flag) == -1){
        cout << "set socket to un-blocked error: " << strerror(errno) << endl;
        exit(-1);
    }
    cout << "set socket to non-block mode success!" << endl;
    
    sockaddr_in addr;
    memset(&addr, 0, sizeof(sockaddr_in));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    // bind
    if(bind(server_socket, (const sockaddr*)&addr, sizeof(sockaddr)) == -1){
        cout << "socket bind error: " << strerror(errno) << endl;
        exit(-1);
    }
    cout << "bind socket success!" << endl;
    
    if(listen(server_socket, 10) == -1){
        cout << "socket listen error: " << strerror(errno) << endl;
        exit(-1);
    }
    cout << "listen socket success!" << endl;
    
    int epoll = epoll_create(max_clients);
    if(epoll == -1){
        cout << "create event poll error: " << strerror(errno) << endl;
        exit(-1);
    }
    cout << "create event poll success: fd=" << epoll << "!" << endl;
    
    if(true){
        epoll_event ev;
        memset(&ev, 0, sizeof(epoll_event));
        ev.events = EPOLLIN | EPOLLOUT; // focus on read|write event.
        ev.data.fd = server_socket;
        if(epoll_ctl(epoll, EPOLL_CTL_ADD, server_socket, &ev) == -1){
            cout << "event poll ctl error: " << strerror(errno) << endl;
            exit(-1);
        }
        cout << "event poll ctl success!" << endl;
    }
    
    epoll_event* ee = new epoll_event[max_clients];
    for(;;){
        int active_fds = epoll_wait(epoll, ee, max_clients, 100);
        if(active_fds == -1){
            cout << "event poll wait error: " << strerror(errno) << endl;
            exit(-1);
        }
        cout << "event poll wait success: active_fds=" << active_fds << "!" << endl;
        
        for(int i = 0; i < active_fds; i++){
            // if client is coming.
            if(ee[i].data.fd == server_socket){
                int client = accept(server_socket, (sockaddr*)NULL, NULL);
                
                if(client == -1){
                    cout << "accept client error: " << strerror(errno) << endl;
                    exit(-1);
                }
                cout << "accept client success: client=" << client << "!" << endl;
                
                epoll_event ev;
                memset(&ev, 0, sizeof(epoll_event));
                ev.events = EPOLLIN;
                ev.data.fd = client;
                if(epoll_ctl(epoll, EPOLL_CTL_ADD, client, &ev) == -1){
                    cout << "client event poll ctl error: " << strerror(errno) << endl;
                    exit(-1);
                }
                cout << "client event poll ctl success!" << endl;
            }
            // do read-write
            else{
                int client = ee[i].data.fd;
                
                // read
                char buffer[1024];
                int len;
                if((len = read(client, buffer, sizeof(buffer))) <= 0){
                    cout << "read from client error: " << strerror(errno) << endl;
                    exit(-1);
                }
                cout << "read from client success: " << len << "bytes received!" << endl;
                
                // write
                char msg[] = "server is ok!";
                if(send(client, msg, sizeof(msg), 0) <= 0){
                    cout << "write to client error: " << strerror(errno) << endl;
                    exit(-1);
                }
                cout << "write to client success!" << endl;
            }
        }
    }
    
    cout << "server cleanup and exit." << endl;
    close(epoll);
    close(server_socket);
    
    return 0;
}

模拟的client，定时发送HTTP请求：

#include<signal.h>
#include<errno.h>
#include<sys/socket.h>
#include<netinet/in.h>
#include<arpa/inet.h>

#include<iostream>
#include<string>
#include<sstream>
using namespace std;

void on_signal_active(int sno)
{
    cout << "get a signal: " << sno
        << ", SIGINT=" << SIGINT << ", SIGTERM=" << SIGTERM << endl;

    if(sno == SIGINT || sno == SIGTERM){
        cout << "get a exit signal, exit" << endl;
        exit(0);
    }
}

class Client
{
private:
    int client;
    
public:
    Client(){
        client = 0;
    }
    ~Client(){
        if(client > 0){
            close(client);
        }
    }
    
    bool initialize(int id, char* host, int port){
        cout << "#" << id << ":" << "client start..." << endl;

        cout << "#" << id << ":" << "register the signal function" << endl;
        signal(SIGINT, on_signal_active);
        signal(SIGTERM, on_signal_active);

        client = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
        if(client < 0){
            cout << "#" << id << ":" << "init socket error!" << "msg: " << strerror(errno) << endl;
            return false;
        }
        cout << "#" << id << ":" << "socket init success" << endl;

        sockaddr_in server_address;

        memset(&server_address, 0, sizeof(server_address));
        server_address.sin_family = AF_INET;
        server_address.sin_addr.s_addr = inet_addr(host);
        server_address.sin_port = htons(port);

        if(connect(client, (sockaddr*)&server_address, sizeof(sockaddr_in)) < 0){
            cout << "#" << id << ":" << "connect to server error." << "msg: " << strerror(errno) << endl;
            return false;
        }
        
        cout << "#" << id << ":" << "socket connect success" << endl;
        return true;
    }
        
    bool run(int id, char* host){
        // HTTP request
        if(true){
            stringstream ss(stringstream::in | stringstream::out);
            string s0x20 = string(1, (char)0x20);
            string sCRLF = string(1, (char)0x0D) + string(1, (char)0x0A);
            ss << string("GET") << s0x20 // Command: GET 47 45 54 20  
                << string("/index.html") << s0x20 // URI: /  2F 20 
                << string("HTTP/1.1") << sCRLF // ProtocolVersion: HTTP/1.1 48 54 54 50 2F 31 2E 31 0D 0A
                << string("Host:") << s0x20 << string(host) << sCRLF // Host:www.baidu.com.. 48 6F 73 74 3A 20 77 77 77 2E 62 61 69 64 75 2E 63 6F 6D 0D 0A 
                << string("Connection:") << s0x20 << string("keep-alive") << sCRLF // Connection: keep-alive..   43 6F 6E 6E 65 63 74 69 6F 6E 3A 20 6B 65 65 70 2D 61 6C 69 76 65 0D 0A
                << string("User-Agent:") << s0x20 << string("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.162 Safari/535.19") << sCRLF // UserAgent:  Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.162 Safari/535.19
                << string("Accept:") << s0x20 << string("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") << sCRLF//Accept:  text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
                << string("Accept-Encoding:") << s0x20 << string("gzip,deflate,sdch") << sCRLF//Accept-Encoding:  gzip,deflate,sdch
                << string("Accept-Language:") << s0x20 << string("zh-CN,zh;q=0.8") << sCRLF//Accept-Language:  zh-CN,zh;q=0.8
                << string("Accept-Charset:") << s0x20 << string("GBK,utf-8;q=0.7,*;q=0.3") << sCRLF//Accept-Charset:  GBK,utf-8;q=0.7,*;q=0.3
                << string("Cookie:") << s0x20 << string("BAIDUID=3BABFCA8602431FA83A9AFBE4FBD6F85:FG=1;") << s0x20 << string("MCITY=-158%3A") << sCRLF//Cookie:  BAIDUID=3BABFCA8602431FA83A9AFBE4FBD6F85:FG=1; MCITY=-158%3A
                // << string("Key:") << s0x20 << string("Value") << sCRLF //Key: Value
                << sCRLF; //HeaderEnd: CRLF
            
            ss.seekg(0, ios::end);
            int len = ss.tellg();
            ss.seekg(0, ios::beg);
            
            char* buffer = new char[len];
            ss.read(buffer, len);
            
            int ret = send(client, buffer, len, 0);
            delete buffer;
            
            if(ret <= 0){
                cout << "send HTTP request error" << endl;
                return false;
            }
            
            cout << "send HTTP request success" << endl;
        }
        
        // HTTP response
        if(true){
            char buffer[1024];
            memset(buffer, 0, sizeof(buffer));
            
            if(read(client, buffer, sizeof(buffer)) <= 0){
                cout << "receive HTTP response error" << "msg: " << strerror(errno)  << endl;
                return false;
            }
            
            cout << "receive HTTP reponse success" << "msg: " << strerror(errno)  << endl;
        }
        
        return true;
    }
};

int main(int argc, char** argv)
{
    int i = 0;
    int id = 0;
    
    if(argc <= 3){
        cout << "Usage: " << argv[0] << " <client_count> <host> <port>" << endl
            << "  client_count: the client count to run." << endl
            << "  host: the host to connect to." << endl
            << "  port: the port to connect to." << endl;
        return -1;
    }
    int client_count = atoi(argv[1]);
    char* host = argv[2];
    int port = atoi(argv[3]);
    
    while(true){
        Client clients[client_count];
        
        for(int j = 0; j < client_count; j++){
            Client& client = clients[j];
            
            if(!client.initialize(++i, host, port)){
                goto error;
            }
        }
        
        while(true){
            for(int j = 0; j < client_count; j++){
                Client& client = clients[j];
                if(!client.run(++id, host)){
                    goto error;
                }
            }
        }

error:                
        cout << "server error, reinitialize all " << client_count << " clients" << endl;
        usleep(10 * 1000 * 1000);
    }
    
    return 0;
}

统计连接脚本：

#!/bin/bash
#vi /bin/status
#################################################################################################
#脚本
if [ $# -lt 2 ]; then
    echo "Usage: $0 <sleep_time> <port>";
    echo "    <sleep_time> the sleep time in seconds.";
    echo "    <port> to display the program which listening at the specified port.";
    echo "    e.g. $0 3 80";
    exit -1;
fi

sleep_time=$1
port=$2

echo "sleep_time:${sleep_time}s"
for((i=0;;i++))
do
    program_listening=`netstat -ntlp|grep ${port}|awk -F "LISTEN" '{print $2}'|awk '{print $1}'`
    if [ -z "${program_listening}" ]; then
        program_listening="None";
    fi

    nginx_master_pid=`ps aux|grep nginx|grep master|awk '{print $2}'`
    if [ -z "${nginx_master_pid}" ]; then
        nginx_master_pid="0";
    fi

    nginx_origin_pid=`ps aux|grep nginx|grep bin|awk '{print $2}'`
    if [ -z "${nginx_origin_pid}" ]; then
        nginx_origin_pid=`ps aux|grep nginx|grep winlin|awk '{print $2}'`
        if [ -z "${nginx_origin_pid}" ]; then
            nginx_origin_pid="0";
        fi
    fi
    if [ ${nginx_origin_pid} == ${nginx_master_pid} ]; then
        # if origin equals to master, the origin is master actually.
        nginx_origin_pid="0";
    fi

    nginx_worker_pid=`ps aux|grep nginx|grep worker|awk '{print $2}'`
    if [ -z "${nginx_worker_pid}" ]; then
        nginx_worker_pid="0";
    fi

    local_ip_address=`/sbin/ifconfig|grep "inet\ "|awk 'NR==1 {print $2}'|awk -F ':' '{print $2}'`
    established_clients=`netstat -nt|grep ESTABLISHED|grep "0\ ${local_ip_address}:${port}\ "|wc -l`
    all_clients=`netstat -nat|grep "0\ ${local_ip_address}:${port}\ "|wc -l`

    echo "nginx-origin:${nginx_origin_pid} master:${nginx_master_pid} worker:${nginx_worker_pid} listen-${port}:(${program_listening}) all:${all_clients} established:${established_clients}"
    sleep ${sleep_time}
done