Linux LVS高并发测试程序，内核参数设置，连接数查询指令

最近在公司参与了一个LVS系统测试的项目，学习到了一些关于高并发测试相关的知识，写到博客里记录下

Linux内核参数设置

• 在服务器端需要调整系统最大文件句柄数
  ulimit -n 1000000
  在服务器硬件支持，以及服务较轻量的情况下，最大连接数未必只限于100000,视情况而定吧

• 客户端需要发起大量连接，理论上是最多可支持65535个连接，但实际上Linux的默认可用端口号并没有这么多，需要修改
  sudo echo 1024 65534 > /proc/sys/net/ipv4/ip_local_port_range

查看连接个数

我们一开始用的是
netstat -nap | grep 程序名 | grep ESTABLISHED| wc -l
来查询，这个指令当连接数上万后就变得很慢，而且我们发现了一个略坑的问题，通过这组合指令查询到的结果是波动的，我特意试了10000个连接下，在程序没有报有连接关闭的情况下，通过netstat查出来的连接个数在9998到10000之间波动。最后我换了个指令
ss state ESTABLISHED dport = :端口号 and dst IP | wc -l
通过ss指令查询，不仅速度很快，而且不会出现计数不准的情况，需要注意的是，查询到的连接数要比实际多1

测试程序

/*client1.cpp
 *该程序需要libevent库支持
 *测试有无LVS时连接延时区别的程序
 *程序发起指定数量的连接后，将每秒给服务器发送一个报文，每个连接收到     
 *服务器返回的报文后计算耗时
 */
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

#include 
#include 
#include 
#include 

#include "event2/bufferevent.h"
#include "event2/buffer.h"
#include "event2/listener.h"
#include "event2/util.h"
#include "event2/event.h"

using namespace std;

typedef struct bufferevent* BufferEvent;

map<struct bufferevent*, long long>    bufferevent_map;
map<struct bufferevent*, long long>::iterator  iter;
static int      index_ = 0;

struct cb_arg
{
    struct event *ev;
    struct timeval tv;
};

void timeout_cb(int fd, short event, void *params)
{
    if (iter == bufferevent_map.end()) {
        return ;
    }

    char buf[1024];
    struct cb_arg *arg = (struct cb_arg*)params;
    struct event *timeout = arg->ev;
    struct timeval tv = arg->tv;
    struct timeval start;

    struct bufferevent *bufevt = iter->first;
    gettimeofday(&start,NULL);
    iter->second =  1000000 * start.tv_sec + start.tv_usec;
    sprintf(buf, "%lld", iter->second);
    bufferevent_write(bufevt, buf, strlen(buf));
    printf("fd %u send %s\n", fd, buf);
    index_++;
    iter++;

    evtimer_add(timeout, &tv);
}

void read_cb(struct bufferevent *bufevt, void *arg)
{
    char    line[1024] = {0};
    size_t     n;
    evutil_socket_t fd = bufferevent_getfd(bufevt);
    struct timeval start;
    map<struct bufferevent*, long long>::iterator it_temp;

    it_temp = bufferevent_map.find(bufevt);
    if (it_temp == bufferevent_map.end()) {

        return ;
    }

    gettimeofday(&start,NULL);
    long long now =  1000000 * start.tv_sec + start.tv_usec;
    long long used = now - it_temp->second;

    while (n = bufferevent_read(bufevt, line, 1024), n > 0)
    {
        printf("fd=%u, used time: %lld us\n", fd, used);
    }
}

void error_cb(struct bufferevent *bufevt, short events, void *user_data)
{
    evutil_socket_t fd = bufferevent_getfd(bufevt);

    if (events & BEV_EVENT_EOF)
    {
        printf("Connection closed.\n");
        printf("fd %u close !\n", fd);
        bufferevent_free(bufevt);
    }
    else if (events & BEV_EVENT_ERROR)
    {
        printf("Got an error on the connection: %s\n", strerror(errno));
        printf("fd %u close !\n", fd);
        bufferevent_free(bufevt);
    }
}

int main(int argc, char *argv[])
{
    if (argc != 4) {
        printf("参数个数不正确 %d, 需要3个参数，连接个数，IP，端口号\n", argc - 1);
        return 0;
    }

    vector<string>  argvs;
    for (int i = 1; i < argc; ++i) {
        argvs.push_back(argv[i]);
    }

    timeval     tv = {1, 0};
    int         times = atoi(argvs[0].c_str());
    int         port = atoi(argvs[2].c_str());
    struct sockaddr_in  sin;
    struct bufferevent* bufevt;

    event_base  *evbase = event_base_new();
    if (evbase == NULL) {
        printf("err in new event_base\n");
        return 0;
    }

    struct event*   timeout;
    struct cb_arg   arg;
    timeout = evtimer_new(evbase, timeout_cb, &arg);
    if (timeout == NULL) {
        printf("err in new evtimer\n");
        return 0;
    }
    arg.ev = timeout;
    arg.tv = tv;

    memset(&sin, 0, sizeof(sin));
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = inet_addr(argvs[1].c_str());
    sin.sin_port = htons(port);
    memset(sin.sin_zero, 0x00, 8);

    for (int i = 0; i < times; ++i) {
        int fd = socket(AF_INET, SOCK_STREAM, 0);
        evutil_make_socket_nonblocking(fd);

        bufevt = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE);
        if (bufevt == NULL) {
            printf("err in new bufferevent, fd %u connect failed !\n", fd);
            continue ;
        }

        bufferevent_setcb(bufevt, read_cb, NULL, error_cb, NULL);
        bufferevent_enable(bufevt, EV_READ | EV_WRITE | EV_PERSIST);

        if (bufferevent_socket_connect(bufevt, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
            printf("err in connect with server, fd %u connect failed !\n", fd);
            bufferevent_free(bufevt);
            continue ;
        }
        bufferevent_map.insert(make_pair(bufevt, 0));
    }

    iter = bufferevent_map.begin();

    evtimer_add(timeout, &tv);
    event_base_dispatch(evbase);
    evtimer_del(timeout);
    event_base_free(evbase);

    return 0;
}

/*server.cpp
 *该程序需要libevent库支持
 *简单的回射服务器
 */
#include 
#include 
#include 
#include 

#include 
#include 

#define LISTEN_BACKLOG 1024

static int time_out_count = 0;
static int err_conn_count = 0;
static int err_accept_count = 0;

void do_accept(evutil_socket_t listener, short event, void *arg);
void read_cb(struct bufferevent *bev, void *arg);
void error_cb(struct bufferevent *bev, short event, void *arg);
void write_cb(struct bufferevent *bev, void *arg);

int main(int argc, char *argv[])
{
    if (argc != 2) {
        printf("参数个数不正确 %d\n", argc);
        return 0;
    }

    int port = atoi(argv[1]);
    evutil_socket_t listener;
    listener = socket(AF_INET, SOCK_STREAM, 0);
    assert(listener > 0);
    evutil_make_listen_socket_reuseable(listener);

    struct sockaddr_in sin;
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = 0;
    sin.sin_port = htons(port);

    if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
        perror("bind");
        return 1;
    }

    if (listen(listener, LISTEN_BACKLOG) < 0) {
        perror("listen");
        return 1;
    }

    printf ("Listening...\n");

    evutil_make_socket_nonblocking(listener);

    struct event_base *base = event_base_new();
    assert(base != NULL);
    struct event *listen_event;
    listen_event = event_new(base, listener, EV_READ | EV_PERSIST, do_accept, (void*)base);
    event_add(listen_event, NULL);
    event_base_dispatch(base);

    printf("The End.");
    return 0;
}

void do_accept(evutil_socket_t listener, short event, void *arg)
{
    struct event_base *base = (struct event_base *)arg;
    evutil_socket_t fd;
    struct sockaddr_in sin;
    socklen_t slen;

    fd = accept(listener, (struct sockaddr *)&sin, &slen);
    evutil_make_socket_nonblocking(fd);
    if (fd < 0) {
        printf("err in accept, err ccode %d, 发生accept错误的个数 %d", errno, ++err_accept_count);
        return;
    }

    printf("ACCEPT: fd = %u\n", fd);

    struct bufferevent *bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE);
    bufferevent_setcb(bev, read_cb, NULL, error_cb, arg);
    bufferevent_enable(bev, EV_READ | EV_WRITE | EV_PERSIST);
}

void read_cb(struct bufferevent *bev, void *arg)
{
#define MAX_LINE    256
    char line[MAX_LINE+1];
    int n;
    evutil_socket_t fd = bufferevent_getfd(bev);

    while (n = bufferevent_read(bev, line, MAX_LINE), n > 0) {
        line[n] = '\0';
        bufferevent_write(bev, line, n);
        printf("recv : %s\n", line);
    }
}

void write_cb(struct bufferevent *bev, void *arg) {}

void error_cb(struct bufferevent *bev, short event, void *arg)
{
    evutil_socket_t fd = bufferevent_getfd(bev);
    printf("fd = %u, ", fd);
    if (event & BEV_EVENT_TIMEOUT) {
        time_out_count++;
        printf("Timed out, 发生超时连接个数 %d", time_out_count);
        printf(", err code %d\n", errno);
    }
    else if (event & BEV_EVENT_EOF) {
        printf("connection closed\n");
    }
    else if (event & BEV_EVENT_ERROR) {
        err_conn_count++;
        printf("some other error, 连接被异常关闭个数 %d", err_conn_count);
        printf(", err code %d\n", errno);
    }
    bufferevent_free(bev);
}

/*client2.cpp
 *该程序需要libevent库支持
 *测试高并发量的测试客户端程序，该程序每隔10s就会通过所有连接向服务
 *器发送一个报文，以免长时间没有交互，连接被LVS断掉，同时测试服务器   
 *的负载
 */

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 


#include 
#include 
#include 
#include 

#include "event2/bufferevent.h"
#include "event2/buffer.h"
#include "event2/listener.h"
#include "event2/util.h"
#include "event2/event.h"

using namespace std;

typedef struct bufferevent* BufferEvent;

map<struct bufferevent*, long long>    bufferevent_map;
map<struct bufferevent*, long long>::iterator  iter;
FILE*           fp = NULL;
static int      index_ = 0;
static int      counter = 0;
static int      begin_time = 0;

struct cb_arg
{
    struct event *ev;
    struct timeval tv;
};

void timeout_cb(int fd, short event, void *params)
{
    char buf[1024];
    struct cb_arg *arg = (struct cb_arg*)params;
    struct event *timeout = arg->ev;
    struct timeval tv = arg->tv;

    for (iter = bufferevent_map.begin(); iter != bufferevent_map.end(); ++iter) {
        memset(buf, 0, sizeof(buf));
        struct bufferevent *bufevt = iter->first;

        sprintf(buf, "%lld", iter->second);
        bufferevent_write(bufevt, buf, strlen(buf));
    }

    evtimer_add(timeout, &tv);
}

void read_cb(struct bufferevent *bufevt, void *arg)
{
    char    line[1024] = {0};
    size_t     n;
    evutil_socket_t fd = bufferevent_getfd(bufevt);
    struct timeval start;
    map<struct bufferevent*, long long>::iterator it_temp;

    it_temp = bufferevent_map.find(bufevt);
    if (it_temp == bufferevent_map.end()) {

        return ;
    }

    while (n = bufferevent_read(bufevt, line, 1024), n > 0)
    {
        ;
    }
}


void write_count(int count)
{
    char  buf[1024] = {0};
    int   now_time = (int)time(NULL);

    sprintf(buf, "pass %d s, now connect count %d\n", now_time - begin_time, count);
    fputs(buf, fp);
}

void error_cb(struct bufferevent *bufevt, short events, void *user_data)
{
    evutil_socket_t fd = bufferevent_getfd(bufevt);

    if (events & BEV_EVENT_EOF)
    {
        counter--;
        printf("Connection closed.\n");
        printf("fd %u close ! count %d\n", fd , counter);
        printf("now EST connect count %d\n", counter);
        bufferevent_free(bufevt);
        bufferevent_map.erase(bufevt);
        write_count(counter);
    }
    else if (events & BEV_EVENT_ERROR)
    {
        counter--;
        printf("Got an error on the connection: %d\n", errno);
        printf("fd %u close ! count %d\n", fd , counter);
        printf("now EST connect count %d\n", counter);
        bufferevent_free(bufevt);
        bufferevent_map.erase(bufevt);
        write_count(counter);
    }
}

void create_file()
{
    char  buf[1024] = {0};
    timeval tv;
    gettimeofday(&tv, NULL);

    int   time_ = (int) (1000000 * tv.tv_sec + tv.tv_usec);

    sprintf(buf, "result_%d.txt", time_);

    fp = fopen(buf, "w+");
    if (fp == NULL) {
        printf("create result file failed!\n");
        exit(-1);
    }
}

int main(int argc, char *argv[])
{
    if (argc != 4) {
        printf("参数个数不正确 %d, 需要3个参数，连接个数，IP，端口号\n", argc - 1);
        return 0;
    }

    vector<string>  argvs;
    for (int i = 1; i < argc; ++i) {
        argvs.push_back(argv[i]);
    }

    create_file();

    timeval     tv = {10, 0};
    int         times = atoi(argvs[0].c_str());
    int         port = atoi(argvs[2].c_str());
    struct sockaddr_in  sin;
    struct bufferevent* bufevt;

    event_base  *evbase = event_base_new();
    if (evbase == NULL) {
        printf("err %d in new event_base\n", errno);
        return 0;
    }

    struct event*   timeout;
    struct cb_arg   arg;
    timeout = evtimer_new(evbase, timeout_cb, &arg);
    if (timeout == NULL) {
        printf("err in new evtimer\n");
        return 0;
    }
    arg.ev = timeout;
    arg.tv = tv;

    memset(&sin, 0, sizeof(sin));
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = inet_addr(argvs[1].c_str());
    sin.sin_port = htons(port);
    memset(sin.sin_zero, 0x00, 8);

    for (int i = 0; i < times; ++i) {
        int fd = socket(AF_INET, SOCK_STREAM, 0);
        evutil_make_socket_nonblocking(fd);
        bufevt = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE);
        if (bufevt == NULL) {
            printf("err %d in new bufferevent, fd %u connect failed !\n", fd, errno);
            continue ;
        }

        bufferevent_setcb(bufevt, read_cb, NULL, error_cb, NULL);
        bufferevent_enable(bufevt, EV_READ | EV_WRITE | EV_PERSIST);

        if (bufferevent_socket_connect(bufevt, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
            printf("err %d in connect with server, fd %u connect failed !\n", fd, errno);
            continue ;
        }

        char buf[64] = {0};
        sprintf(buf, "%d", i);
        bufferevent_map.insert(make_pair(bufevt, 0));
        bufferevent_write(bufevt, buf, strlen(buf));
        usleep(10);
        counter++;
    }

    printf("now EST connect count %d\n", counter);
    write_count(counter);

    evtimer_add(timeout, &tv);
    event_base_dispatch(evbase);
    evtimer_del(timeout);
    event_base_free(evbase);

    return 0;
}