golang UDP发送实在太慢了 系统调用，上下文切换消耗可观。

目录

UDP服务 PPS关键点

golang UDP WriteToUDP太慢了

如果使用C++呢 直接调用sendto呢？

C++多线程 sento

C++多线程 sendmmsg

C++ 多socket同时发送

使用recvfrom和recvmmsg,结果没有区别

---

UDP服务 PPS关键点

1. 上下文切换 在大量小包的情况下，每次调用sendto, recvfrom都会进行一次上下文切换，消耗不小。解决方法就是使用sendmmsg,recvmmsg这两个系统调用，可以批量发送。也可以使用用户态协议栈，彻底不需要上下文切换。

2. 锁 竞争，写入同一个socket需要加锁(看内核实现),并发写会因抢写锁而阻塞。解决方案，使用多个socket,多个地址。或者使用SO_REUSEPORT选项，在一个ip:port上创建多个socket。

3. 拷贝，无论读写都要在内核态和用户态之间复制数据，如果数据量很大，也会很消耗性能。 可以使用零拷贝技术解决一部分问题。也可以使用用户态协议栈。

4. 软中断，网卡收包会使得软中断频繁。当收包量巨大时，使用轮询效率更高。用户态协议栈也可以解决些问题。

5. 发送接收缓冲区大小， 如果包量巨大，流量也很大，可以设置更大的收发缓冲区，既可以避免接收不及时导致的丢包，也可以避免发送太快导致缓冲区不足而阻塞。

golang UDP WriteToUDP太慢了

由于发送大量的小包，导致系统调用过于频繁，Packet Per Second: PPS=152300

golang目前没有提供C中的API: sendmmsg, 这是个批量发送数据包的接口，一次系统调用可以发多个包。

udp_test.go

package main

import (
	"fmt"
	"net"
	"sync"
	"testing"
	"time"
)

func ListenUDP(addrStr string) (*net.UDPAddr, *net.UDPConn, error) {
	fmt.Println("Try to listen:", addrStr)
	addr, err := net.ResolveUDPAddr("udp", addrStr)
	if err != nil {
		fmt.Println("resolve udp addr failed:", err)
		return nil, nil, err
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		fmt.Println("listen udp svr failed:", err)
		return nil, nil, err
	}

	return addr, conn, nil
}
func TestUDPSendSpeed(t *testing.T) {
	_, svrConn, err := ListenUDP("127.0.0.1:21117")
	if err != nil {
		t.Fatal(err)
	}
	svrConn.SetWriteBuffer(300000000) //300MB
	cliAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:29875")
	data := []byte("Hello, test small packet")
	N := 100000
	M := 4
	begin := time.Now().UnixNano()
	var wg sync.WaitGroup
	for j := 0; j < M; j++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for i := 0; i < N; i++ {
				n, err := svrConn.WriteToUDP(data, cliAddr)
				if err != nil || n != len(data) {
					t.Fatal(err, n, len(data))
				}
			}
		}()
	}
	wg.Wait()
	end := time.Now().UnixNano()
	usedTimeMS := float64(end-begin) / 1e6
	usedTimeS := float64(end-begin) / 1e9
	fmt.Printf("Send %d packets, use %.2fms, PPS:%.0f\n", N*M, usedTimeMS, float64(N*M)/usedTimeS)

}

 

 

go test -v -cpuprofile=cpu.prof udp_test.go 
=== RUN   TestUDPSendSpeed
Try to listen: 127.0.0.1:21117
Send 1000000 packets, use 6565.98ms, PPS:152300
--- PASS: TestUDPSendSpeed (6.57s)
PASS
ok      command-line-arguments  6.713s

#性能分析
go tool pprof -http=:7897 cpu.prof 
Serving web UI on http://localhost:7897

 

如果使用C++呢 直接调用sendto呢？

经过多次运行，发线单线程效率最好，可达到25W PPS。 这也是单线程的极限了。

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
//#include 

#include 
#include 
#include 
using namespace std;

#define BUF_SIZE 64
char data[BUF_SIZE];
sockaddr_in BuildAddr(const char *ip, int port)
{
    sockaddr_in svrAddr;
    memset(&svrAddr, 0, sizeof(svrAddr));
    svrAddr.sin_family = AF_INET;
    svrAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    svrAddr.sin_port = htons(port);
    return svrAddr;
}
int ListenUDP(const char *ip, int port)
{
    sockaddr_in svrAddr = BuildAddr(ip, port);
    int sockFd = socket(AF_INET, SOCK_DGRAM, 0);
    if (sockFd < 0)
    {
        perror("create socket failed!");
        return sockFd;
    }
    if (bind(sockFd, (sockaddr *)&svrAddr, sizeof(svrAddr)) < 0)
    {
        perror("bind svr addr failed");
        return -1;
    }
    return sockFd;
}

void SendPacket(int n, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char *data = new char[size];

    for (int i = 0; i < n; i++)
    {
        int ret = sendto(svrFd, data, size, 0, (sockaddr *)&targetAddr, sizeof(targetAddr));
        if (ret < 0)
        {
            perror("sendto failed");
        }
        //printf("send packet:%d\n", ret);
    }

    delete[] data;
}
void SendMsgPacket(int n, int m, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char *data = new char[size];
    struct iovec msg1;
    msg1.iov_base = data;
    msg1.iov_len = 1;
    mmsghdr *msg = new mmsghdr[m];
    for (int i = 0; i < m; i++)
    {
        msg[i].msg_len = 1;
        msg[i].msg_hdr.msg_name = &targetAddr;
        msg[i].msg_hdr.msg_namelen = sizeof(targetAddr);
        msg[i].msg_hdr.msg_iov = &msg1;
        msg[i].msg_hdr.msg_iovlen = 1;
    }
    for (int i = 0; i < n; i++)
    {
        int t = m;
        while (t > 0)
        {
            int ret = sendmmsg(svrFd, msg, m, 0);
            if (ret < 0)
            {
                perror("sendto failed");
            }
            //printf("send %d\n", t);
            t -= ret;
        }
    }
    delete[] msg;
    delete[] data;
}
void TestSendThread(int n, int svrFd)
{
    SendPacket(n, 32, svrFd, "127.0.0.1", 55667);
}
void TestSendMsgThread(int n, int m, int svrFd)
{
    SendMsgPacket(n, m, 32, svrFd, "127.0.0.1", 55667);
}
int main(int argc, char *argv[])
{
    int svrFd = ListenUDP("127.0.0.1", 65432);
    if (svrFd < 0)
    {
        perror("listen upd failed");
    }
    else
    {
        printf("UDP Server at 127.0.0.1:65432\n");
    }
    //ProfilerStart("udpsend.prof");
    auto begin = chrono::steady_clock::now();
    int n = 100000;
    int m = 100;
    TestSendThread(n, svrFd);
    //TestSendMsgThread(n, m, svrFd);
    auto end = chrono::steady_clock::now();
    //ProfilerStop();
    double dr_ms = std::chrono::duration(end - begin).count();
    printf("Use time:%.2fms, PPS:%.0f\n", dr_ms, n / (dr_ms / 1000));
    close(svrFd);
}
//g++ -O3 -o send_speed send_speed.cpp -lpthread

//g++  -g -pg -o send_speed send_speed.cpp -lpthread

/*
g++ -O3 -o send_speed send_speed.cpp 
valgrind --tool=callgrind ./send_speed
python ../../gprof2dot/gprof2dot.py -f callgrind -n 10 -s callgrind.out.2938509 > val.dot
dot -Tpng val.dot -o val.png
python -m http.server 7897
*/

./send_speed 
UDP Server at 127.0.0.1:65432
Use time:383.56ms, PPS:260712

./send_speed 
UDP Server at 127.0.0.1:65432
Use time:382.87ms, PPS:261182

 ./send_speed 
UDP Server at 127.0.0.1:65432
Use time:381.96ms, PPS:261810

经过分析发现，sendto系统调用占用了34%

 

C++多线程 sento

120WQPS

本机是8核，所以用8线程。发送速度几乎提升了8倍。CPU终于到了755%但是golang使用多现成确不能提升。golang的writeToUDP自带锁。所以只能达到单核心的25W PPS。

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
//#include 

#include 
#include 
#include 
using namespace std;

#define BUF_SIZE 64
char data[BUF_SIZE];
sockaddr_in BuildAddr(const char *ip, int port)
{
    sockaddr_in svrAddr;
    memset(&svrAddr, 0, sizeof(svrAddr));
    svrAddr.sin_family = AF_INET;
    svrAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    svrAddr.sin_port = htons(port);
    return svrAddr;
}
int ListenUDP(const char *ip, int port)
{
    sockaddr_in svrAddr = BuildAddr(ip, port);
    int sockFd = socket(AF_INET, SOCK_DGRAM, 0);
    if (sockFd < 0)
    {
        perror("create socket failed!");
        return sockFd;
    }
    if (bind(sockFd, (sockaddr *)&svrAddr, sizeof(svrAddr)) < 0)
    {
        perror("bind svr addr failed");
        return -1;
    }
    return sockFd;
}

void SendPacket(int n, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char *data = new char[size];

    for (int i = 0; i < n; i++)
    {
        int ret = sendto(svrFd, data, size, 0, (sockaddr *)&targetAddr, sizeof(targetAddr));
        if (ret < 0)
        {
            perror("sendto failed");
        }
        //printf("send packet:%d\n", ret);
    }

    delete[] data;
}
void SendMsgPacket(int n, int m, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char data[] = "{\"cmd\":\"echo\"}";
    int iov_len = strlen(data);
    struct iovec msg2[2];
    msg2[0].iov_base = data;
    msg2[0].iov_len = iov_len;
    msg2[1] = msg2[0];
    mmsghdr *msg = new mmsghdr[m];
    for (int i = 0; i < m; i++)
    {
        msg[i].msg_len = 1;
        msg[i].msg_hdr.msg_name = &targetAddr;
        msg[i].msg_hdr.msg_namelen = sizeof(targetAddr);
        msg[i].msg_hdr.msg_iov = msg2;
        msg[i].msg_hdr.msg_iovlen = 1;
    }
    for (int i = 0; i < n; i++)
    {
        int t = m;
        while (t > 0)
        {
            int ret = sendmmsg(svrFd, msg, m, 0);
            if (ret < 0)
            {
                perror("sendto failed");
            }
            //printf("send %d\n", t);
            t -= ret;
        }
    }
    delete[] msg;
}
void TestSendThread(int n, int svrFd)
{
    SendPacket(n, 32, svrFd, "127.0.0.1", 16666);
}
void TestSendMsgThread(int n, int m, int svrFd)
{
    SendMsgPacket(n, m, 32, svrFd, "127.0.0.1", 16666);
}
int main(int argc, char *argv[])
{
    int svrFd = ListenUDP("127.0.0.1", 65432);
    if (svrFd < 0)
    {
        perror("listen upd failed");
    }
    else
    {
        printf("UDP Server at 127.0.0.1:65432\n");
    }
    int value = 200000000;
    ::setsockopt(svrFd, SOL_SOCKET, SO_RCVBUF, (char *)&value, sizeof(value));
    ::setsockopt(svrFd, SOL_SOCKET, SO_SNDBUF, (char *)&value, sizeof(value));
    //ProfilerStart("udpsend.prof");
    auto begin = chrono::steady_clock::now();
    int n = 1000000;
    int m = 1;
    //TestSendThread(n, svrFd);
    int t = 8;
    thread *threads[20];
    for (int i = 0; i < t; i++)
    {
        //threads[i] = new thread(TestSendMsgThread, n, m, svrFd);
        threads[i] = new thread(TestSendThread, n, svrFd);
    }
    for (int i = 0; i < t; i++)
    {
        threads[i]->join();
        delete threads[i];
    }

    auto end = chrono::steady_clock::now();
    //ProfilerStop();
    double dr_ms = std::chrono::duration(end - begin).count();
    printf("Use time:%.2fms, PPS:%.0f\n", dr_ms, n * m * t / (dr_ms / 1000));
    close(svrFd);
}

g++ -O3 -o send_speed send_speed.cpp -lpthread
./send_speed 
UDP Server at 127.0.0.1:65432
Use time:6903.19ms, PPS:1158884

 ./send_speed 
UDP Server at 127.0.0.1:65432
Use time:6737.55ms, PPS:1187375

 ./send_speed 
UDP Server at 127.0.0.1:65432
Use time:6742.78ms, PPS:1186455

 

C++多线程 sendmmsg

也就130W PPS,但是可以看出sendmmsg系统调用不是瓶颈。

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
//#include 

#include 
#include 
#include 
using namespace std;

#define BUF_SIZE 64
char data[BUF_SIZE];
sockaddr_in BuildAddr(const char *ip, int port)
{
    sockaddr_in svrAddr;
    memset(&svrAddr, 0, sizeof(svrAddr));
    svrAddr.sin_family = AF_INET;
    svrAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    svrAddr.sin_port = htons(port);
    return svrAddr;
}
int ListenUDP(const char *ip, int port)
{
    sockaddr_in svrAddr = BuildAddr(ip, port);
    int sockFd = socket(AF_INET, SOCK_DGRAM, 0);
    if (sockFd < 0)
    {
        perror("create socket failed!");
        return sockFd;
    }
    if (bind(sockFd, (sockaddr *)&svrAddr, sizeof(svrAddr)) < 0)
    {
        perror("bind svr addr failed");
        return -1;
    }
    return sockFd;
}

void SendPacket(int n, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char *data = new char[size];

    for (int i = 0; i < n; i++)
    {
        int ret = sendto(svrFd, data, size, 0, (sockaddr *)&targetAddr, sizeof(targetAddr));
        if (ret < 0)
        {
            perror("sendto failed");
        }
        //printf("send packet:%d\n", ret);
    }

    delete[] data;
}
void SendMsgPacket(int n, int m, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char data[] = "{\"cmd\":\"echo\"}";
    int iov_len = strlen(data);
    struct iovec msg2[2];
    msg2[0].iov_base = data;
    msg2[0].iov_len = iov_len;
    msg2[1] = msg2[0];
    mmsghdr *msg = new mmsghdr[m];
    for (int i = 0; i < m; i++)
    {
        msg[i].msg_len = 1;
        msg[i].msg_hdr.msg_name = &targetAddr;
        msg[i].msg_hdr.msg_namelen = sizeof(targetAddr);
        msg[i].msg_hdr.msg_iov = msg2;
        msg[i].msg_hdr.msg_iovlen = 1;
    }
    for (int i = 0; i < n; i++)
    {
        int t = m;
        while (t > 0)
        {
            int ret = sendmmsg(svrFd, msg, m, 0);
            if (ret < 0)
            {
                perror("sendto failed");
            }
            //printf("send %d\n", t);
            t -= ret;
        }
    }
    delete[] msg;
}
void TestSendThread(int n, int svrFd)
{
    SendPacket(n, 32, svrFd, "127.0.0.1", 16666);
}
void TestSendMsgThread(int n, int m, int svrFd)
{
    SendMsgPacket(n, m, 32, svrFd, "127.0.0.1", 16666);
}
int main(int argc, char *argv[])
{
    int svrFd = ListenUDP("127.0.0.1", 65432);
    if (svrFd < 0)
    {
        perror("listen upd failed");
    }
    else
    {
        printf("UDP Server at 127.0.0.1:65432\n");
    }
    int value = 200000000;
    ::setsockopt(svrFd, SOL_SOCKET, SO_RCVBUF, (char *)&value, sizeof(value));
    ::setsockopt(svrFd, SOL_SOCKET, SO_SNDBUF, (char *)&value, sizeof(value));
    //ProfilerStart("udpsend.prof");
    auto begin = chrono::steady_clock::now();
    int n = 1000;
    int m = 1000;
    //TestSendThread(n, svrFd);
    int t = 8;
    thread *threads[20];
    for (int i = 0; i < t; i++)
    {
        threads[i] = new thread(TestSendMsgThread, n, m, svrFd);
        //threads[i] = new thread(TestSendThread, n, svrFd);
    }
    for (int i = 0; i < t; i++)
    {
        threads[i]->join();
        delete threads[i];
    }

    auto end = chrono::steady_clock::now();
    //ProfilerStop();
    double dr_ms = std::chrono::duration(end - begin).count();
    printf("Use time:%.2fms, PPS:%.0f\n", dr_ms, n * m * t / (dr_ms / 1000));
    close(svrFd);
}

//g++ -O3 -o send_speed send_speed.cpp -lpthread
//g++ -O3 -o send_speed send_speed.cpp -lpthread

//g++  -g -pg -o send_speed send_speed.cpp -lpthread

/*
g++ -O3 -o send_speed send_speed.cpp 
valgrind --tool=callgrind ./send_speed

}
//g++ -O3 -o send_speed send_speed.cpp -lpthread

//g++  -g -pg -o send_speed send_speed.cpp -lpthread

/*
g++ -O3 -o send_speed send_speed.cpp 
valgrind --tool=callgrind ./send_speed
python ../../gprof2dot/gprof2dot.py -f callgrind -n 10 -s callgrind.out.2938509 > val.dot
dot -Tpng val.dot -o val.png
python -m http.server 7897
*/

 ./send_speed 
UDP Server at 127.0.0.1:65432
Use time:6025.68ms, PPS:1327650

 ./send_speed 
UDP Server at 127.0.0.1:65432
Use time:5970.89ms, PPS:1339833

 ./send_speed 
UDP Server at 127.0.0.1:65432
Use time:6180.63ms, PPS:1294366

 

C++ 多socket同时发送

由于本机不支持SO_REUSEPORT,所以使用多个socket模拟。 PPS可达 170W。

操作系统版本：uname -a

Linux n227-020-135 4.14.81.bm.15-amd64 #1 SMP Debian 4.14.81.bm.15 Sun Sep 8 05:02:31 UTC 2019 x86_64 GNU/Linux

发现TCP可以reuse port, UDP就不行。

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
//#include 

#include 
#include 
#include 
using namespace std;

#define BUF_SIZE 64
char data[BUF_SIZE];
sockaddr_in BuildAddr(const char *ip, int port)
{
    sockaddr_in svrAddr;
    memset(&svrAddr, 0, sizeof(svrAddr));
    svrAddr.sin_family = AF_INET;
    svrAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    svrAddr.sin_port = htons(port);
    return svrAddr;
}
int ListenUDP(const char *ip, int port)
{
    sockaddr_in svrAddr = BuildAddr(ip, port);
    int sockFd = socket(AF_INET, SOCK_DGRAM, 0);
    if (sockFd < 0)
    {
        perror("create socket failed!");
        return sockFd;
    }
    if (bind(sockFd, (sockaddr *)&svrAddr, sizeof(svrAddr)) < 0)
    {
        perror("bind svr addr failed");
        return -1;
    }
    int val = 1;
    if (setsockopt(sockFd, SOL_SOCKET, SO_REUSEPORT | SO_REUSEADDR, &val, sizeof(val)) < 0)
    {
        perror("setsockopt()");
    }
    return sockFd;
}

void SendPacket(int n, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char *data = new char[size];

    for (int i = 0; i < n; i++)
    {
        int ret = sendto(svrFd, data, size, 0, (sockaddr *)&targetAddr, sizeof(targetAddr));
        if (ret < 0)
        {
            perror("sendto failed");
        }
        //printf("send packet:%d\n", ret);
    }

    delete[] data;
}
void SendMsgPacket(int n, int m, int size, int svrFd, const char *targetIP, int targetPort)
{
    sockaddr_in targetAddr = BuildAddr(targetIP, targetPort);
    char data[] = "{\"cmd\":\"echo\"}";
    int iov_len = strlen(data);
    struct iovec msg2[2];
    msg2[0].iov_base = data;
    msg2[0].iov_len = iov_len;
    msg2[1] = msg2[0];
    mmsghdr *msg = new mmsghdr[m];
    for (int i = 0; i < m; i++)
    {
        msg[i].msg_len = 1;
        msg[i].msg_hdr.msg_name = &targetAddr;
        msg[i].msg_hdr.msg_namelen = sizeof(targetAddr);
        msg[i].msg_hdr.msg_iov = msg2;
        msg[i].msg_hdr.msg_iovlen = 1;
    }
    for (int i = 0; i < n; i++)
    {
        int t = m;
        while (t > 0)
        {
            int ret = sendmmsg(svrFd, msg, m, 0);
            if (ret < 0)
            {
                perror("sendto failed");
            }
            //printf("send %d\n", t);
            t -= ret;
        }
    }
    delete[] msg;
}
void TestSendThread(int n, int svrFd)
{
    SendPacket(n, 32, svrFd, "127.0.0.1", 16666);
}

int fds[128];
void TestSendMsgThread(int n, int m, int svrFd)
{
    if (svrFd <= 0)
    {
        int i = -svrFd;
        svrFd = ListenUDP("127.0.0.1", 65432 + svrFd);
        fds[i] = svrFd;
        printf("%d %d\n", i, svrFd);
        if (svrFd < 0)
        {
            perror("listen upd failed");
            exit(1);
        }
        else
        {
            printf("UDP Server at 127.0.0.1:%d\n", 65432 + svrFd);
        }
    }
    SendMsgPacket(n, m, 32, svrFd, "127.0.0.1", 16666);
}
int main(int argc, char *argv[])
{
    //int fds[20];

    //int value = 4000;
    //::setsockopt(svrFd, SOL_SOCKET, SO_RCVBUF, (char *)&value, sizeof(value));
    //::setsockopt(svrFd, SOL_SOCKET, SO_SNDBUF, (char *)&value, sizeof(value));
    //ProfilerStart("udpsend.prof");
    auto begin = chrono::steady_clock::now();
    int n = 1000;
    int m = 1000;
    //TestSendThread(n, svrFd);
    int t = 10;

    // int svrFd = ListenUDP("127.0.0.1", 65431);
    // //fork();
    // if (svrFd < 0)
    // {
    //     perror("listen upd failed");
    //     exit(1);
    // }
    // else
    // {
    //     printf("UDP Server at 127.0.0.1:65432\n");
    // }

    thread *threads[128];
    for (int i = 0; i < t; i++)
    {
        threads[i] = new thread(TestSendMsgThread, n, m, i >= 8 ? fds[i % 8] : -i);
        //threads[i] = new thread(TestSendThread, n, svrFd);
    }

    for (int i = 0; i < t; i++)
    {
        threads[i]->join();
        delete threads[i];
    }

    auto end = chrono::steady_clock::now();
    //ProfilerStop();
    double dr_ms = std::chrono::duration(end - begin).count();
    printf("Use time:%.2fms, PPS:%.0f\n", dr_ms, n * m * t / (dr_ms / 1000));
    // close(svrFd);
}

//g++ -O3 -o send_speed send_speed.cpp -lpthread
//g++ -O3 -o send_speed send_speed.cpp -lpthread

//g++  -g -pg -o send_speed send_speed.cpp -lpthread

/*
g++ -O3 -o send_speed send_speed.cpp 
valgrind --tool=callgrind ./send_speed

}
//g++ -O3 -o send_speed send_speed.cpp -lpthread

//g++  -g -pg -o send_speed send_speed.cpp -lpthread

/*
g++ -O3 -o send_speed send_speed.cpp 
valgrind --tool=callgrind ./send_speed
python ../../gprof2dot/gprof2dot.py -f callgrind -n 10 -s callgrind.out.2938509 > val.dot
dot -Tpng val.dot -o val.png
python -m http.server 7897
*/

 

使用recvfrom和recvmmsg,结果没有区别

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
using namespace std;

#define VLEN 100
#define BUFSIZE 200
#define TIMEOUT 10

int sockfd, retval, i;
struct sockaddr_in sa;
struct mmsghdr msgs[VLEN];
struct iovec iovecs[VLEN];
char bufs[VLEN][BUFSIZE + 1];
struct timespec timeout;

int Recv()
{
    memset(msgs, 0, sizeof(msgs));
    for (i = 0; i < VLEN; i++)
    {
        iovecs[i].iov_base = bufs[i];
        iovecs[i].iov_len = BUFSIZE;
        msgs[i].msg_hdr.msg_iov = &iovecs[i];
        msgs[i].msg_hdr.msg_iovlen = 1;
    }

    timeout.tv_sec = TIMEOUT;
    timeout.tv_nsec = 0;

    retval = recvmmsg(sockfd, msgs, VLEN, 0, &timeout);
    if (retval == -1)
    {
        perror("recvmmsg()");
        exit(EXIT_FAILURE);
    }
    return retval;
}
int RecvOne()
{
    socklen_t len = 0;
    char buf[64];
    int retval = recvfrom(sockfd, buf, 64, 0, NULL, &len);
    if (retval == -1)
    {
        perror("recvfrom()");
        exit(EXIT_FAILURE);
    }
    return 1;
}

void RecvThread(int i){
    int total = 0;
    while (true)
    {
        int retval = Recv();
        //int retval = RecvOne();
        total += retval;
        //if(total % 100000 == 0){
            printf("thread %d: %d messages received, total %d\n", i, retval, total);
        //}
    }
}
int main(void)
{

    sockfd = socket(AF_INET, SOCK_DGRAM, 0);
    if (sockfd == -1)
    {
        perror("socket()");
        exit(EXIT_FAILURE);
    }
    int value = 400000000;
    ::setsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, (char *)&value, sizeof(value));
    ::setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, (char *)&value, sizeof(value));
    sa.sin_family = AF_INET;
    sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    sa.sin_port = htons(16666);
    if (bind(sockfd, (struct sockaddr *)&sa, sizeof(sa)) == -1)
    {
        perror("bind()");
        exit(EXIT_FAILURE);
    }
    printf("UDP server listened on 127.0.0.1:16666\n");
    
    int t = 8;
    thread *ths[128];
    for(int i = 0; i < t; i++){
        ths[i] = new thread(RecvThread, i);
    }
    for(int i = 0; i < t; i++){
        ths[i]->join();
    }
    exit(EXIT_SUCCESS);
}

//g++ -O3 -o recv_speed recv_speed.cpp}