/* g++ sp_block.cpp -g -O0 -o sp_block */ #include <unistd.h> #include <stdio.h> #include <time.h> // socketpair #include <sys/types.h> #include <sys/socket.h> // fork #include <sys/types.h> #include <unistd.h> // atoi #include <stdlib.h> // errno #include <errno.h> // strerror #include <string.h> #define srs_trace(msg, ...) \ printf("[%d][%d]", time(NULL), getpid()); \ printf(msg, ##__VA_ARGS__); \ if(errno) \ printf(", errno=%d(%s)\n", errno, strerror(errno)); \ else \ printf("\n") int RecvFD(int fd, int* pfd){ *pfd = 0; msghdr msg; // msg_iov iovec iov[1]; iov[0].iov_base = pfd; iov[0].iov_len = sizeof(int); msg.msg_iov = iov; msg.msg_iovlen = 1; // msg_name msg.msg_name = NULL; msg.msg_namelen = 0; // msg_control union { // union to create a 8B aligned memory. struct cmsghdr cm; // 16B = 8+4+4 char space[CMSG_SPACE(sizeof(int))]; // 24B = 16+4+4 } cmsg; memset(&cmsg, 0, sizeof(cmsg)); msg.msg_control = (cmsghdr*)&cmsg; msg.msg_controllen = sizeof(cmsg); if((recvmsg(fd, &msg, 0)) == -1){ srs_trace("st_recvmsg recv fd error."); return -1; } *pfd = *(int *)CMSG_DATA(&cmsg.cm); if(*pfd <= 0){ srs_trace("if transfer fd, the fd must be positive. actual is %d", *pfd); return -1; } srs_trace("get a fd from msg, fd=%d", *pfd); return 0; } int SendFD(int fd, int clientfd){ // transfer the fd. msghdr msg; union { struct cmsghdr cm; char space[CMSG_SPACE(sizeof(int))]; } cmsg; memset(&cmsg, 0, sizeof(cmsg)); cmsg.cm.cmsg_level = SOL_SOCKET; cmsg.cm.cmsg_type = SCM_RIGHTS; // we are sending fd. cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int)); msg.msg_control = (cmsghdr*)&cmsg; msg.msg_controllen = sizeof(cmsg); *(int *)CMSG_DATA(&cmsg.cm) = clientfd; // init msg_iov iovec iov[1]; iov[0].iov_base = &clientfd; iov[0].iov_len = sizeof(int); msg.msg_iov = iov; msg.msg_iovlen = 1; // init msg_name msg.msg_name = NULL; msg.msg_namelen = 0; // send fd if(sendmsg(fd, &msg, 0) == -1){ srs_trace("st_sendmsg failed"); return -1; } srs_trace("st_sendmsg send fd=%d", clientfd); return 0; } int main(int argc, char** argv){ srs_trace("Error value trace:"); srs_trace(" EMSGSIZE=%d(%s)", EMSGSIZE, strerror(EMSGSIZE)); srs_trace(" EAGAIN=%d(%s)", EAGAIN, strerror(EAGAIN)); if(argc <= 7){ srs_trace("Usage: <%s> " "<send_count> <send_size> <send_sleep> " "<recv_count> <recv_size> <recv_sleep> " "<transfer_fd>", argv[0]); return -1; } int send_count = atoi(argv[1]); int send_size = atoi(argv[2]); int send_sleep = atoi(argv[3]); int recv_count = atoi(argv[4]); int recv_size = atoi(argv[5]); int recv_sleep = atoi(argv[6]); int transfer_fd = atoi(argv[7]); srs_trace("send_size=%dB send_count=%d send_sleep=%dms " "recv_size=%dB recv_count=%d recv_sleep=%dms transfer_fd=%d", send_size, send_count, send_sleep, recv_size, recv_count, recv_sleep, transfer_fd); char* packet_send = new char[send_size]; char* packet_recv = new char[recv_size]; int size = 0; int fds[2]; if((socketpair(PF_UNIX, SOCK_STREAM, 0, fds)) == -1){ srs_trace("create socketpair failed"); return -1; } srs_trace("create socketpair success"); int pid = fork(); if(pid == -1){ srs_trace("create [sender] failed"); return -1; } if(pid == 0){ int client = socket(PF_INET, SOCK_STREAM, 0); int fd = fds[1]; srs_trace("[sender] pid=%d, fd=%d, peer=%d, client=%d", getpid(), fd, fds[0], client); int nb_sent = 0; for(int i = 0; send_size > 0 && i < send_count; i++){ errno = 0; srs_trace("[sender] start to send msg"); if((size = send(fd, packet_send, send_size, 0)) != send_size){ srs_trace("[sender] send packet error. i=%d, size=%d", i, size); return -1; } nb_sent += size; srs_trace("[sender] send packet. size=%d", size); if(send_sleep > 0 && i < send_count - 1){ usleep(send_sleep * 1000); } } srs_trace("[sender] send send_count=%d send_size=%dB sent=%dB, sleep=%dms success", send_count, send_size, nb_sent, send_sleep); if(transfer_fd && SendFD(fd, client) == -1){ srs_trace("[sender] send fd error"); return -1; } srs_trace("[sender] send fd ok, fd=%d", client); } else{ int fd = fds[0]; srs_trace("[receiver] pid=%d, fd=%d, peer=%d", getpid(), fd, fds[1]); int nb_recv = 0; for(int i = 0; recv_size > 0 && i < recv_count; i++){ errno = 0; srs_trace("start to read msg"); if((size = recv(fd, packet_recv, recv_size, 0)) == -1){ srs_trace("[receiver] recv packet error. i=%d, size=%d", i, size); return -1; } nb_recv += size; srs_trace("[receiver] recv packet. size=%d", size); if(recv_sleep > 0 && i < recv_count - 1){ usleep(recv_sleep * 1000); } } srs_trace("[receiver] recv recv_count=%d recv_size=%dB recv=%dB, sleep=%dms success", recv_count, recv_size, nb_recv, recv_sleep); int client = 0; if(transfer_fd && RecvFD(fd, &client) == -1){ srs_trace("[receiver] recv fd error"); return -1; } srs_trace("[receiver] recv fd ok, fd=%d", client); } srs_trace("test completed, sleep a while then exit"); sleep(30); return 0; }
./sp_block 2 2 3000 1 4 0 0 [1414330113][13832][receiver] pid=13832, fd=3, peer=4 [1414330113][13833][sender] pid=13833, fd=4, peer=3, client=5 [1414330113][13832][receiver] recv recv_count=1 recv_size=4B recv=2B, sleep=0ms success [1414330116][13833][sender] send send_count=2 send_size=2B sent=4B, sleep=3000ms success
./sp_block 2 2 0 1 4 0 0 [1414330176][13849][receiver] pid=13849, fd=3, peer=4 [1414330176][13850][sender] pid=13850, fd=4, peer=3, client=5 [1414330176][13850][sender] send send_count=2 send_size=2B sent=4B, sleep=0ms success [1414330176][13849][receiver] recv recv_count=1 recv_size=4B recv=4B, sleep=0ms success
./sp_block 2 2 3000 1 4 0 1 [1414330769][13986][receiver] pid=13986, fd=3, peer=4 [1414330769][13987][sender] pid=13987, fd=4, peer=3, client=5 [1414330769][13986][receiver] recv recv_count=1 recv_size=4B recv=2B, sleep=0ms success [1414330772][13987][sender] send send_count=2 send_size=2B sent=4B, sleep=3000ms success [1414330772][13986][receiver] recv fd error [1414330772][13987][sender] send fd ok, fd=5
./sp_block 2 2 0 1 4 0 1 [1414330912][14018][receiver] pid=14018, fd=3, peer=4 [1414330912][14019][sender] pid=14019, fd=4, peer=3, client=5 [1414330912][14019][sender] send send_count=2 send_size=2B sent=4B, sleep=0ms success [1414330912][14018][receiver] recv recv_count=1 recv_size=4B recv=4B, sleep=0ms success [1414330912][14019][sender] send fd ok, fd=5 [1414330912][14018][receiver] recv fd ok, fd=5
/* g++ sp_noblock.cpp -g -O0 -o sp_noblock */ #include <unistd.h> #include <stdio.h> #include <time.h> // socketpair #include <sys/types.h> #include <sys/socket.h> // fork #include <sys/types.h> #include <unistd.h> // atoi #include <stdlib.h> // errno #include <errno.h> // strerror #include <string.h> #define srs_trace(msg, ...) \ printf("[%d][%d]", time(NULL), getpid()); \ printf(msg, ##__VA_ARGS__); \ if(errno) \ printf(", errno=%d(%s)\n", errno, strerror(errno)); \ else \ printf("\n") int RecvFD(int fd, int* pfd){ *pfd = 0; msghdr msg; // msg_iov iovec iov[1]; iov[0].iov_base = pfd; iov[0].iov_len = sizeof(int); msg.msg_iov = iov; msg.msg_iovlen = 1; // msg_name msg.msg_name = NULL; msg.msg_namelen = 0; // msg_control union { // union to create a 8B aligned memory. struct cmsghdr cm; // 16B = 8+4+4 char space[CMSG_SPACE(sizeof(int))]; // 24B = 16+4+4 } cmsg; memset(&cmsg, 0, sizeof(cmsg)); msg.msg_control = (cmsghdr*)&cmsg; msg.msg_controllen = sizeof(cmsg); if((recvmsg(fd, &msg, MSG_DONTWAIT)) == -1){ srs_trace("st_recvmsg recv fd error."); return -1; } *pfd = *(int *)CMSG_DATA(&cmsg.cm); if(*pfd <= 0){ srs_trace("if transfer fd, the fd must be positive. actual is %d", *pfd); return -1; } srs_trace("get a fd from msg, fd=%d", *pfd); return 0; } int SendFD(int fd, int clientfd){ // transfer the fd. msghdr msg; union { struct cmsghdr cm; char space[CMSG_SPACE(sizeof(int))]; } cmsg; memset(&cmsg, 0, sizeof(cmsg)); cmsg.cm.cmsg_level = SOL_SOCKET; cmsg.cm.cmsg_type = SCM_RIGHTS; // we are sending fd. cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int)); msg.msg_control = (cmsghdr*)&cmsg; msg.msg_controllen = sizeof(cmsg); *(int *)CMSG_DATA(&cmsg.cm) = clientfd; // init msg_iov iovec iov[1]; iov[0].iov_base = &clientfd; iov[0].iov_len = sizeof(int); msg.msg_iov = iov; msg.msg_iovlen = 1; // init msg_name msg.msg_name = NULL; msg.msg_namelen = 0; // send fd if(sendmsg(fd, &msg, MSG_DONTWAIT) == -1){ srs_trace("st_sendmsg failed"); return -1; } srs_trace("st_sendmsg send fd=%d", clientfd); return 0; } int main(int argc, char** argv){ srs_trace("Error value trace:"); srs_trace(" EMSGSIZE=%d(%s)", EMSGSIZE, strerror(EMSGSIZE)); srs_trace(" EAGAIN=%d(%s)", EAGAIN, strerror(EAGAIN)); if(argc <= 7){ srs_trace("Usage: <%s> " "<send_count> <send_size> <send_sleep> " "<recv_count> <recv_size> <recv_sleep> " "<transfer_fd>", argv[0]); return -1; } int send_count = atoi(argv[1]); int send_size = atoi(argv[2]); int send_sleep = atoi(argv[3]); int recv_count = atoi(argv[4]); int recv_size = atoi(argv[5]); int recv_sleep = atoi(argv[6]); int transfer_fd = atoi(argv[7]); srs_trace("send_size=%dB send_count=%d send_sleep=%dms " "recv_size=%dB recv_count=%d recv_sleep=%dms transfer_fd=%d", send_size, send_count, send_sleep, recv_size, recv_count, recv_sleep, transfer_fd); char* packet_send = new char[send_size]; char* packet_recv = new char[recv_size]; int size = 0; int fds[2]; if((socketpair(PF_UNIX, SOCK_STREAM, 0, fds)) == -1){ srs_trace("create socketpair failed"); return -1; } srs_trace("create socketpair success"); int pid = fork(); if(pid == -1){ srs_trace("create [sender] failed"); return -1; } if(pid == 0){ int client = socket(PF_INET, SOCK_STREAM, 0); int fd = fds[1]; srs_trace("[sender] pid=%d, fd=%d, peer=%d, client=%d", getpid(), fd, fds[0], client); int nb_sent = 0; for(int i = 0; send_size > 0 && i < send_count; i++){ errno = 0; srs_trace("[sender] start to send msg"); if(send_sleep > 0){ usleep(send_sleep * 1000); } if((size = send(fd, packet_send, send_size, MSG_DONTWAIT)) != send_size){ srs_trace("[sender] send packet error. i=%d, size=%d", i, size); return -1; } nb_sent += size; srs_trace("[sender] send packet. size=%d", size); } srs_trace("[sender] send send_count=%d send_size=%dB sent=%dB, sleep=%dms success", send_count, send_size, nb_sent, send_sleep); if(transfer_fd && SendFD(fd, client) == -1){ srs_trace("[sender] send fd error"); return -1; } srs_trace("[sender] send fd ok, fd=%d", client); } else{ int fd = fds[0]; srs_trace("[receiver] pid=%d, fd=%d, peer=%d", getpid(), fd, fds[1]); int nb_recv = 0; for(int i = 0; recv_size > 0 && i < recv_count; i++){ errno = 0; srs_trace("start to read msg"); if(recv_sleep > 0){ usleep(recv_sleep * 1000); } if((size = recv(fd, packet_recv, recv_size, MSG_DONTWAIT)) == -1){ srs_trace("[receiver] recv packet error. i=%d, size=%d", i, size); return -1; } nb_recv += size; srs_trace("[receiver] recv packet. size=%d", size); } srs_trace("[receiver] recv recv_count=%d recv_size=%dB recv=%dB, sleep=%dms success", recv_count, recv_size, nb_recv, recv_sleep); int client = 0; if(transfer_fd && RecvFD(fd, &client) == -1){ srs_trace("[receiver] recv fd error"); return -1; } srs_trace("[receiver] recv fd ok, fd=%d", client); } srs_trace("test completed, sleep a while then exit"); sleep(30); return 0; }
附上st的处理方式:
/* g++ sp_st_fd.cpp -g -O0 -Ist-1.9/obj st-1.9/obj/libst.a -o sp_st_fd */ #include <unistd.h> #include <stdio.h> // socketpair #include <sys/types.h> #include <sys/socket.h> // fork #include <sys/types.h> #include <unistd.h> // atoi #include <stdlib.h> // errno #include <errno.h> // strerror #include <string.h> #include <st.h> int RecvFD(st_netfd_t stfd, int* pfd){ *pfd = 0; msghdr msg; // msg_iov iovec iov[1]; iov[0].iov_base = pfd; iov[0].iov_len = sizeof(int); msg.msg_iov = iov; msg.msg_iovlen = 1; // msg_name msg.msg_name = NULL; msg.msg_namelen = 0; // msg_control union { // union to create a 8B aligned memory. struct cmsghdr cm; // 16B = 8+4+4 char space[CMSG_SPACE(sizeof(int))]; // 24B = 16+4+4 } cmsg; memset(&cmsg, 0, sizeof(cmsg)); msg.msg_control = (cmsghdr*)&cmsg; msg.msg_controllen = sizeof(cmsg); if((st_recvmsg(stfd, &msg, 0, ST_UTIME_NO_TIMEOUT)) == -1){ printf("st_recvmsg recv fd error. errno=%d(%#x)\n", errno, errno); return -1; } *pfd = *(int *)CMSG_DATA(&cmsg.cm); if(*pfd <= 0){ printf("if transfer fd, the fd must be positive. actual is %d\n", *pfd); return -1; } printf("get a fd from msg, fd=%d\n", *pfd); return 0; } int SendFD(st_netfd_t stfd, int clientfd){ // transfer the fd. msghdr msg; union { struct cmsghdr cm; char space[CMSG_SPACE(sizeof(int))]; } cmsg; memset(&cmsg, 0, sizeof(cmsg)); cmsg.cm.cmsg_level = SOL_SOCKET; cmsg.cm.cmsg_type = SCM_RIGHTS; // we are sending fd. cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int)); msg.msg_control = (cmsghdr*)&cmsg; msg.msg_controllen = sizeof(cmsg); *(int *)CMSG_DATA(&cmsg.cm) = clientfd; // init msg_iov iovec iov[1]; iov[0].iov_base = &clientfd; iov[0].iov_len = sizeof(int); msg.msg_iov = iov; msg.msg_iovlen = 1; // init msg_name msg.msg_name = NULL; msg.msg_namelen = 0; // send fd if(st_sendmsg(stfd, &msg, 0, ST_UTIME_NO_TIMEOUT) == -1){ printf("st_sendmsg failed, errno=%d(%#x)\n", errno, errno); return -1; } printf("st_sendmsg send fd=%d\n", clientfd); return 0; } void* thread(void* arg){ for(;;){ printf("in st thread.\n"); st_sleep(1); } return NULL; } int main(int argc, char** argv){ #if 1 printf("errno=EMSGSIZE(%d), %s\n", EMSGSIZE, strerror(EMSGSIZE)); if(argc <= 6){ printf("Usage: <%s> <send_count> <recv_count> <packet_send_size> <packet_recv_size> <parent_process_sleep> <sub_process_sleep>\n", argv[0]); return -1; } int send_count = atoi(argv[1]); int recv_count = atoi(argv[2]); int packet_send_size = atoi(argv[3]); int packet_recv_size = atoi(argv[4]); int parent_process_sleep = atoi(argv[5]); int sub_process_sleep = atoi(argv[6]); printf("packet_send_size=%dB packet_recv_size=%dB send_count=%d recv_count=%d parent_process_sleep=%dms sub_process_sleep=%dms\n", packet_send_size, packet_recv_size, send_count, recv_count, parent_process_sleep, sub_process_sleep); char* packet_send = new char[packet_send_size]; char* packet_recv = new char[packet_recv_size]; int size = 0; #endif int fds[2]; if((socketpair(PF_UNIX, SOCK_STREAM, 0, fds)) == -1){ printf("create socketpair failed\n"); return -1; } printf("create socketpair success\n"); int pid = fork(); if(pid == -1){ printf("create [sub] failed\n"); return -1; } // init st. if(st_init() == -1){ printf("st_init error\n"); return -1; } if(st_thread_create(thread, NULL, 0, 0) == NULL){ printf("create st thread error.\n"); return -1; } if(pid == 0){ int client = socket(PF_INET, SOCK_STREAM, 0); int fd = fds[1]; printf("[sub] pid=%d, fd=%d, peer=%d, client=%d\n", getpid(), fd, fds[0], client); st_netfd_t stfd = st_netfd_open(fd); if(stfd == NULL){ printf("[sub] open stfd failed.\n"); return -1; } for(int i = 0; i < send_count; i++){ errno = 0; if(sub_process_sleep > 0){ usleep(sub_process_sleep * 1000); } printf("[sub] start to send msg and fd.\n"); if(packet_send_size > 0){ if((size = st_write(stfd, packet_send, packet_send_size, ST_UTIME_NO_TIMEOUT)) != packet_send_size){ printf("[sub] send packet error. i=%d, size=%d, errno=%d, msg=%s\n", i, size, errno, strerror(errno)); return -1; } printf("[sub] send packet. size=%d, errno=%d(%s)\n", size, errno, strerror(errno)); } } if(SendFD(stfd, client) == -1){ printf("[sub] send fd error. errno=%d, msg=%s\n", errno, strerror(errno)); return -1; } printf("[sub] send send_count=%d packet_send_size=%dB, sleep=%dms success\n", send_count, packet_send_size, sub_process_sleep); } else{ int fd = fds[0]; printf("[parent] pid=%d, fd=%d, peer=%d\n", getpid(), fd, fds[1]); st_netfd_t stfd = st_netfd_open(fd); if(stfd == NULL){ printf("[parent] open stfd failed.\n"); return -1; } int client = 0; for(int i = 0; i < recv_count; i++){ errno = 0; if(parent_process_sleep > 0){ usleep(parent_process_sleep * 1000); } if(client > 0){ close(client); } printf("start to read msg and fd.\n"); if(packet_recv_size > 0){ if((size = st_read(stfd, packet_recv, packet_recv_size, ST_UTIME_NO_TIMEOUT)) == -1){ printf("[parent] recv packet error. i=%d, size=%d, errno=%d, msg=%s\n", i, size, errno, strerror(errno)); return -1; } printf("[parent] recv packet. size=%d, errno=%d(%s)\n", size, errno, strerror(errno)); } } if(RecvFD(stfd, &client) == -1){ printf("[sub] send fd error. errno=%d, msg=%s", errno, strerror(errno)); return -1; } printf("[parent] recv recv_count=%d packet_recv_size=%dB, sleep=%dms success\n", recv_count, packet_recv_size, parent_process_sleep); } sleep(30); return 0; }
发送2次,每次发送10字节。
收取时,每次收取20字节,就会有EAGAIN消息。
第1次收取,收取10字节。
第2次收取,收取10字节。
[parent] recv packet. size=10, errno=11(Resource temporarily unavailable)
[parent] recv packet. size=10, errno=11(Resource temporarily unavailable)
[parent] recv recv_count=2 packet_recv_size=20B, sleep=0ms success
然后正常的收到fd。
结论是:
1. 发送端和接收端,同时操作的数据应该是一样的。譬如,发送端一次发送100个字节的头,接收端先接受100个字节的头。发送端再发送x字节的数据,接收端接受x字节的数据。
2.发送端对于每次来回不能发送多次,否则接收端就会出现EAGAIN或者其他的情况。譬如头是100字节,发送端就必须一次发送100字节,而不能做两次发送,一次发送20一次发送80.
3.可见每个消息不超过16000,超过后需要分多次发送了。
4.只要单次一次发送,不管是否是非阻塞,都能收到。譬如发送fd,一次发送一个msghdr,只要发送端一次发送,接收端一次接收,就没有问题。不会出现收到一半的情况。
5.只有当发送端因为某种原因发送了一部分,接收端一次接收的超过了发送的部分,就会出现非阻塞的EAGAIN。
原因就是因为sendmsg和recvmsg是原子操作。
譬如有以下协议:
1. 发送4字节类型
2. 发送4字节长度
3. 发送长度指定的数据
4. 发送一个cmsg消息
读取时不能超过发送的组合单元,譬如假设是分四次发送的,那么就必须分四次接收,因为不能保证1和2是一起收到的。
假设1和2是一起发送,3单独发送,4单独发送;那么接收时可以一起接收1和2,但是不能一起接收1和2和3。也可以分开接收1,然后收2.
打个比方,这个就像包裹一样的,如果快递时1和2是一起打包的,到门口接收时肯定是1和2一起到的,进门接收时可以收1,然后收2,或者一起收。