C++ SOCKET通信模型（一）select

以前做游戏服务器的时候我就听说过IOCP和EPOLL，一直没来得及去填这个坑，从今天开始有多余的时间打算把这个重要的坑填上。说IOCP和EPOLL前，先说说阻塞+多线程模式，我以前都是写竞技类的游戏服务器，所以TCP基本上都是长连接，其实感觉也还可以，只要同时在线不是太多不会有什么问题。但如果是短连接高并发的话，那问题可就严重了，如果没有线程池的话，光创建销毁线程的开销可不小，而且相当的费内存，响应速度也比较慢。非阻塞式IO 怎么说呢，就像是一块还没有打磨的石头，原本他也属于同步IO，至于异步IO概念，我个人觉得可以算是和协程几乎相同的一个概念吧，异步并不代表是多线程，windows众所周知的Overlapped IO模型，linux上的aio，都是异步IO模型，只不过我没怎么用过AIO。我这不提单独非阻塞IO的用法，直接从多路复用IO select开始。关于select我的用法是，使其成为多线程的同步IO。举个例来说，现在有N个玩家在服务器上，在accept的时候我保存了socket并放在list中，开N个线程用户接收，开N个线程用于发送，并以一定的规则将list分段分配给不同的线程处理，不断的去检查是否有新的消息，我觉得这应该算一个比较中性的策略吧，第一在于对list的维护不容易，第二线程安全问题，第三肯定就出现了负载均衡的问题，第四select有一定限制，由于没有事件通知，只能不断的循环检测，不断的重置SET，所以一定就会有所限制，否则一定会影响性能，在服务器编程上这种耗费资源的玩意也不可取。还有一点就是recv send都是从内核缓冲区去处理数据，就是说来的数据并不是直接到用户BUF中，还有一次额外的内存拷贝，性能损失也不可忽略，所以在这之上还有Overlapped IO事件通知模型，Overlapped IO 完成例程模型，也就是IOCP

不过把非阻塞IO作为多路复用IO，在同时在线不过万的情况下，经过我这种多线程分配，感觉也还行，那就先来看看这个非阻塞IO吧。做这种多线程的东西，逻辑一定要严谨，严谨！！一不留神就是BUG

为了代码简洁，socket上那些函数的返回错误值我就不再捕获了，windows平台

server.cpp

#define FD_SETSIZE 2048

// Normal.cpp: 定义控制台应用程序的入口点。
//

#include "stdafx.h"
#include 
#include
#include
#include
#include
#include
#include
#include 
#include 
#pragma comment(lib,"ws2_32.lib")
unsigned int WINAPI CreateServ(LPVOID args);
unsigned int WINAPI Proc(LPVOID args);
using namespace std;
const int _thread_count = 8;

FD_SET _readSet;
FD_SET _writeSet;
list _sockList;
list _removeList;
mutex lock4cv;
mutex lock4cv2;
condition_variable cv;
condition_variable cv2;
int _thread_unfinish;
vector _vec;
char* buf2 = "hello client";

int main()
{
	for (int i = 0; i<_thread_count; i++)
	{
		_vec.push_back(0);
	}
	_beginthreadex(0, 0, CreateServ, 0, 0, 0);
	for(int i=0;i<_thread_count;i++)
	{
		int* temp = new int(i);
		_beginthreadex(0, 0, Proc, temp, 0, 0);
	}
	cin.get();
	cin.get();
    return 0;
}
bool _isFinish()
{
	return _thread_unfinish <=0;
}


unsigned int WINAPI Proc(LPVOID args)
{
	int* I = (int*)args;

	while(true)
	{
		{
			unique_lock l(lock4cv);
			if (_vec[*I] <= 0)
			{
				cv.wait(l);
			}
			_vec[*I] = 0;
		}
		int start =  ceil(_sockList.size() / (double)_thread_count)* *I;
		if(start<*I)
		{
			start = *I;
		}
		if(_sockList.size()<=start)
		{
			_thread_unfinish-=1;
			lock4cv2.lock();
			cv2.notify_all();
			lock4cv2.unlock();
			continue;
		}
		int end =  start+ ceil(_sockList.size() / (double)_thread_count);

		int i = 0;
		for (auto iter = _sockList.begin(); iter != _sockList.end(); ++iter)
		{
			if (i >= start&&i < end) {
				SOCKET& s = *iter;
				if (FD_ISSET(s, &_readSet))
				{
					cout << "proc by:" << *I << endl;
					char buf[128];
					int len = recv(s, buf, 128, 0);
					if (len <= 0)
					{
						closesocket(s);
						_removeList.push_back(s);
						continue;
					}
					FD_SET(s, &_writeSet);
					//cout << buf << endl;
				}
				if (FD_ISSET(s, &_writeSet))
				{
					send(s, buf2, 128, 0);
				}
			}
			i++;
		}
		lock4cv2.lock();
		_thread_unfinish -= 1;
		cv2.notify_all();
		lock4cv2.unlock();
	}
}
unsigned int WINAPI CreateServ(LPVOID args) {
	WORD wVersion;
	WSADATA wsaData;
	int err;
	wVersion = MAKEWORD(2, 1);
	err = WSAStartup(wVersion, &wsaData);
	if (err != 0) {
		return 0;
	}
	if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 1) {
		WSACleanup();
		return 0;
	}
	SOCKET sockSrv = socket(AF_INET, SOCK_STREAM, 0);
	const char chOpt = 1;
	setsockopt(sockSrv, IPPROTO_TCP, TCP_NODELAY, &chOpt, sizeof(chOpt));

	int nSendBufLen = 16 * 1024 * 1024; 
	setsockopt(sockSrv, SOL_SOCKET, SO_SNDBUF, (const char*)&nSendBufLen, sizeof(int));

	ULONG NonBlock = 1;
	ioctlsocket(sockSrv, FIONBIO, &NonBlock);

	//	struct in_addr s;
	//	inet_pton(AF_INET, "127.0.0.1",(void*)&s);
	SOCKADDR_IN addrSrv;
	addrSrv.sin_addr.S_un.S_addr = htonl(ADDR_ANY);
	addrSrv.sin_family = AF_INET;
	addrSrv.sin_port = htons(6001);

	::bind(sockSrv, (SOCKADDR*)&addrSrv, sizeof(SOCKADDR));

	err = listen(sockSrv, 100);
	if (err == SOCKET_ERROR) {
		cout << "listen failed" << endl;
		WSACleanup();
		return 0;
	}

	
	//accept loop
	while (true) {
		
		FD_ZERO(&_readSet);
		FD_ZERO(&_writeSet);

		FD_SET(sockSrv, &_readSet);
		for (auto iter = _sockList.begin(); iter != _sockList.end(); ++iter)
		{
			SOCKET& s = *iter;
			FD_SET(s, &_readSet);
		}

		select(0, &_readSet, &_writeSet, 0, 0);

		if(FD_ISSET(sockSrv,&_readSet))
		{
			SOCKET s = accept(sockSrv, 0, 0);
			ioctlsocket(s, FIONBIO, &NonBlock);
			_sockList.push_back(s);
		}
		else {
			_thread_unfinish = _thread_count;

			for (int i = 0; i < _thread_count; i++)
			{
				_vec[i] = 1;
			}
			cv.notify_all();

			unique_lock l(lock4cv2);
			cv2.wait(l, _isFinish);

			for (auto iter = _removeList.begin(); iter != _removeList.end(); ++iter)
			{
				_sockList.remove(*iter);
			}
			_removeList.clear();
		}
	}
}

client.cpp

// Client.cpp: 定义控制台应用程序的入口点。
//

#include "stdafx.h"
#include 
#include
#include
#include 
#include 
#pragma comment(lib,"ws2_32.lib")
unsigned int WINAPI CreateClient(LPVOID args);

using namespace std;

int _time;
int main()
{
	WORD wVersion;
	WSADATA wsaData;
	int err;
	wVersion = MAKEWORD(2, 1);
	err = WSAStartup(wVersion, &wsaData);
	if (err != 0) {
		return 0;
	}
	if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 1) {
		WSACleanup();
		return 0;
	}
	for (int i = 0; i < 1024 ; i++) {
		_beginthreadex(0, 0, CreateClient, 0, 0, 0);
	}
	cin.get();
	cin.get();
    return 0;
}
mutex m;
int c = 0;
unsigned int WINAPI CreateClient(LPVOID args)
{
	SOCKET sock = socket(AF_INET, SOCK_STREAM, 0);
	SOCKADDR_IN addr;
	IN_ADDR in;
	inet_pton(AF_INET, "127.0.0.1", &in);
	addr.sin_addr = in;
	addr.sin_family = AF_INET;
	addr.sin_port = htons(6001);

	int val=connect(sock, (SOCKADDR*)&addr, sizeof(SOCKADDR));
	if(val!=0)
	{
		cout << "conn failed" << endl;
	}

	char buf[128];
	int i = 0;
	sprintf_s(buf, "hello serv %d", i);

	char buf2[128];
	while(true)
	{
		send(sock, buf, 128, 0);
		
		recv(sock, buf2, 128, 0);
		//cout << buf2 << endl;
		Sleep(1000);
	}
	
	closesocket(sock);
}