《ASCE1885的源码分析》の基于完成端口模型的TCP服务器框架

使用IOCP的TCP服务器使用过程大体如下：

1） 使用CreateIoCompletionPort函数创建完成端口，并以该I/O完成端口为参数创建多个服务线程；

2） 创建监听套接字；

3） 接收客户端连接请求，返回服务套接字；

4） 将服务套接字与完成端口绑定，并在该套接字上投递初始I/O操作请求；

5） 返回步骤3）；

服务线程的流程如下：

1） 调用GetQueuedCompletionPort函数等待获取完成信息；

2） 根据需要对数据进行处理并投递后续的I/O操作请求；

3） 返回步骤1）。

程序代码及注释如下：

#include <stdio.h>

#include <winsock2.h>

#include <process.h>

#include <Windows.h>

#pragma comment(lib, "ws2_32.lib")

#define MAX_THREAD_NUM 24 //最大服务器线程数

#define MAX_BUF_LEN 5000 //最大服务器I/O缓冲区大小

//枚举类型，用于指示服务器I/O操作的类型

typedef enum _IO_OPER

{

SVR_IO_READ,

SVR_IO_WRITE

}IO_OPER, *LPIO_OPER;

//扩展重叠结构体，单I/O数据

typedef struct _OverLappedEx

{

OVERLAPPED OverLapped;

WSABUF wbuf; //I/O操作的数据对象

char data[MAX_BUF_LEN];//实际的数据缓冲区

IO_OPER oper; //用于标志I/O操作的类型

DWORD flags; //用于设定或者返回I/O操作的标志

} PER_IO_DATA, *LPPER_IO_DATA;

//完成键结构体，单句柄数据，对应每个服务套接字---每个连接

typedef struct _CONN_CTX

{

SOCKET sockAccept; //该连接的服务器侦听服务套接字

LPPER_IO_DATA pPerIOData; //指向该连接的I/O操作信息

struct _CONN_CTX *pPrev; //用于形成服务器当前所有连接信息的双向链表

struct _CONN_CTX *pNext; //分别指向链表中前一个节点和后一个节点

} CONN_CTX, *LPCONN_CTX;

CRITICAL_SECTION g_CriticalSection; //防止对连接信息链表的访问冲突

LPCONN_CTX g_ptrConnCtxHead = NULL; //指向双向链表头节点，用于对该链表的访问和维护

SOCKET g_sockListen = INVALID_SOCKET;

//该函数用于对控制台消息进行处理，当接收到CTRL_C_EVENT、

//CTRL_LOGOFF_EVENT、¡CTRL_SHUTDOWN_EVENT或者CTRL_CLOSE_EVENT

//事件时，服务器将关闭监听套接字，从而使主线程从接收连接的死循环中

//退出，并最终结束所有服务线程，释放连接等

BOOL WINAPI CtrlHandler(DWORD dwEvent);

//完成端口操作函数

HANDLE CreateNewIoCompletionPort(DWORD dwNumberOfConcurrentThreads);

BOOL AssociateWithIoCompletePort(HANDLE hComPort, HANDLE hDevice, DWORD dwCompKey);

//创建监听套接字

SOCKET CreateListenSock();

//当服务器接受了客户端连接请求后，将返回的服务套接字和完成端口

//作为参数调用该函数，该函数完成服务套接字与完成端口的绑定

//以及为该连接的相关信息分配存储区的工作

LPCONN_CTX CreateConnCtx(SOCKET sockAccept, HANDLE hIOCP);

//将新的连接信息加入到全局的连接信息链表中

void ConnListAdd(LPCONN_CTX lpConnCtx);

//将指定的连接信息从全局连接信息链表中删除，

//并关闭连接，释放相应的存储区资源

void ConnListRemove(LPCONN_CTX lpConnCtx);

//完成服务器退出时关闭连接、释放资源的工作

void ConnListClear();

//由于printf函数只能在用C运行库中函数创建的线程中使用

//因此，本程序重写自己的输出函数

int ASCEPrintf(const char* lpFormat, ...);

//工作线程函数

DWORD WINAPI WorkThread(LPVOID lpParam);

int main(int argc, char* argv[])

{

HANDLE hIOCP;

HANDLE hThreadHandles[MAX_THREAD_NUM];

int nThreadCount;

WSADATA wsaData;

if(WSAStartup(MAKEWORD(2,2), &wsaData) != 0)

{

ASCEPrintf("Winsock initialized failed.../n");

return -1;

}

int i;

for(i=0; i<MAX_THREAD_NUM; i++)

{

hThreadHandles[i] = NULL;

}

//设置控制台事件响应函数º

if(!SetConsoleCtrlHandler(CtrlHandler, TRUE))

{

ASCEPrintf("SetConsoleCtrlHandler:%d/n", GetLastError());

return -1;

}

InitializeCriticalSection(&g_CriticalSection);

__try

{

//创建I/O完成端口

hIOCP = CreateNewIoCompletionPort(0);

if(hIOCP == NULL)

{

ASCEPrintf("CreateIoCompletionPort:%d/n", GetLastError());

__leave;

}

//创建多个工作线程

SYSTEM_INFO sysInfo;

GetSystemInfo(&sysInfo);

//将sysInfo.dwNumberOfProcessors*2+2和¨ª

//MAX_THREAD_NUM之间的较小者赋给nThreadCount

nThreadCount = (sysInfo.dwNumberOfProcessors*2+2) < MAX_THREAD_NUM

? (sysInfo.dwNumberOfProcessors*2+2) : MAX_THREAD_NUM;

for(int i=0; i<nThreadCount; i++)

{

HANDLE hThread = CreateThread(NULL, 0, WorkThread, hIOCP, 0, NULL);

if(hThread == NULL)

{

ASCEPrintf("CreateThread:%d/n", GetLastError());

__leave;

} else {

hThreadHandles[i] = hThread;

}

}

g_sockListen = CreateListenSock();

if(g_sockListen == INVALID_SOCKET)

__leave;

SOCKET sockAccept;

LPCONN_CTX lpConnCtx;

int nResult;

while(true)

{

sockAccept = accept(g_sockListen, NULL, NULL);

if(sockAccept == INVALID_SOCKET)

__leave;

lpConnCtx = CreateConnCtx(sockAccept, hIOCP);

if(lpConnCtx == NULL)

_leave;

else

ConnListAdd(lpConnCtx);

//投递初始I/O操作

nResult = WSARecv(sockAccept,

&(lpConnCtx->pPerIOData->wbuf),

1, NULL,

&(lpConnCtx->pPerIOData->flags),

&(lpConnCtx->pPerIOData->OverLapped),

NULL);

if((nResult == SOCKET_ERROR) &&

(WSAGetLastError() != ERROR_IO_PENDING))

{

ASCEPrintf("WSARecv:%d/n", WSAGetLastError());

ConnListRemove(lpConnCtx);

break;

}

}

} __finally {

if(hIOCP)

{

for(int i=0; i<nThreadCount; i++)

{

PostQueuedCompletionStatus(hIOCP, 0, 0, NULL);

}

}

//等待所有工作线程结束

if(WAIT_OBJECT_0 != WaitForMultipleObjects(nThreadCount,

hThreadHandles, TRUE, 1000))

ASCEPrintf("WaitForMultipleObjects failed:%d/n", GetLastError());

else {

for(int i=0; i<nThreadCount; i++)

{

if(hThreadHandles[i] != NULL)

{

if(!CloseHandle(hThreadHandles[i]))

ASCEPrintf("CloseHandle:%d/n", GetLastError());

}

hThreadHandles[i] = NULL;

}

}

if(hIOCP)

{

CloseHandle(hIOCP);

hIOCP = NULL;

}

if(g_sockListen != INVALID_SOCKET)

{

closesocket(g_sockListen);

g_sockListen = INVALID_SOCKET;

}

if(g_ptrConnCtxHead)

ConnListClear();

ASCEPrintf("...............Stopped./n");

DeleteCriticalSection(&g_CriticalSection);

SetConsoleCtrlHandler(CtrlHandler, FALSE);

WSACleanup();

return 0;

}

}

BOOL WINAPI CtrlHandler(DWORD dwEvent)

{

SOCKET sockTemp = INVALID_SOCKET;

switch(dwEvent)

{

case CTRL_C_EVENT:

case CTRL_LOGOFF_EVENT:

case CTRL_SHUTDOWN_EVENT:

case CTRL_CLOSE_EVENT:

ASCEPrintf("Server Stopping........./n");

sockTemp = g_sockListen;

g_sockListen = INVALID_SOCKET;

if(sockTemp != INVALID_SOCKET)

{

closesocket(sockTemp);

sockTemp = INVALID_SOCKET;

}

break;

default:

return FALSE;

}

return TRUE;

}

//创建I/O完成端口

HANDLE CreateNewIoCompletionPort(DWORD dwNumberOfConcurrentThreads)

{

return (CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0,

dwNumberOfConcurrentThreads));

}

//将套接字与完成端口关联

BOOL AssociateWithIoCompletionPort(HANDLE hComPort, HANDLE hDevice,

DWORD dwCompKey)

{

return (CreateIoCompletionPort(hDevice, hComPort, dwCompKey, 0)

== hComPort);

}

//创建服务器监听套接字

SOCKET CreateListenSock()

{

//创建WSA_FLAG_OVERLAPPED属性的套接字

SOCKET sock = WSASocket(AF_INET, SOCK_STREAM, 0,

NULL, 0, WSA_FLAG_OVERLAPPED);

if(sock == INVALID_SOCKET)

return sock;

BOOL bReuseAddr = true;

if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)&bReuseAddr,

sizeof(bReuseAddr)) == SOCKET_ERROR)

{

ASCEPrintf("setsockopt:%d/n", WSAGetLastError());

closesocket(sock);

return INVALID_SOCKET;

}

struct sockaddr_in local;

memset(&local, 0, sizeof(local));

local.sin_addr.s_addr = INADDR_ANY;

local.sin_family = AF_INET;

local.sin_port = htons(9999);

if(bind(sock, (struct sockaddr*)&local, sizeof(local)) == SOCKET_ERROR)

{

ASCEPrintf("bind:%d/n", WSAGetLastError());

closesocket(sock);

return INVALID_SOCKET;

}

if(listen(sock, 5) == SOCKET_ERROR)

{

ASCEPrintf("listen:%d/n", WSAGetLastError());

closesocket(sock);

return INVALID_SOCKET;

}

return sock;

}

LPCONN_CTX CreateConnCtx(SOCKET sockAccept, HANDLE hIOCP)

{

LPCONN_CTX lpConnCtx = (LPCONN_CTX)GlobalAlloc(GPTR, sizeof(CONN_CTX));

if(lpConnCtx == NULL)

return NULL;

lpConnCtx->pPerIOData = (LPPER_IO_DATA)GlobalAlloc(GPTR, sizeof(PER_IO_DATA));

if(lpConnCtx->pPerIOData == NULL)

{

GlobalFree(lpConnCtx);

lpConnCtx = NULL;

return NULL;

}

//赋值

lpConnCtx->pNext = NULL;

lpConnCtx->pPrev = NULL;

lpConnCtx->sockAccept = sockAccept;

ZeroMemory(lpConnCtx->pPerIOData, sizeof(PER_IO_DATA));

lpConnCtx->pPerIOData->OverLapped.hEvent = NULL;

lpConnCtx->pPerIOData->OverLapped.Internal = 0;

lpConnCtx->pPerIOData->OverLapped.InternalHigh = 0;

lpConnCtx->pPerIOData->OverLapped.Offset = 0;

lpConnCtx->pPerIOData->OverLapped.OffsetHigh = 0;

lpConnCtx->pPerIOData->wbuf.buf = (char*)lpConnCtx->pPerIOData->data;

lpConnCtx->pPerIOData->wbuf.len = MAX_BUF_LEN;

lpConnCtx->pPerIOData->oper = SVR_IO_READ;

lpConnCtx->pPerIOData->flags = 0;

//将套接字和完成端口绑定

if(!AssociateWithIoCompletionPort(hIOCP, (HANDLE)sockAccept,

(DWORD)lpConnCtx))

{

ASCEPrintf("AssociateWithIoCompletionPort:%d/n", GetLastError());

GlobalFree(lpConnCtx->pPerIOData);

GlobalFree(lpConnCtx);

lpConnCtx = NULL;

return NULL;

}

return lpConnCtx;

}

void ConnListAdd(LPCONN_CTX lpConnCtx)

{

LPCONN_CTX pTemp;

EnterCriticalSection(&g_CriticalSection);

if(g_ptrConnCtxHead == NULL)

{

//链表的第一个节点

lpConnCtx->pPrev = NULL;

lpConnCtx->pNext = NULL;

g_ptrConnCtxHead = lpConnCtx;

} else {

//加到链表头部

pTemp = g_ptrConnCtxHead;

g_ptrConnCtxHead = lpConnCtx;

lpConnCtx->pNext = pTemp;

lpConnCtx->pPrev = NULL;

pTemp->pPrev = lpConnCtx;

}

LeaveCriticalSection(&g_CriticalSection);

}

void ConnListRemove(LPCONN_CTX lpConnCtx)

{

LPCONN_CTX pPrev = NULL;

LPCONN_CTX pNext = NULL;

EnterCriticalSection(&g_CriticalSection);

if(lpConnCtx != NULL)

{

pPrev = lpConnCtx->pPrev;

pNext = lpConnCtx->pNext;

if((pPrev == NULL) && (pNext == NULL)) //链表唯一的节点

{

g_ptrConnCtxHead = NULL;

} else if((pPrev == NULL) && (pNext != NULL)){ //链表首节点

pNext->pPrev = NULL;

g_ptrConnCtxHead = pNext;

} else if((pPrev != NULL) && (pNext == NULL)){ //链表尾节点

pPrev->pNext = NULL;

} else if((pPrev && pNext)){ //链表中间节点

pPrev->pNext = pNext;

pNext->pPrev = pPrev;

}

//关闭连接，释放资源

closesocket(lpConnCtx->sockAccept);

GlobalFree(lpConnCtx->pPerIOData);

GlobalFree(lpConnCtx);

lpConnCtx = NULL;

}

LeaveCriticalSection(&g_CriticalSection);

return;

}

void ConnListClear()

{

LPCONN_CTX pTemp1, pTemp2;

EnterCriticalSection(&g_CriticalSection);

pTemp1 = g_ptrConnCtxHead;

while(pTemp1)

{

pTemp2 = pTemp1->pNext;

ConnListRemove(pTemp1);

pTemp1 = pTemp2;

}

LeaveCriticalSection(&g_CriticalSection);

return;

}

int ASCEPrintf(const char* lpFormat, ...)

{

int nLen = 0;

int nRet = 0;

char cBuffer[512];

va_list arglist;

HANDLE hOut = NULL;

ZeroMemory(cBuffer, sizeof(cBuffer));

va_start(arglist, lpFormat);

nLen = lstrlen(lpFormat);

nRet = wvsprintf(cBuffer, lpFormat, arglist);

if(nRet >= nLen || GetLastError() == 0)

{

hOut = GetStdHandle(STD_OUTPUT_HANDLE);

if(hOut != INVALID_HANDLE_VALUE)

{

WriteConsole(hOut, cBuffer, lstrlen(cBuffer),

(LPDWORD)&nLen, NULL);

}

}

return nLen;

}

DWORD WINAPI WorkThread(LPVOID lpParam)

{

HANDLE hIOCP = (HANDLE)lpParam;

BOOL bSuccess = FALSE;

DWORD dwIOSize;

LPPER_IO_DATA lpPerIOData;

LPOVERLAPPED lpOverLapped;

LPCONN_CTX lpConnCtx;

int nResult;

while(1)

{

bSuccess = GetQueuedCompletionStatus(hIOCP, &dwIOSize,

(LPDWORD)&lpConnCtx, &lpOverLapped, INFINITE);

if(!bSuccess)

{

ASCEPrintf("GetQueuedCompletionStatus:%d/n", GetLastError());

}

if(lpConnCtx == NULL)

{

return 1;

}

lpPerIOData = (LPPER_IO_DATA)(lpOverLapped);

if(!bSuccess || (bSuccess && (dwIOSize == 0)))

{

ConnListRemove(lpConnCtx);

continue;

}

#ifdef _DEBUG

ASCEPrintf("Different way to obtain PER_IO_DATA/n");

ASCEPrintf("The two one must be equal - A:%x/tB:%x/n",

lpConnCtx->pPerIOData, lpPerIOData);

#endif

switch(lpPerIOData->oper)

{

case SVR_IO_WRITE: //send then reveive

#ifdef _DEBUG

ASCEPrintf("Socket %d Send: %s/n", lpConnCtx->sockAccept,

lpPerIOData->wbuf.buf);

#endif

ZeroMemory(lpPerIOData, sizeof(PER_IO_DATA));

lpPerIOData->OverLapped.hEvent = NULL;

lpPerIOData->OverLapped.Internal = 0;

lpPerIOData->OverLapped.InternalHigh = 0;

lpPerIOData->OverLapped.Offset = 0;

lpPerIOData->OverLapped.OffsetHigh = 0;

lpPerIOData->wbuf.buf = (char*)&(lpPerIOData->data);

lpPerIOData->wbuf.len = MAX_BUF_LEN;

lpPerIOData->oper = SVR_IO_READ;

lpPerIOData->flags = 0;

nResult = WSARecv(lpConnCtx->sockAccept,

&(lpPerIOData->wbuf),

1, NULL, &(lpPerIOData->flags),

&(lpPerIOData->OverLapped),

NULL);

if(nResult == SOCKET_ERROR && WSAGetLastError() != ERROR_IO_PENDING)

{

ASCEPrintf("WSARecv:%d/n", WSAGetLastError());

ConnListRemove(lpConnCtx);

}

break;

case SVR_IO_READ: //receive then echo

#ifdef _DEBUG

ASCEPrintf("Socket %d recv:%s/n", lpConnCtx->sockAccept,

lpPerIOData->wbuf.buf);

#endif

lpPerIOData->wbuf.len = dwIOSize;

lpPerIOData->oper = SVR_IO_WRITE;

lpPerIOData->flags = 0;

nResult = WSASend(lpConnCtx->sockAccept,

&(lpPerIOData->wbuf),

1, NULL, lpPerIOData->flags,

&(lpPerIOData->OverLapped),

NULL);

if(nResult == SOCKET_ERROR &&

WSAGetLastError() != ERROR_IO_PENDING)

{

ASCEPrintf("WSASend:%d/n", WSAGetLastError());

ConnListRemove(lpConnCtx);

}

break;

default:

break;

}

}

return 0;

}