Completion Port
Windows2000之后的版本提供的完成端口的应用，这使得开发服务器端程序变得更简便了，我们不用再为每一个连接而去维护令人厌烦的线程池。Windows提供的完成端口封装了一切繁琐的工作，我们唯一要做的就是对完成端口的应用。// 创建监听套接字
SOCKET m_socListen = WSASocket(AF_INET,SOCK_STREAM,0,NULL,0,WSA_FLAG_OVERLAPPED);
// 设置异步连接标志
if (WSAAsyncSelect(m_socListen, m_hMainWnd, UM_SOCKETEVENT, FD_ACCEPT)==SOCKET_ERROR) return SOCKET_ERROR;
// 初始化完成端口
IOCP_Initialize();
/*
IOCP_Initialize()
{
SYSTEM_INFO systeminfo;
DWORD dwThreadID;
m_hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
if( m_hCompletionPort == NULL
{
TRACE(_T("CreateIoCompletionPort failed with error: %d\n", GetLastError());
return -1;
}
m_bStopIOCP=FALSE;
m_hThreadIOCP=NULL;
GetSystemInfo( &systeminfo ;
for(DWORD i=0; i<systeminfo.dwNumberOfProcessors; i++)
{
m_hThreadIOCP = CreateThread( NULL, // Security
0, // Stack size - use default
(LPTHREAD_START_ROUTINE)IOCP_ThreadPoolFunc, // Thread function entry point
(void*) this, // Param for thread
0, // Init flag
&dwThreadID); // Thread ID
if (m_hThreadIOCP!=NULL)
{
InterlockedIncrement((LPLONG)&m_nWorkerCnt);
CloseHandle(m_hThreadIOCP);
}
}
}
IOCP_ThreadPoolFunc (LPVOID lpParam)
{
ASSERT(lpParam);
CIOCPServer* pThis = reinterpret_cast<CIOCPServer*>(lpParam);
HANDLE hCompletionPort = pThis->m_hCompletionPort;
DWORD dwBytesTransferred=0; // Bytes transferred in the Completion packet
DWORD dwIoType=0; // The operation type of completion packet
DWORD dwFlags=0;
LPPERHANDLEDATA lpPerHandleData;
LPPERIOOPERATIONDATA lpPerIoData;
while(!pThis->IsIOCPstopped())
{
BOOL bIoRet = FALSE;
if((bIoRet = GetQueuedCompletionStatus(
hCompletionPort,
&dwBytesTransferred,
(LPDWORD) &lpPerHandleData,
(LPOVERLAPPED*)&lpPerIoData,
INFINITE))==0)
{
DWORD dwError=GetLastError();
if(dwError==ERROR_NETNAME_DELETED)
{
// 服务器主动关闭套接字
if(lpPerIoData) { pThis->FreePerIoData(lpPerIoData); lpPerIoData=NULL; };
/*
FreePerIoData(PERIOOPERATIONDATA* lpPerIoData)
{
if(lpPerIoData->m_pOutBuf)
{
delete[] lpPerIoData->m_pOutBuf;
lpPerIoData->m_pOutBuf=NULL;
}
delete(lpPerIoData);
lpPerIoData=NULL;
}
*/
continue;
}
TRACE("GetQueuedCompletionStatus failed with error %d\n", dwError);
InterlockedDecrement(&pThis->m_nWorkerCnt);
return 0;
}
if(lpPerIoData==NULL && lpPerHandleData==NULL)
{
// 服务器关闭完成端口
InterlockedDecrement(&pThis->m_nWorkerCnt);
return 0;
}
// First check to see if an error has occured on the socket and if so
// then close the socket and cleanup the SOCKET_INFORMATION structure
// associated with the socket.
if ((dwBytesTransferred==0) && ((lpPerIoData->m_ioType==IORead)||(lpPerIoData->m_ioType==IOWrite)))
{
pThis->RemoveStaleClient(lpPerHandleData->m_Socket,FALSE);
if(lpPerIoData) { pThis->FreePerIoData(lpPerIoData); lpPerIoData=NULL;
continue;
}
if( lpPerHandleData!=NULL && lpPerIoData!=NULL
{
BOOL bRet = pThis->IOCP_ProcessIOMessage(lpPerHandleData, lpPerIoData, dwBytesTransferred);
// Before Every PostQueuedCompletionStatus() call we allocate a memory for PerIoData.
// And Before every WSASend() and WSARecv() call we also allocate such a block of memory.
// So After handling every QueuedCompletion Packet, we Free this block if it's not useful.
/*
enum IOType { IOIdle,IORead,IOWrite,IOInitialize };
IOCP_ProcessIOMessage(PERHANDLEDATA* pContext, PERIOOPERATIONDATA* pPerIoData, DWORD dwSize = 0)
{
int nRet = 0;
IOType clientIO = pPerIoData->m_ioType;
if( clientIO == IOInitialize)
nRet = IOCP_OnClientInitializing(pContext, pPerIoData, dwSize);
else if( clientIO == IORead)
nRet = IOCP_OnClientReading(pContext, pPerIoData, dwSize);
else if( clientIO == IOWrite)
nRet = IOCP_OnClientWriting(pContext, pPerIoData, dwSize);
}
IOCP_OnClientInitializing(PERHANDLEDATA* lpContext, PERIOOPERATIONDATA* lpPerIoData, DWORD dwIoSize)
{
// launch another recv operation.
DWORD dwBytesTransferred=0;
PERIOOPERATIONDATA* lpPerIoDataResv = new PERIOOPERATIONDATA;
memset(lpPerIoDataResv,0,sizeof(PERIOOPERATIONDATA));
lpPerIoDataResv->m_ioType = IORead;
lpPerIoDataResv->m_wsaDataBuf.buf = lpPerIoDataResv->m_Buffer;
lpPerIoDataResv->m_wsaDataBuf.len = sizeof(lpPerIoDataResv->m_Buffer);
ULONG ulFlags = 0;
int nRet = WSARecv( lpContext->m_Socket,
&lpPerIoDataResv->m_wsaDataBuf,
1,
&dwBytesTransferred,
&ulFlags,
(LPWSAOVERLAPPED)&lpPerIoDataResv->Overlapped,
NULL);
if ( nRet == SOCKET_ERROR
{
DWORD dwError = WSAGetLastError();
if( dwError != WSA_IO_PENDING)
{
RemoveStaleClient( lpContext->m_Socket, FALSE ;
return NC_E_IOCP_INITIALIZE|dwError;
}
}
}
IOCP_OnClientReading(PERHANDLEDATA* pContext, PERIOOPERATIONDATA* pPerIoData, DWORD dwSize)
{
// make sure to issue a read after this
// pContext->m_wsaInBuffer.buf/*==pContext->m_byInBuffer*/是该套接字的缓冲区，
// 如果该缓冲区中有数据(pContext->m_wsaInBuffer.len!=0时)，
// 则说明上次收到了一个请求包的一部分数据，把这次的数据继续放入
memmove(pContext->m_wsaInBuffer.buf+pContext->m_wsaInBuffer.len,pPerIoData->m_wsaDataBuf.buf,dwSize);
pContext->m_wsaInBuffer.len+=dwSize;
HandleRequest(pContext);
// launch another recv operation.
DWORD dwBytesTransferred=0;
PERIOOPERATIONDATA* lpPerIoDataResv = new PERIOOPERATIONDATA;
memset(lpPerIoDataResv,0,sizeof(PERIOOPERATIONDATA));
lpPerIoDataResv->m_ioType = IORead;
lpPerIoDataResv->m_wsaDataBuf.buf = lpPerIoDataResv->m_Buffer;
lpPerIoDataResv->m_wsaDataBuf.len = sizeof(lpPerIoDataResv->m_Buffer);
int nRet = WSARecv( pContext->m_Socket,
&lpPerIoDataResv->m_wsaDataBuf,
1,
&dwBytesTransferred,
&ulFlags,
(LPWSAOVERLAPPED)&lpPerIoDataResv->Overlapped,
NULL);
if ( nRet == SOCKET_ERROR
{
DWORD dwError = WSAGetLastError();
if( dwError != WSA_IO_PENDING && dwError != WSAENOTSOCK)
{
RemoveStaleClient( pContext->m_Socket, FALSE ;
return NC_E_IOCP_READ|dwError;
}
}
}
IOCP_OnClientWriting(PERHANDLEDATA* pContext, PERIOOPERATIONDATA* pPerIoData, DWORD dwSize)
{
// SendPacket调用异步发送数据，当数据发送完成后，将来此处
}
SendPacket(PERHANDLEDATA* pContext,BYTE* sendbuf,DWORD dwLength)
{
// 发送数据代码断如下:
PERIOOPERATIONDATA* lpPerIoDataRPP = new PERIOOPERATIONDATA;
memset(lpPerIoDataRPP,0,sizeof(PERIOOPERATIONDATA));
lpPerIoDataRPP->m_ioType = IOWrite;
memset(lpPerIoDataRPP->m_Buffer,0,sizeof(lpPerIoDataRPP->m_Buffer));
lpPerIoDataRPP->m_pOutBuf=new BYTE[dwLength];
memmove(lpPerIoDataRPP->m_pOutBuf,(BYTE*)sendbuf,dwLength);
lpPerIoDataRPP->m_wsaDataBuf.buf = (char*)lpPerIoDataRPP->m_pOutBuf;
lpPerIoDataRPP->m_wsaDataBuf.len = dwLength;
int nRetVal = WSASend(pContext->m_Socket,
&lpPerIoDataRPP->m_wsaDataBuf,
1,
&lpPerIoDataRPP->m_wsaDataBuf.len,
0,
&(lpPerIoDataRPP->Overlapped),
NULL);
if ( nRetVal == SOCKET_ERROR
{
DWORD dwError=WSAGetLastError();
if( dwError != WSA_IO_PENDING
{
RemoveStaleClient( socClient, FALSE ;
return NC_E_TCP_SENDRESPONSE|dwError;
}
}
// 发送完毕
}
*/
if( lpPerIoData { pThis->FreePerIoData(lpPerIoData); lpPerIoData=NULL; }
}
}
InterlockedDecrement(&pThis->m_nWorkerCnt);
return 0;
}
*/
// 绑定服务器地址
SOCKADDR_IN saiServer;
saiServer.sin_family = AF_INET;
saiServer.sin_port = htons(NC_S_PORT_LISTEN);
saiServer.sin_addr.S_un.S_addr = inet_addr(pszLocalAddr);
if(bind(m_socListen,(SOCKADDR*)&saiServer,sizeof(saiServer))==SOCKET_ERROR) return NC_E_TCP_LISTEN|WSAGetLastError();
// 开始侦听
if(listen(m_socListen,SOMAXCONN)==SOCKET_ERROR) return NC_E_TCP_LISTEN|WSAGetLastError();
case FD_ACCEPT: IOCP_OnAccept();break;
IOCP_OnAccept()
{
SOCKET socClient;
SOCKADDR_IN saiRemote;
int nLen = sizeof(saiRemote);
if( (socClient=accept(m_socListen,(LPSOCKADDR)&saiRemote,&nLen))==SOCKET_ERROR) return NC_E_TCP_ONACCEPT|WSAGetLastError();
// 1、设置套接字选项
BOOL bOpt = true;
int nRcvBuf = BUFSIZE_NC_S_RCVBUF;
int nSndBuf = BUFSIZE_NC_S_SNDBUF;
int nErr = 0;
int nTimeout = 10*1000;
nErr = setsockopt(socClient, SOL_SOCKET, SO_SNDTIMEO, (char*)&nTimeout, sizeof(nTimeout));
nErr = setsockopt(socClient, SOL_SOCKET, SO_RCVBUF, (const char*)&nRcvBuf,sizeof(nRcvBuf));
nErr = setsockopt(socClient, SOL_SOCKET, SO_SNDBUF, (const char*)&nSndBuf,sizeof(nSndBuf));
// 2、将该套接字关联完成端口
// Create the Client context to be associted with the completion port
PERHANDLEDATA* pContext = IOCP_AllocatePerHandleContext();
/*
typedef struct tagPerHandleData
{
SOCKET m_Socket;
DWORD m_dwStatus;
// Input Elements for Winsock
WSABUF m_wsaInBuffer;
BYTE m_byInBuffer[DATA_BUFSIZE];
// Output elements for Winsock
WSABUF m_wsaOutBuffer;
}PERHANDLEDATA, *LPPERHANDLEDATA;
IOCP_AllocatePerHandleContext()
{
PERHANDLEDATA* pContext = NULL;
if (!m_listFreePool.IsEmpty())
{
pContext = m_listFreePool.RemoveHead();
}
else
{
pContext = new PERHANDLEDATA;
memset(pContext,0,sizeof(PERHANDLEDATA));
}
if(pContext) memset(pContext,0,sizeof(PERHANDLEDATA));
return pContext;
}
*/
pContext->m_Socket = socClient;
pContext->m_wsaInBuffer.buf = (char*)pContext->m_byInBuffer;
pContext->m_wsaInBuffer.len = 0;
// 保存PerHandleContext的列表
PERHANDLEDATA* pContextTmp = NULL;
if (m_listContexts.Lookup(socClient, pContextTmp))
RemoveStaleClient(socClient,TRUE);
else
m_listContexts.SetAt(nSocket, pContext);
/*
RemoveStaleClient(SOCKET s,BOOL bGraceful)
{
PERHANDLEDATA* pContext=NULL;
m_listContexts.Lookup(nSocket,pContext);
if(pContext==NULL) return;
LINGER lingerStruct;
if ( bGraceful
{
lingerStruct.l_onoff = 1;
lingerStruct.l_linger= 30; // linger for 30 seconds, 0 for abort
}
else
{
lingerStruct.l_onoff = 0;
lingerStruct.l_linger= 0; // ignored
}
setsockopt( pContext->m_Socket, SOL_SOCKET, SO_LINGER, (char *)&lingerStruct, sizeof(lingerStruct) ;
while (!HasOverlappedIoCompleted((LPOVERLAPPED)pContext)) Sleep(0);
MoveToFreePool(nSocket);
//
//MoveToFreePool(SOCKET s)
//{
// PERHANDLEDATA* pContext = NULL;
//
// // Free context structures
// if (m_listContexts.Lookup(nSocket, pContext))
// {
// m_listFreePool.AddTail(pContext);
// m_listContexts.RemoveKey(nSocket);
// }
//}
//
// Free context structures
// Now close the socket handle. This will do an abortive or graceful close, as requested.
closesocket( pContext->m_Socket ;
pContext->m_Socket = INVALID_SOCKET;
}
*/
// Associate the new socket with a completion port.
if(IOCP_AssociateSocketWithPort(pContext->m_Socket, m_hCompletionPort, (DWORD)pContext)!=0)
{
RemoveStaleClient(socClient,TRUE);
DWORD dwError=GetLastError();
return NC_E_TCP_ONACCEPT;
}
/*
IOCP_AssociateSocketWithPort(SOCKET socket, HANDLE hCompletionPort, DWORD dwCompletionKey)
{
HANDLE h=CreateIoCompletionPort((HANDLE) socket, hCompletionPort, dwCompletionKey, 0);
return (h==hCompletionPort)?0:-1;
}
*/
// 3、该套接字的完成端口初始化
// Trigger first IO Completion Request
// Otherwise the Worker thread will remain blocked waiting for GetQueuedCompletionStatus...
// The first message that gets queued up is ClientIoInitializing - see ThreadPoolFunc
/*
typedef struct tagPerIoOperationData
{
OVERLAPPED Overlapped;
IOType m_ioType;
BYTE* m_pOutBuf;
WSABUF m_wsaDataBuf;
CHAR m_Buffer[DATA_BUFSIZE];
} PERIOOPERATIONDATA,*LPPERIOOPERATIONDATA;
*/
PERIOOPERATIONDATA* lpPerIoData = NULL;
lpPerIoData = new PERIOOPERATIONDATA;
memset(lpPerIoData,0,sizeof(PERIOOPERATIONDATA));
lpPerIoData->m_ioType = IOInitialize;
lpPerIoData->m_wsaDataBuf.len = sizeof(lpPerIoData->m_Buffer);
lpPerIoData->m_wsaDataBuf.buf = lpPerIoData->m_Buffer;
BOOL bSuccess = PostQueuedCompletionStatus(m_hCompletionPort, 0, (DWORD) pContext, &lpPerIoData->Overlapped);
if(!bSuccess) return NC_E_TCP_ONACCEPT|WSAGetLastError();
return socClient;
}
// 结束的时候需要关闭完成端口
CloseListenSocket()
{
// 关闭每个连接, 从列表中移去关键字
PERHANDLEDATA* pContext = NULL;
int nSocket=0;
do
{
POSITION pos = m_listContexts.GetStartPosition();
if (pos)
{
m_listContexts.GetNextAssoc(pos, nSocket, pContext);
RemoveStaleClient(nSocket, FALSE);
}
}while (!m_listContexts.IsEmpty());
// 释放每个连接相关的分配内存
while (!m_listFreePool.IsEmpty())
{
PERHANDLEDATA* pContext = m_listFreePool.RemoveTail();
delete pContext;
}
m_bStopIOCP=TRUE;
if(m_bIOCPInitialed==TRUE)
{
IOCP_CloseCompletionPort();
/*
IOCP_CloseCompletionPort()
{
while (m_nWorkerCnt)
{
PostQueuedCompletionStatus(m_hCompletionPort, 0, (DWORD) NULL, NULL);
Sleep(1000);
}
// Close the CompletionPort and stop any more requests
CloseHandle(m_hCompletionPort);
m_bIOCPInitialed=FALSE;
m_bStopIOCP=TRUE;
}
} */
}
if(m_socListen!=0)
{
closesocket(m_socListen);
m_socListen=0;
}
return 0;
}