完成端口iocp编程模型

简介

完成端口的基本思想和计算机组成原理中的dma非常类似，客户调用overlap操作抛出一个读写请求，然后在工作线程在一个位置（port）等待完成，如果完成，客户根据返回值进行后续的读写操作。在用户抛出异步读写请求以后，就可以转而去做其他事情了。

1，定义异步请求保存数据的结构

enum IO_OPERATION {
    IO_READ     =   1, 
    IO_SEND     =   2,
    IO_ACCEPT   =   3,
};

struct IO_DATA{
    WSAOVERLAPPED               Overlapped;
    // 原子锁
    DWORD                       dwLock;
    // 上次接受到数据的时间, 判断连接的不活动时间
    DWORD                       dwEnterTick;    
    // 操作代码
    IO_OPERATION                opCode;
    // 网络套接字句柄
    SOCKET                      activeSocket;
    // 网络地址
    DWORD                       dwIpAddr;
    // 端口
    WORD                        wPort;
    // 请求类型 
    char                        cReqType; //PROXY_HTTP;PROXY_NONE;
    // 缓冲区
    WSABUF                      wsabuf;
    // 已经处理的io字节数
    DWORD                       dwRecved;
    // 缓冲区总字节数
    DWORD                       dwRecvBufTail;
    // wsaBuf指向的缓冲区
    char                        RecvBuffer[MAX_BUFF_SIZE];
    // 已经处理的io字节数
    DWORD                       dwSent;
    // 缓冲区总字节数
    DWORD                       dwSendBufTail;
    // wsaBuf指向的缓冲区
    char                        SendBuffer[MAX_BUFF_SIZE];
};

2, 初始化listen

void ListenThread(void * args)
{
    { // Init winsock2
        WSADATA wsaData;
        ZeroMemory(&wsaData,sizeof(WSADATA));
        int retVal = -1;
        if( (retVal = WSAStartup(MAKEWORD(2,2), &wsaData)) != 0 ) {
            PUTLOG("WSAStartup Failed::Reason Code:: %u\n", retVal);
            return;
        }
    }

    { // Init Tls Index
        if (!InitTlsIndex())
            return ;
    }

    {  //Create socket
        g_ServerSocket = WSASocket(AF_INET,SOCK_STREAM
                                  , IPPROTO_TCP
                                  , NULL
                                  ,0
                                  ,WSA_FLAG_OVERLAPPED);

        if( g_ServerSocket == INVALID_SOCKET ) 
        {
            PUTLOG("Server Socket Creation Failed::Reason Code:: %u\n"
                    , WSAGetLastError());
            return;
        }
    }

    {   //bind
        sockaddr_in service;
        service.sin_family = AF_INET;
        service.sin_addr.s_addr = htonl(INADDR_ANY);
        service.sin_port = htons(PHONE_PORT);
        int retVal = bind(g_ServerSocket,(SOCKADDR *)&service,sizeof(service));
        if( retVal == SOCKET_ERROR ) {
            PUTLOG("Server Soket Bind Failed::Reason Code:: %u\n"
                    , WSAGetLastError());
            return;
        }
    }

    {   //listen
        int retVal = listen(g_ServerSocket, 100);
        if( retVal == SOCKET_ERROR ) 
        {
            PUTLOG("Server Socket Listen Failed::Reason Code:: %u\n" 
                    , WSAGetLastError());
            return;
        }
        PUTLOG("Server Socket Listen at:: %d \n", PHONE_PORT);;
    }

    {   // Create IOCP
        SYSTEM_INFO sysInfo;
        ZeroMemory(&sysInfo,sizeof(SYSTEM_INFO));
        GetSystemInfo(&sysInfo);
        g_ThreadCount = sysInfo.dwNumberOfProcessors + 1;
        g_hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,0,g_ThreadCount);
        if (g_hIOCP == NULL) 
        {
            PUTLOG("CreateIoCompletionPort() Failed::Reason:: %u\n"
                    , GetLastError());
            return;            
        }

       if (CreateIoCompletionPort((HANDLE)g_ServerSocket,g_hIOCP,0,0) == NULL)
       {
            PUTLOG("Binding Server Socket to IO Completion Port Failed::Reason Code:: %u\n"
                    , GetLastError());
            return ;    
        }
    }

    {  //Create worker threads
        for( DWORD dwThread=0; dwThread < g_ThreadCount; dwThread++ )
        {
            HANDLE  hThread;
            DWORD   dwThreadId;

            hThread = CreateThread(NULL, 0, WorkerThread, 0, 0, &dwThreadId);
            CloseHandle(hThread);
        }
    }

    { 
        // accept new connection
        // Load the AcceptEx function into memory using WSAIoctl.
        // The WSAIoctl function is an extension of the ioctlsocket()
        // function that can use overlapped I/O. The function's 3rd
        // through 6th parameters are input and output buffers where
        // we pass the pointer to our AcceptEx function. This is used
        // so that we can call the AcceptEx function directly, rather
        // than refer to the Mswsock.lib library.
        GUID GuidAcceptEx = WSAID_ACCEPTEX;
        DWORD dwBytes = 0;
        int iResult = WSAIoctl(g_ServerSocket
            , SIO_GET_EXTENSION_FUNCTION_POINTER,
            &GuidAcceptEx, sizeof (GuidAcceptEx), 
            &g_lpfnAcceptEx, sizeof (g_lpfnAcceptEx), 
            &dwBytes, NULL, NULL);

        if (iResult == SOCKET_ERROR) {
            wprintf(L"WSAIoctl failed with error: %u\n", WSAGetLastError());
            closesocket(g_ServerSocket);
            WSACleanup();
            return ;
        }

        // 投递accept操作
        enum {
            nMaxAcceptRequest = 1,
        };
        IO_DATA * pIoData = new IO_DATA [nMaxAcceptRequest];
        memset(pIoData, 0, sizeof(IO_DATA)*nMaxAcceptRequest);

        for (int xReq = 0; xReq < nMaxAcceptRequest; xReq++)
        {
            if (!PostAcceptRequest(pIoData + xReq))
                break;
        } //END OF POST ACCEPT
    }
}

2， 3个投递函数，

2.a 投递accept请求

bool PostAcceptRequest(IO_DATA * pIoData)
{
    DWORD dwBytes = 0;
    // Create an accepting socket
    SOCKET AcceptSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (AcceptSocket == INVALID_SOCKET) 
    {
        PUTLOG("Create accept socket failed with error: %u\n"
                , WSAGetLastError());
        return false;
    }

    // Empty our overlapped structure and accept connections.
    memset(&pIoData->Overlapped, 0, sizeof(pIoData->Overlapped));

    pIoData->opCode = IO_ACCEPT;
    pIoData->activeSocket = AcceptSocket;

    BOOL bRetVal = g_lpfnAcceptEx(g_ServerSocket
        , AcceptSocket
        , pIoData->RecvBuffer
        , 0 //, sizeof(pIoData[nCocurrent].Buffer)
            //  -((sizeof(sockaddr_in)+16)*2)
        , sizeof (sockaddr_in) + 16
        , sizeof (sockaddr_in) + 16
        , &dwBytes
        , &pIoData->Overlapped);

    if (bRetVal == FALSE && GetLastError() != ERROR_IO_PENDING) 
    {
        PUTLOG("AcceptEx failed with error: %u\n"
                , WSAGetLastError());
        closesocket(AcceptSocket);
        return false;
    }

    // associate
    if (NULL == (CreateIoCompletionPort((HANDLE) AcceptSocket
                                        , g_hIOCP
                                        , (u_long) 0
                                        , 0))) 
    {
        PUTLOG("CreateIoCompletionPort associate failed with error: %u\n"
                , GetLastError());                
        closesocket(AcceptSocket);
        return false;
    }

    return true;
}

3.b 抛出recv请求

bool PostRecvRequest(IO_DATA * const ioReq, SOCKET sockfd)
{
     IO_DATA * data = ioReq;
     data->activeSocket = sockfd;
     ZeroMemory(&data->Overlapped,sizeof(data->Overlapped));
     data->opCode = IO_OPERATION(IO_READ);
     data->wsabuf.buf  = data->RecvBuffer + data->dwRecved;
     data->dwRecvBufTail = sizeof(data->RecvBuffer);
     data->wsabuf.len  = data->dwRecvBufTail - data->dwRecved;

     DWORD dwRecvNumBytes=0;
     DWORD dwFlags=(MSG_PEEK & (~MSG_PEEK));

     int nRet = WSARecv(data->activeSocket
                     , &data->wsabuf
                     , 1
                     , &dwRecvNumBytes
                     , &dwFlags
                     , &data->Overlapped
                     , NULL);

     if(nRet == SOCKET_ERROR  && (ERROR_IO_PENDING != WSAGetLastError()))
     {
         PUTLOG("WASRecv Failed::Reason Code:: %u\n"
                , WSAGetLastError());
         return false;
     }

     return true;
}

3.c 抛出send请求

bool PostSendRequest(IO_DATA * const ioReq, SOCKET sockfd)
{
     IO_DATA * data = ioReq;
     data->activeSocket = sockfd;
     ZeroMemory(&data->Overlapped,sizeof(data->Overlapped));
     data->opCode = IO_OPERATION(IO_SEND);
     data->wsabuf.buf  = data->SendBuffer + data->dwSent;
     data->wsabuf.len  = data->dwSendBufTail - data->dwSent;

     DWORD dwSendNumBytes=0;
     DWORD dwFlags=(MSG_PEEK & (~MSG_PEEK));

     int nRet = WSASend(data->activeSocket
                     , &data->wsabuf
                     , 1
                     , &dwSendNumBytes
                     , dwFlags
                     , &data->Overlapped
                     , NULL);

     if(nRet == SOCKET_ERROR  && (ERROR_IO_PENDING != WSAGetLastError()))
     {
         PUTLOG("WSASend Failed::Reason Code:: %u\n"
                , WSAGetLastError());
         return false;
     }

     return true;
}

4，工作线程

DWORD WINAPI WorkerThread (LPVOID WorkThreadContext) 
{
    for (;;) 
    {
        BOOL bSuccess = FALSE;
        DWORD dwIoSize = 0;
        void * lpCompletionKey = NULL;
        LPWSAOVERLAPPED lpOverlapped = NULL;
        bSuccess = GetQueuedCompletionStatus(g_hIOCP
                                            , &dwIoSize
                                            , (LPDWORD)&lpCompletionKey
                                            , (LPOVERLAPPED *)&lpOverlapped
                                            , INFINITE);

         if(!bSuccess)
         {
            PUTLOG("GetQueuedCompletionStatus() failed: %u\n",
                    GetLastError());
         }

         IO_DATA * lpIOContext = (IO_DATA *)lpOverlapped;

         // 新的套接字连接上来
         if(lpIOContext->opCode == IO_ACCEPT) // a read operation complete
         {
            // 更新AccpetEx返回的socket状态
            int ErrorCode= setsockopt(lpIOContext->activeSocket, 
                                    SOL_SOCKET,
                                    SO_UPDATE_ACCEPT_CONTEXT,
                                    (char*)&g_ServerSocket,
                                    sizeof(g_ServerSocket));

            // 提交读取请求
            IO_DATA * data = new IO_DATA;
            if (data)
            {
                PostRecvRequest(data, lpIOContext->activeSocket);   
            }
            else
            {
                closesocket(data->activeSocket); 
                data->activeSocket = INVALID_SOCKET;
                delete data;
            }

            // 提交一个新的accept
            PostAcceptRequest(lpIOContext);

            continue ;
         }
        
        if(dwIoSize == 0) //socket closed?
        {
            PUTLOG("Client Dis-connected.\n");
            closesocket(lpIOContext->activeSocket); 
            lpIOContext->activeSocket = INVALID_SOCKET;
            continue;
        }
        
        // 读取完毕
        if(lpIOContext->opCode == IO_READ)
        {
            // 重置超时
            lpIOContext->dwEnterTick = GetTickCount();

            // 累加总请求长度
            lpIOContext->dwRecved += dwIoSize;

            bool IsReqReady = true;
            // 
            // 如果数据包读取完毕
            // 开始处理
            // 处理完投递发送请求
            // 
            // ...
            if (IsReqReady)
            {
                lpIOContext->dwSent = 0;
                lpIOContext->dwSendBufTail = 0;

                memcpy(lpIOContext->SendBuffer + lpIOContext->dwSendBufTail
                        , "\r\n\r\nServerEcho: "
                        , 16);
                lpIOContext->dwSendBufTail += 16;

                memcpy(lpIOContext->SendBuffer + lpIOContext->dwSendBufTail
                    , lpIOContext->RecvBuffer
                    , lpIOContext->dwRecved);

                lpIOContext->dwSendBufTail += lpIOContext->dwRecved;

                memcpy(lpIOContext->SendBuffer + lpIOContext->dwSendBufTail
                    , "\r\n\r\n"
                    , 4);

                lpIOContext->dwSendBufTail += 4;

                PostSendRequest(lpIOContext, lpIOContext->activeSocket);
            }
            // 
            // 如果没有读取完毕
            // 继续投递读取请求
            //
            // ...
            else
            {
                PostRecvRequest(lpIOContext, lpIOContext->activeSocket);
            }
            continue ;
        }

        // 发送完毕
        if (lpIOContext->opCode == IO_SEND)
        {
            // 重置超时
            lpIOContext->dwEnterTick = GetTickCount();

            lpIOContext->dwSent += dwIoSize;

            bool IsRespReady 
                = (lpIOContext->dwSent == lpIOContext->dwSendBufTail);

            // 
            // 如果数据包发送完毕
            // 开始回话后处理
            // 处理完投递读取请求
            // 
            // ...
            if (IsRespReady)
            {
                lpIOContext->dwRecved = 0;
                lpIOContext->dwRecvBufTail = sizeof(lpIOContext->RecvBuffer);

                PostRecvRequest(lpIOContext, lpIOContext->activeSocket);
            }

            // 
            // 如果没有发送完毕
            // 继续投递发送请求
            //
            // ...
            else
            {
                PostSendRequest(lpIOContext, lpIOContext->activeSocket);
            }
            continue ;
        }
    }
    return 0;
}

5, 用一个main启动看看效果

int main(int argc, char * argv [])
{
    ListenThread(NULL);
    
    PUTLOG("Press any key to quit server.");
    getchar();
    return 0;
}

输出：

=====================================================================

项目文件下载：

http://hi.csdn.net/attachment/201112/26/0_1324859981Zd6C.gif