C++完成端口组件实现

在windows平台上，处理众多的TCP连接的机制中效率最高的是完成端口模型，关于完成端口模型的介绍可参照《Win32 多线程程序设计》（侯捷翻译）和《windows网络编程》。

异步机制是完成端口的基础，完成端口模型的本质思想是将"启动异步操作的线程"和"提供服务的线程"（即工作者线程）拆伙。

先看看所有TCP连接管理组件都要实现的接口 ITcpManager

#ifndef ITCPMANAGER_H
#define ITCPMANAGER_H
#include "../../Infrastructure/IEvent.h"
#include "IMessageDispatcher.h"

/*
 ITcpManager 所有基于TCP网络管理者的基础接口 。
 
 创 作 者：sky
 时    间：2005.6 
 修订时间：2005.6
*/

enum TcpUserAction
{
 UA_Connected , UA_Disconnected , UA_FunctionAccess , //标准的功能访问  
};

enum DisconnectedCause
{
 Logoff ,
 LineOff ,
 ServerStopped , //服务器停止时关闭连接
 InvalidUser ,   //强行关闭非法用户的连接
 TimeOut         //在一段时间内没有收到check消息
};

//
interface ITcpClientsController
{
public:
 virtual ~ITcpClientsController(){}
 
 //主动给某个客户发信息 ，线程安全
 virtual void SendData(int ConnectID, byte* data ,int offset ,int size) = 0  ;
 
 //主动关闭连接
 virtual void DisposeOneConnection(int connectID ,DisconnectedCause cause) = 0 ;
};

class UserActionEventArgs
{
public:
 int ConnectID ;
 TcpUserAction action ;
};

class ServiceCommittedEventArgs
{
public:
 char* data ;
 int   length ;
 int   connectID ;
};

interface ITcpManager : public ITcpClientsController
{
 
public:
 virtual ~ITcpManager(){}
 
 //Events
 IEvent<ITcpManager* ,int>* ConnectionCountChanged ;
 IEvent<ITcpManager* ,ServiceCommittedEventArgs&>* ServiceCommitted ;
 IEvent<ITcpManager* ,UserActionEventArgs&>* UserAction ;

 //Method
 virtual void Start() = 0 ;
 virtual void Stop() = 0 ;
 
 virtual int ConnectionCount() = 0 ; 
};

#endif

接口的定义很清晰明了－－这也是接口的本质，如果你对IEvent事件还不是很熟悉，请参见我前面的文章。

接下来看看实现了ITcpManager的完成端口组件的头文件：

#ifndef COMPLETINGPORT_H
#define COMPLETINGPORT_H

#include <WINSOCK2.H>
#include <afx.h>

#include "../Foundation/ITcpManager.h"
#include "../Foundation/TCPListener.h"
#include "../Foundation/IMessageDispatcher.h"
#include "../../Collection/SafeObjectList.h"
#include "WorkerThreadAssistant.h"
#include "ContextKeyManager.h"

typedef SafeObjectList<HANDLE> OverlapEventList ;

void WorkerThreadStart(void* para) ;
void ListenThreadStart(void* para) ;

class CompletingPortManager :public ITcpManager ,public IEventHandler<ContextKeyManager* ,int>
{
private:
 WorkerThreadAssistant workerThread_assistant ;
 OverlapEventList overlapEventList ;
 ContextKeyManager* context_manager ; 
 TcpListener* listener ;
 IMessageDispatcher* curDispatcher ;

 HANDLE completing_port ;
 int workThreadNum ;
 volatile bool isStop ;

 

public: //interface接口
 //Events
 IEvent<ITcpManager* ,int>*       ConnectionCountChanged ;
 IEvent<ITcpManager* ,ServiceCommittedEventArgs&>* ServiceCommitted ;
 IEvent<ITcpManager* ,UserActionEventArgs&>*   UserAction ;

 //Method
 void Start()  ;
 void Stop()  ;
 
 int ConnectionCount()  ;
 
 //主动给某个客户发信息 
 void SendData(int ConnectID, byte* data ,int offset ,int size)   ; 
 //主动关闭连接
 void DisposeOneConnection(int connectID ,DisconnectedCause cause) ;
 
public:
 friend void WorkerThreadStart(void* para) ;
 friend void ListenThreadStart(void* para) ;
 
 CompletingPortManager(IMessageDispatcher* i_dispatcher ,int port ,int workThread_Num )
 {
  this->isStop = true ; 
  this->workThreadNum = workThread_Num ;
  this->context_manager = new ContextKeyManager() ;
  this->listener = new TcpListener(port) ;
  this->curDispatcher = i_dispatcher ; 

  //subscribe Event
  this->context_manager->ConnectionCountChanged->Register(this) ;
  
  //event 
  this->ConnectionCountChanged = new EventPublisher<ITcpManager* ,int> ;
  this->ServiceCommitted       = new EventPublisher<ITcpManager* ,ServiceCommittedEventArgs&> ;
  this->UserAction    = new EventPublisher<ITcpManager* ,UserActionEventArgs&> ;
  
  this->completing_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE ,NULL ,0 ,0 ) ; 
 }

 ~CompletingPortManager()
 {
  CloseHandle(this->completing_port) ;
  delete this->context_manager ;
  delete this->listener ;

  //event
  delete this->ConnectionCountChanged ;
  delete this->ServiceCommitted ;
  delete this->UserAction ;
 }

 //ContextKeyManager::ConnectionCountChanged事件
 void HandleEvent(ContextKeyManager* sender ,int para)
 {
  ((EventPublisher<ITcpManager* ,int>*)(this->ConnectionCountChanged))->Invoke(this ,para) ;
  
 } 
 
private:
 void ListenThread() ;
 void CreateWorkerThreads() ;
 void ServeCmpltdOverlap() ;
 void BindConnectionToCompPort(SOCKET connection ,ContextKey** out_pky);
 //进行异步接收数据
 void RecieveData(ContextKey* pkey);

 void CloseAllWorkerThread() ;
    void CloseAllOverlapEvent() ;
 void ClearRespondStreamList(RespondStreamList* rs_list) ;
 
};

#endif

完成端口组件的.CPP文件如下：

#include "CompletingPortManager.h"
#include "../../Threading/Thread.h"

#ifdef _DEBUG
#define new DEBUG_NEW
#undef THIS_FILE
static char THIS_FILE[] = __FILE__;
#endif

//global
void WorkerThreadStart(void* para)
{
    CompletingPortManager* cpm = (CompletingPortManager*)para ;
    cpm->ServeCmpltdOverlap() ;
}

void ListenThreadStart(void* para)
{
    CompletingPortManager* cpm = (CompletingPortManager*)para ;
    cpm->ListenThread() ;
}

//public
void CompletingPortManager::Start()
{
 if(this->isStop)
 {
  this->isStop = FALSE ;
  this->CreateWorkerThreads() ;

  Thread thread ;
  thread.Start(ListenThreadStart ,this) ;
  //unsigned int  dwThreadId;  
  //HANDLE hThread = (HANDLE)_beginthreadex(NULL, 0 , (unsigned int (_stdcall*)(void*))&ListenThreadStart , this, 0, &dwThreadId);   
 }
}

//public
void CompletingPortManager::Stop()
{
 if(this->isStop)
 {  
  return ;
 } 
 
 this->isStop = true ;
 Sleep(500) ; //等待监听线程结束 
    this->listener->Close() ;
     
 this->CloseAllWorkerThread() ;//等所有线程退出后才返回  
 this->context_manager->ClearAllKeys() ; //关闭所有连接
 this->CloseAllOverlapEvent() ;  
}

//public
int CompletingPortManager::ConnectionCount()
{
 return this->context_manager->GetConnectionCount() ;
}

//public
void CompletingPortManager::SendData(int ConnectID, byte* data ,int offset ,int size)
{
 ContextKey* key = this->context_manager->GetContextKey(ConnectID) ;
 if(key != NULL)
 {
  key->netStream->Write((char*)(data +offset) ,size) ;
 }
}

//public
void CompletingPortManager::DisposeOneConnection(int connectID ,DisconnectedCause cause)
{
 this->context_manager->ClearPkey(connectID) ;
 
 UserActionEventArgs args ;
 args.action    = UA_Disconnected ;
 args.ConnectID = connectID ;
 ((EventPublisher<ITcpManager* ,UserActionEventArgs&>*)(this->UserAction))->Invoke(this ,args) ;
 
}

//private
void CompletingPortManager::CloseAllWorkerThread()
{
 while(! this->workerThread_assistant.IsSafeToExit())//等待到达安全点
 {
  Sleep(200) ;  
 } 
}

//private
void CompletingPortManager::CloseAllOverlapEvent()
{
    int count = this->overlapEventList.Count() ;
 for(int i=0 ;i<count ;i++)
 {
  HANDLE handle ;
  this->overlapEventList.GetElement(i ,handle) ;
  CloseHandle(handle) ;
 }
 
 this->overlapEventList.Clear() ;
}

//private
void CompletingPortManager::CreateWorkerThreads()
{
    SYSTEM_INFO  sysinfo;  
    DWORD        dwThreads;
    DWORD        i;
   
    GetSystemInfo(&sysinfo);
    dwThreads = this->workThreadNum ; //sysinfo.dwNumberOfProcessors * 32 + 2;
    for (i=0; i<dwThreads; i++)
    {
  Thread thread ;
  thread.Start(WorkerThreadStart ,this) ;
       // HANDLE hThread;
       // hThread = (HANDLE)_beginthreadex(NULL, 0, (unsigned int (_stdcall*)(void*))&WorkerThreadStart , this, 0, &dwThreadId);
        workerThread_assistant.IncreaseWorkingThreadNum() ;
    }
}

//private
void CompletingPortManager::BindConnectionToCompPort(SOCKET connection ,ContextKey** out_pky)
{
    ContextKey* pkey = new ContextKey(connection) ;  
    *out_pky = pkey ;
   
    CreateIoCompletionPort((HANDLE)connection ,this->completing_port ,(DWORD)pkey ,0);   
}

//public
void CompletingPortManager::ListenThread()
{
 if(! this->isStop)
 {
  return ;
 }
 
 SOCKET connection ;
 
 BOOL succeed = this->listener->Start() ;
 if(!succeed)
 { 
  MessageBox(NULL ,"无法启动监听线程 !" ,"Tip" ,0) ;  
  return ;
 } 
 
 ContextKey* pkey  ;    
 
 while((! this->isStop) && (this->listener->Pending()))
 { 
  connection = this->listener->AcceptSocket() ; //当关闭监听socket时，该函数也会返回
  if(this->isStop)
  {   
   closesocket(connection) ;  
   break ;
  }

  this->BindConnectionToCompPort(connection ,&pkey) ;
  this->context_manager->RegisterContextKey(pkey) ;

  UserActionEventArgs args ;
  args.action    = UA_Connected ;
  args.ConnectID = connection ;
  ((EventPublisher<ITcpManager* ,UserActionEventArgs&>*)(this->UserAction))->Invoke(this ,args) ;
  
  this->RecieveData(pkey) ;  
 }
}

//private 进行异步接收数据
void CompletingPortManager::RecieveData(ContextKey* pkey)
{
 if(this->context_manager->ContainsPkey(pkey))
 {
  DWORD Flags = 0;  
  pkey->wsa_buf.len = pkey->requestData.BufferSize() - pkey->requestData.LeftDataLength ;
  pkey->wsa_buf.buf = (char*)(pkey->requestData.Buff + pkey->requestData.LeftDataLength) ;
  ZeroMemory(&(pkey->ovlapped) ,sizeof(OVERLAPPED)) ;
  
  unsigned long numRead = 0 ;
  pkey->netStream->AsynRead(pkey->wsa_buf ,&numRead ,&(pkey->ovlapped)) ;
 }
}

//private 工作者线程
void CompletingPortManager::ServeCmpltdOverlap()
{
 BOOL    bResult;
    int   dwNumRead;
    ContextKey *pkey;
    LPOVERLAPPED lpOverlapped;
    DWORD temp ;
   
    //回复客户
    RespondStreamList* respond_list = new RespondStreamList() ;
 RespondStream* respond_stream = NULL ;
 BOOL socket_closed = FALSE ;

    BOOL succeed = FALSE ;
   // DWORD bytes_send ;
   // WSABUF wsabuff_send ;
    OVERLAPPED ovlapped_send ;
    memset(&ovlapped_send ,0 ,sizeof(OVERLAPPED)) ;
    ovlapped_send.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); 
    this->overlapEventList.Add(ovlapped_send.hEvent) ;
    ovlapped_send.hEvent = (HANDLE)((DWORD)ovlapped_send.hEvent | 0x1);//表示不关心异步结果,避免Completion Packets
   
    while(! this->isStop)
    {
        bResult = GetQueuedCompletionStatus(  //在底层由系统提供高效的同步机制
            this->completing_port,
            (DWORD*)&dwNumRead,
            &temp,
            &lpOverlapped,
            10
            );  

   if(bResult == 0)
  {
   DWORD rc = GetLastError() ;
   
   if((this->isStop) &&(rc == WAIT_TIMEOUT))
   {
    break ;
   }
   else
   {
    continue ;
   }
  }        
       
        pkey = (ContextKey *)temp ;
  socket_closed = FALSE  ;
       
        if ((bResult == FALSE) || (dwNumRead == 0)) //当客户端正常关闭socket或掉线时，异步WSARecv调用会向队列投递一个特殊的Completion Packets－并且读到的字节长度为0
        {   
   socket_closed = TRUE ;
     }      
        else// Got a valid data block! 处理数据
        {
   try
   {

    pkey->requestData.ValidCount = dwNumRead + pkey->requestData.LeftDataLength ;  
    
    succeed = this->curDispatcher->DealRequestMessage(pkey->requestData ,respond_list) ;   
  
    if(succeed)
    {
     int count = respond_list->Count() ;
     for(int i=0 ;i<count ;i++)
     {
      respond_stream = respond_list->GetElement(i) ;          
      
      //同步发送
      BOOL sendSucceed = pkey->netStream->Write(respond_stream->data ,respond_stream->length) ;
      if(! sendSucceed)
      {
       socket_closed = TRUE ;
       break ;
      }
      else
      {
       ServiceCommittedEventArgs arg ;
       arg.connectID = pkey->netStream->SocketID() ;
       arg.data   = respond_stream->data ;
       arg.length   = respond_stream->length ;

       ((EventPublisher<ITcpManager* ,ServiceCommittedEventArgs&>*)this->ServiceCommitted)->Invoke(this,arg) ;
       
       UserActionEventArgs actArgs ;
       actArgs.action    = UA_Connected ;
       actArgs.ConnectID = pkey->netStream->SocketID() ;
       ((EventPublisher<ITcpManager* ,UserActionEventArgs&>*)this->UserAction)->Invoke(this ,actArgs);
      }
      
      respond_stream = NULL ;     
     }       
    } 
    else
    {
     socket_closed = TRUE ;
    }
   }
   catch(...)//(ContextKey* )
   {
    socket_closed = TRUE ;    
   }

   if(socket_closed)
   {
    this->DisposeOneConnection(pkey->netStream->SocketID() ,LineOff) ;
   }

   this->ClearRespondStreamList(respond_list) ; //清空并删除所有的RespondStream

   if((! this->isStop) && (!socket_closed))
   {  
    if(pkey->requestData.IsFirstMsg)
    {
     pkey->requestData.IsFirstMsg = false ;
    }
    
    this->RecieveData(pkey) ;//继续从socket接收消息
   }      
        }
    }

 delete respond_list ;

    this->workerThread_assistant.DecreaseWorkingThreadNum() ;    //工作线程正常退出 
}

//private
void CompletingPortManager::ClearRespondStreamList(RespondStreamList* rs_list)
{
 RespondStream* respond_stream = NULL ;
 int count = rs_list->Count() ;
 for(int i=0 ;i<count ;i++)
 {
  respond_stream = rs_list->GetElement(i) ;
  respond_stream->ClearAll() ;
  delete respond_stream ; 
  respond_stream = NULL ;
 }

 rs_list->Clear() ;
}

当然，还有很多的基本设施没有包含进来，所以如果你打算使用我的这个完成端口组件可email至[email protected]索取所有相关的源码。