Linux下C++ EPOLL TCP服务器的设计与实现

背景简介：

       基于TCP协议的应用网络服务器程序开发一般对吞吐量和实时性有较高的要求，早期的Linux下实现一般采用I/O多路复用技术，比如使用select,aio等等。为了满足日益高涨的网络应用需求，Linux 自从2.6内核版本后实现了一种性能更高的I/O多路复用技术-EPOLL，与传统的POLL技术不同的是：使用接口API更简单（总共就三个函数），可控制的并发数更大（取决于可用的内存），稳定性更强(异步I/O，伸缩灵活)。

开发环境：

       开发平台：云服务器 CentOS 7.3 64bit， 2核CPU，4G内存

       编译器：   gcc

设计目标:

       实现并发10240个客户端连接交互业务数据

       实现WEBSOCKET协议（我的项目中服务对象是微信小程序的websocket客户端）

核心代码：

Server.h

#pragma once

#include
#include
#include
#include
#include
#include
#include
#include
#include
#include

#include "ace/Task.h"
#include "ace/Synch.h"
#include "MLog.h"
#include "WebSocketManager.h"

#define MAX_EPOLL_EVENT 1024

class CWebSocketServer : public ACE_Task
{
public:
    CWebSocketServer(unsigned short wServerPort, CMLog & mLog, CClientManager & clientMgr, CLoginServer & loginServer);
    virtual ~CWebSocketServer();
 
    bool Start();
    void Stop();
 
protected:
    virtual int open();
    virtual int svc();
 
    int  SetNonBlock(int nFd);
    bool OnNewConnection(int nNewSocket);
 void OnCloseConnection(int nFd);
    bool OnDataReceived(int nFd, const char * pszDataRecved, size_t nLen);
 
 
protected:
    unsigned short       m_wServerPort;
    int                  m_nServerSocket;
    int                  m_nEpollFd;
    struct epoll_event * m_pEpollEvents;
 
    //pthread_t            m_thrIdServer;
    bool                 m_bQuit;
    CMLog   &            m_mLog;
 
    char                 m_szRecvBuf[MAX_WS_RECV_BUF_SIZE];
    CWebSocketManager    m_webSocketMgr;
};

Server.cpp

#include "WebSocketServer.h"

CWebSocketServer::CWebSocketServer(unsigned short wServerPort, CMLog & mLog, CClientManager & clientMgr, CLoginServer& loginServer):m_mLog(mLog),m_webSocketMgr(MAX_WEBSOCKET_MEMPOOL_SIZE, mLog, clientMgr, loginServer)
{
 m_wServerPort = wServerPort;
 m_nServerSocket = -1;
 m_nEpollFd = -1;
 m_pEpollEvents = NULL;
 m_bQuit = false;
}

CWebSocketServer::~CWebSocketServer()
{
}

int CWebSocketServer::SetNonBlock(int nFd)
{
 int nFlags = fcntl(nFd, F_GETFL, 0);
 if (nFlags == -1)
 {
  return -1;
 }
 return fcntl(nFd, F_SETFL, nFlags | O_NONBLOCK);
}

bool CWebSocketServer::Start()
{
 if(!m_webSocketMgr.Start())
 {
  printf("m_webSocketMgr.Start() error...\n");
  return false;
 }
 
 open();
 return true;
}

void CWebSocketServer::Stop()
{
 m_bQuit = true;
 this->wait();
 m_webSocketMgr.Stop();
 m_mLog.Log("WebSocket网络服务线程即将退出...\n");
}

int CWebSocketServer::open()
{
 return this->activate();
}

bool CWebSocketServer::OnNewConnection(int nNewSocket)
{
 epoll_event e_evt;
 e_evt.events = EPOLLIN|EPOLLERR|EPOLLRDHUP|EPOLLET;
 e_evt.data.fd = nNewSocket;
 SetNonBlock(nNewSocket);

 if( epoll_ctl(m_nEpollFd, EPOLL_CTL_ADD, nNewSocket, &e_evt) == -1)
 {
         return false;
 }
 
 m_webSocketMgr.NewWebSocket(nNewSocket);
 return true;
}

bool CWebSocketServer::OnDataReceived(int nFd, const char * pszDataRecved, size_t nLen)
{
 return m_webSocketMgr.OnDataArrived(nFd, pszDataRecved, nLen);
}

void CWebSocketServer::OnCloseConnection(int nFd)
{
 struct epoll_event ev;
 ev.data.fd = nFd;
 ev.events = 0;
 epoll_ctl(m_nEpollFd, EPOLL_CTL_DEL, nFd, &ev);
 m_webSocketMgr.FreeWebSocket(nFd);
 shutdown(nFd, SHUT_RDWR);
 close(nFd);
}

int CWebSocketServer::svc()
{
 int nRet = 0;
 int nNewSocket = -1;
 struct sockaddr_in addrPeer;
 socklen_t addrLen = sizeof(addrPeer);
 ssize_t nRetLen = 0;
 size_t nRecvedLen = 0;
 
 m_nServerSocket = socket(AF_INET, SOCK_STREAM, 0);
 if(m_nServerSocket == -1)
 {
  printf("socket(AF_INET, SOCK_STREAM, 0) error...\n");
  return 1;
 }

        int nReusePort = 1;
        setsockopt(m_nServerSocket, SOL_SOCKET,SO_REUSEADDR,(const void *)&nReusePort, sizeof(nReusePort) ); 

    struct sockaddr_in addrServer;
    memset(&addrServer, 0, sizeof(sockaddr_in)); 
 addrServer.sin_family = AF_INET;
 addrServer.sin_addr.s_addr = htonl(INADDR_ANY);
 addrServer.sin_port = htons(m_wServerPort);
 
 if(bind(m_nServerSocket, (struct sockaddr *)&addrServer, sizeof(addrServer)) == -1)
 {
  close(m_nServerSocket);
  printf("bind(m_nServerSocket, (sockaddr*)&addrServer, sizeof(addrServer)) error...\n");
  return 2;
 }
 
 if(listen(m_nServerSocket, SOMAXCONN) == -1)
 {
  close(m_nServerSocket);
  m_mLog.Log("WebSocket网络服务线程监听端口失败，正在退出...\n");
  return 3;
 }
 
 struct epoll_event * m_pEpollEvents = new struct epoll_event[MAX_EPOLL_EVENT];
 if(NULL == m_pEpollEvents)
 {
  close(m_nServerSocket);
  printf("new struct epoll_event[MAX_EPOLL_EVENT]; error...\n");
  return 4;
 }
 
 m_nEpollFd = epoll_create(MAX_EPOLL_EVENT);
 if(m_nEpollFd == -1)
 {
  close(m_nServerSocket);
  delete []m_pEpollEvents;
  printf("epoll_create(MAX_EPOLL_EVENT); error...\n");
  return 5;
 }
 
 struct epoll_event e_evt;
 e_evt.events = EPOLLIN|EPOLLET;
 e_evt.data.fd = m_nServerSocket;
 if( epoll_ctl(m_nEpollFd, EPOLL_CTL_ADD, m_nServerSocket, &e_evt) == -1)
 {
     close(m_nServerSocket);
  delete []m_pEpollEvents;
  close(m_nEpollFd);
  m_mLog.Log("WebSocket网络服务线程执行epoll_ctl失败，正在退出...\n");
  return 6;
 }
 
 SetNonBlock(m_nServerSocket);
 
 m_mLog.Log("WebSocket网络服务线程启动成功...\n");
 

 while(!m_bQuit)
 {
  nRet = epoll_wait(m_nEpollFd, m_pEpollEvents, MAX_EPOLL_EVENT, 3000);
  if(nRet > 0)
  {
   //m_mLog.Log("Epoll探测到新事件...\n");
   for(int i=0; i    {
    if ((m_pEpollEvents[i].events & EPOLLERR) || (m_pEpollEvents[i].events & EPOLLHUP) || (!(m_pEpollEvents[i].events & EPOLLIN)))
    {
     m_mLog.Log("Epoll探测到错误事件...\n");
                    /* An error has occured on this fd, or the socket is not
                       ready for reading (why were we notified then?) */
     if(m_pEpollEvents[i].data.fd > 0)
     {
      OnCloseConnection(m_pEpollEvents[i].data.fd);
     }
     continue;
    }
    else if(m_pEpollEvents[i].data.fd == m_nServerSocket)//there are new connections
    {
     m_mLog.Log("Epoll探测到新连接...\n");
     while(1)//one or more connections need to be processed
     {
      addrLen = sizeof(addrPeer);
      nNewSocket = accept(m_pEpollEvents[i].data.fd, (sockaddr*)&addrPeer, &addrLen);
      if(nNewSocket != -1)
      {
       OnNewConnection(nNewSocket);
      }
      else//All new connections have been processed
      {
       break;
      }
     }
     m_mLog.Log("Epoll新连接已处理完毕...\n");
    }
    else if(m_pEpollEvents[i].events & EPOLLIN)//data arrived at the current socket
    {
     m_mLog.Log("Epoll探测到新数据...\n");
     nRecvedLen = 0;
     nRetLen = 0;
     while(1)
     {
           nRetLen = read(m_pEpollEvents[i].data.fd, m_szRecvBuf+nRecvedLen, MAX_WS_RECV_BUF_SIZE);
        if(nRetLen < 0 )
        {
          if(errno==EAGAIN)
       {
        break;
       }
       else if(errno==EINTR)
       {
        m_mLog.Log("调用read时遇到EINTR中断...\n");
        continue;
       }
       else
       {
        OnCloseConnection(m_pEpollEvents[i].data.fd);
        nRecvedLen = -1;
        break;
       }
        }
        else if(nRetLen > 0)
        {
             nRecvedLen += nRetLen;
        }
        else
        {
         OnCloseConnection(m_pEpollEvents[i].data.fd);
         nRecvedLen = -1;
            break;
        }
     }
     if(nRecvedLen > 0)
     {
      if(!OnDataReceived(m_pEpollEvents[i].data.fd, m_szRecvBuf, nRecvedLen))
      {
       OnCloseConnection(m_pEpollEvents[i].data.fd);
      }
     }
     m_mLog.Log("Epoll新数据已处理完毕...\n");
    }
   }
   
   m_webSocketMgr.CheckRequestCloseConnection(m_nEpollFd);
  }
  else
  {
   //m_mLog.Log("Epoll探测超时...\n");
   m_webSocketMgr.CheckRequestCloseConnection(m_nEpollFd);
  }
   
 }
 

 close(m_nServerSocket);
 close(m_nEpollFd);
 delete []m_pEpollEvents;
 return nRet;
}