ps-lite源码分析: include/ps/internal/van.h

Van是核心通信类，每个节点只有一个该对象，是Postoffice对象的成员。它负责建立起节点之间的互相连接（例如Worker与Scheduler之间的连接），并且开启本地的receiving thread用来监听收到的message。它发送消息至远程节点，如果环境变量PS_RESEND=1，并且在PS_RESEND_TIMEOUT微秒内没有收到ACK消息，那么它会重发消息。Van的子类为ZMQVan，即为用zmq库实现了连接的底层细节（zmq库是一个开源库，对socket进行了优良的封装）。

启动Van后，首先调用Start，初始化到其它节点的所有连接；启动接收线程，接收消息。如果接收到系统控制消息，传给postoffice::manager，否则传给对应的应用。

/**
 *  Copyright (c) 2015 by Contributors
 */
#ifndef PS_INTERNAL_VAN_H_
#define PS_INTERNAL_VAN_H_
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include "ps/base.h"
#include "ps/internal/message.h"
namespace ps {

class Resender;
class PBMeta;


/**
 * \brief Van sends messages to remote nodes
 *
 * If environment variable PS_RESEND is set to be 1, then van will resend a
 * message if it no ACK messsage is received within PS_RESEND_TIMEOUT
 * millisecond
 */
class Van {
 public:
  /** 
  * 创建Van，延迟到具体子类实现 
  * type 表示Van的具体实现类型如zmq，socket等
 */
  /**
   * \brief create Van
   * \param type zmq, socket, ...
   */
  static Van *Create(const std::string &type);

  /** \brief constructer, do nothing. use \ref Start for real start */
  Van() {}

  /**\brief deconstructer, do nothing. use \ref Stop for real stop */
  virtual ~Van() {}

  /**
 * 启动Van，必须在调用Send之前调用Start, 初始化到其它节点的所有连接。启动接收线程，一直接收消息。
 * 如果接收到系统控制消息，传给postoffice::manager，否则传给对应的应用。
 *
 */
  /**
   * \brief start van
   *
   * must call it before calling Send
   *
   * it initalizes all connections to other nodes.  start the receiving
   * threads, which keeps receiving messages. if it is a system
   * control message, give it to postoffice::manager, otherwise, give it to the
   * accoding app.
   */
  virtual void Start(int customer_id);

  /** 线程安全，发送消息，返回发送的字节数量，-1表示失败 */
  /**
   * \brief send a message, It is thread-safe
   * \return the number of bytes sent. -1 if failed
   */
  int Send(const Message &msg);

  /** 返回节点 */
  /**
   * \brief return my node
   */
  inline const Node &my_node() const {
    CHECK(ready_) << "call Start() first";
    return my_node_;
  }

  /** 停止接收线程 */
  /**
   * \brief stop van
   * stop receiving threads
   */
  virtual void Stop();

  /** 线程安全，获取下一个可用时间戳 */
  /**
   * \brief get next available timestamp. thread safe
   */
  inline int GetTimestamp() { return timestamp_++; }

  /** 线程安全，是否就绪待发送 */
  /**
   * \brief whether it is ready for sending. thread safe
   */
  inline bool IsReady() { return ready_; }

 protected:
  /** 连接到节点 */
  /**
   * \brief connect to a node
   */
  virtual void Connect(const Node &node) = 0;

 /** 
 * 绑定到节点，在绑定端口上进行多次尝试。因为相同机器上的不同节点可能选取相同端口 
 * 返回绑定的端口，失败返回-1
 */
  /**
   * \brief bind to my node
   * do multiple retries on binding the port. since it's possible that
   * different nodes on the same machine picked the same port
   * \return return the port binded, -1 if failed.
   */
  virtual int Bind(const Node &node, int max_retry) = 0;

  /** 阻塞直至收到消息，返回接收的字节数量，失败或超时返回-1 */
  /**
   * \brief block until received a message
   * \return the number of bytes received. -1 if failed or timeout
   */
  virtual int RecvMsg(Message *msg) = 0;

 /** 发送消息，返回发送的字节数量 */
  /**
   * \brief send a mesage
   * \return the number of bytes sent
   */
  virtual int SendMsg(const Message &msg) = 0;

  /** 打包meta数据成字符串 */
  /**
   * \brief pack meta into a string
   */
  void PackMeta(const Meta &meta, char **meta_buf, int *buf_size);

 /** 打包meta成protobuf */
  /**
   * \brief pack meta into protobuf
   */
  void PackMetaPB(const Meta &meta, PBMeta *pb);

  /** 从字符串中拆包meta */
  /**
   * \brief unpack meta from a string
   */
  void UnpackMeta(const char *meta_buf, int buf_size, Meta *meta);

  /** 调度节点 */
  Node scheduler_;
  /** 当前节点 */
  Node my_node_;
 /** 当前节点是否是调度节点 */
  bool is_scheduler_;
 /** 互斥锁用以Start */
  std::mutex start_mu_;

 private:
  /** 线程接收函数*/
  /** thread function for receving */
  void Receiving();

 /** 线程心跳函数 */
  /** thread function for heartbeat */
  void Heartbeat();
  
  /** 
  * 连接的节点表：节点地址字符串ip:port与节点ID映射表 
  * 当第一次收到ip:port时更新连接的节点表
 */
  // node's address string (i.e. ip:port) -> node id
  // this map is updated when ip:port is received for the first time
  std::unordered_map connected_nodes_;
 
  /**
  * 映射后来增加的节点的ID到第一次增加具有相同ip:port的节点的ID  
  */
  // maps the id of node which is added later to the id of node
  // which is with the same ip:port and added first
  std::unordered_map shared_node_mapping_;

  /** 是否就绪待发送 */
  /** whether it is ready for sending */
  std::atomic ready_{false};
  /** 发送字节数 */
  std::atomic send_bytes_{0};
  /** 接收字节数 */
  size_t recv_bytes_ = 0;
  /** server数量 */
  int num_servers_ = 0;
  /** worker数量 */
  int num_workers_ = 0;
  /** 接收消息的线程表 */
  /** the thread for receiving messages */
  std::unique_ptr receiver_thread_;
  /** 发送心跳的线程表 */
  /** the thread for sending heartbeat */
  std::unique_ptr heartbeat_thread_;
  /** barrier数量 */
  std::vector barrier_count_;
  /** 重新发送消息器 */
  /** msg resender */
  Resender *resender_ = nullptr;
  /** 丢失率 */
  int drop_rate_ = 0;
  /** 时间戳 */
  std::atomic timestamp_{0};
  int init_stage = 0;

  /** 为调度器处理AddNode控制消息的逻辑 */
  /**
   * \brief processing logic of AddNode message for scheduler
   */
  void ProcessAddNodeCommandAtScheduler(Message *msg, Meta *nodes,
                                        Meta *recovery_nodes);
  
  /** 处理Terminate控制消息的逻辑 */
  /**
   * \brief processing logic of Terminate message
   */
  void ProcessTerminateCommand();

  /** 运行于每个节点上，处理AddNode控制消息的逻辑 */
  /**
   * \brief processing logic of AddNode message (run on each node)
   */
  void ProcessAddNodeCommand(Message *msg, Meta *nodes, Meta *recovery_nodes);

  /** 运行于每个节点上，处理Barrier控制消息的逻辑 */
  /**
   * \brief processing logic of Barrier message (run on each node)
   */
  void ProcessBarrierCommand(Message *msg);

  /** 运行于每个节点上，处理心跳消息的逻辑 */
  /**
   * \brief processing logic of AddNode message (run on each node)
   */
  void ProcessHearbeat(Message *msg);

 /** 处理数据消息的逻辑 */
  /**
   * \brief processing logic of Data message
   */
  void ProcessDataMsg(Message *msg);

  /**
  *  由AddNode消息处理逻辑调用，在调度器上运行时，它对新增加的节点分配ID；
  *  在其它节点上，它使用从调度器接收的信息来更新节点ID 
  */ 
  /**
   * \brief called by ProcessAddNodeCommand, in scheduler it assigns an id to
   * the newly added node; in other nodes, it updates the node id with what is
   * received from scheduler
   */
  void UpdateLocalID(Message *msg, std::unordered_set *deadnodes_set,
                     Meta *nodes, Meta *recovery_nodes);

  /** 心跳超时值 */
  const char *heartbeat_timeout_val =
      Environment::Get()->find("PS_HEARTBEAT_TIMEOUT");
  int heartbeat_timeout_ =
      heartbeat_timeout_val ? atoi(heartbeat_timeout_val) : 0;
 
  /** 禁用拷贝赋值构造函数 */
  DISALLOW_COPY_AND_ASSIGN(Van);
};
}  // namespace ps
#endif  // PS_INTERNAL_VAN_H_