zmq源码阅读笔记之网络消息与命令
mailbox
mailbox是线程之间用于收发命令的类定义如下
class mailbox_t : public i_mailbox
{
public:
mailbox_t ();
~mailbox_t ();
fd_t get_fd () const;
void send (const command_t &cmd_);
int recv (command_t *cmd_, int timeout_);
#ifdef HAVE_FORK
// close the file descriptors in the signaller. This is used in a forked
// child process to close the file descriptors so that they do not interfere
// with the context in the parent process.
void forked () { signaler.forked (); }
#endif
private:
// The pipe to store actual commands.
typedef ypipe_t cpipe_t;
cpipe_t cpipe;
// Signaler to pass signals from writer thread to reader thread.
signaler_t signaler;
// There's only one thread receiving from the mailbox, but there
// is arbitrary number of threads sending. Given that ypipe requires
// synchronised access on both of its endpoints, we have to synchronise
// the sending side.
mutex_t sync;
// True if the underlying pipe is active, ie. when we are allowed to
// read commands from it.
bool active;
// Disable copying of mailbox_t object.
mailbox_t (const mailbox_t&);
const mailbox_t &operator = (const mailbox_t&);
};
}
#endif mailbox中有一个signaler_t类型的signaler和一个mutex_t类型的sync，sync用于多线程同时写入的互斥，signaler用于通知接受法已经有命令到来
。mutex_t的实现就是根据操作系统封装了线程锁，signaler_t的底层实现本质是一个socketpair。maibox内部通过一个ypipe来存储命令，每个命令是command_t类型定义如下，可以看到每个命令分为三个部分，一个是目的地destination，类型type，参数args
struct command_t
{
// Object to process the command.
zmq::object_t *destination;
enum type_t
{
...
} type;
union {
...
} args;
};一个对象如果想收发命令，那么就需要继承object_t对象，主要定义如下
class object_t
{
public:
object_t (zmq::ctx_t *ctx_, uint32_t tid_);
object_t (object_t *parent_);
virtual ~object_t ();
uint32_t get_tid ();
void set_tid(uint32_t id);
ctx_t *get_ctx ();
void process_command (zmq::command_t &cmd_);
void send_inproc_connected (zmq::socket_base_t *socket_);
void send_bind (zmq::own_t *destination_, zmq::pipe_t *pipe_, bool inc_seqnum_ = true);
// Chooses least loaded I/O thread.
zmq::io_thread_t *choose_io_thread (uint64_t affinity_);
....
virtual void process_seqnum ();
private:
// Context provides access to the global state.
zmq::ctx_t *ctx;
// Thread ID of the thread the object belongs to.
uint32_t tid;
void send_command (command_t &cmd_);
object_t (const object_t&);
const object_t &operator = (const object_t&);
};
}成员变量中有一个ctx_t和一个tid对象，ctx_t是zmq的上下文环境对象，实现在ctx.hpp/cpp中，里面会存储当前线程中的mailbox，通过tid可以找到对应的mailbox在slots的位置，从而写入命令
ctx.hpp
// I/O threads.
typedef std::vector io_threads_t;
io_threads_t io_threads;
// Array of pointers to mailboxes for both application and I/O threads.
uint32_t slot_count;
i_mailbox **slots;
// Mailbox for zmq_ctx_term thread.
mailbox_t term_mailbox;
2，poller
zmq封装了poll，epoll，select等常用的IO复用，统一为poller轮询器，根据不同的操作系统选择对应的IO复用。
#ifndef __ZMQ_POLLER_HPP_INCLUDED__
#define __ZMQ_POLLER_HPP_INCLUDED__
#if defined ZMQ_USE_KQUEUE + defined ZMQ_USE_EPOLL \
+ defined ZMQ_USE_DEVPOLL + defined ZMQ_USE_POLLSET \
+ defined ZMQ_USE_POLL + defined ZMQ_USE_SELECT > 1
#error More than one of the ZMQ_USE_* macros defined
#endif
#if defined ZMQ_USE_KQUEUE
# include "kqueue.hpp"
#elif defined ZMQ_USE_EPOLL
# include "epoll.hpp"
#elif defined ZMQ_USE_DEVPOLL
# include "devpoll.hpp"
#elif defined ZMQ_USE_POLLSET
# include "pollset.hpp"
#elif defined ZMQ_USE_POLL
# include "poll.hpp"
#elif defined ZMQ_USE_SELECT
# include "select.hpp"
#elif defined ZMQ_HAVE_GNU
# define ZMQ_USE_POLL
# include "poll.hpp"
#else
# error None of the ZMQ_USE_* macros defined
#endif
select.hpp,poll.hpp,epoll.hpp分别封装了不同的IO复用，所有的IO复用都会继承一个Poller_base类
namespace zmq
{
struct i_poll_events;
class poller_base_t
{
public:
poller_base_t ();
virtual ~poller_base_t ();
// Returns load of the poller. Note that this function can be
// invoked from a different thread!
int get_load ();
// Add a timeout to expire in timeout_ milliseconds. After the
// expiration timer_event on sink_ object will be called with
// argument set to id_.
void add_timer (int timeout_, zmq::i_poll_events *sink_, int id_);
// Cancel the timer created by sink_ object with ID equal to id_.
void cancel_timer (zmq::i_poll_events *sink_, int id_);
protected:
// Called by individual poller implementations to manage the load.
void adjust_load (int amount_);
// Executes any timers that are due. Returns number of milliseconds
// to wait to match the next timer or 0 meaning "no timers".
uint64_t execute_timers ();
private:
// Clock instance private to this I/O thread.
clock_t clock;
// List of active timers.
struct timer_info_t
{
zmq::i_poll_events *sink;
int id;
};
typedef std::multimap timers_t;
timers_t timers;
// Load of the poller. Currently the number of file descriptors
// registered.
atomic_counter_t load;
poller_base_t (const poller_base_t&);
const poller_base_t &operator = (const poller_base_t&);
};
} poller_base_t主要用于封装定时器，里面用个multimap装载所有的定时器，方便遍历，实际使用中，poller会根据不同的实现去调用不同的IO复用类。以epoll为例通过add_fd添加fd和注册事件,在loop中循环监听注册的fd，是否有数据poll in和poll out，
// "poller" concept.
handle_t add_fd (fd_t fd_, zmq::i_poll_events *events_);
void rm_fd (handle_t handle_);
void set_pollin (handle_t handle_);
void reset_pollin (handle_t handle_);
void set_pollout (handle_t handle_);
void reset_pollout (handle_t handle_);
void start ();
void stop ();epoll_t里有个 thread_t类型的工作线程成员，thread_t的实现在thread.hpp和thread.cpp里linux下用pthread库封装，windows下是用windows的原生线程库。epoll_t中海定义了一个poll_entry_t结构体,用来存储fd和event，之后其他函数的操作都是用handle_t来操作
typedef void* handle_t;
struct poll_entry_t
{
fd_t fd;
epoll_event ev;
zmq::i_poll_events *events;
};
// List of retired event sources.
typedef std::vector retired_t;
retired_t retired; 每一个io线程(io_thread_t)都包含一个mailbox和一个poller
class io_thread_t : public object_t, public i_poll_events
{
public:
io_thread_t (zmq::ctx_t *ctx_, uint32_t tid_);
// Clean-up. If the thread was started, it's necessary to call 'stop'
// before invoking destructor. Otherwise the destructor would hang up.
~io_thread_t ();
// Launch the physical thread.
void start ();
// Ask underlying thread to stop.
void stop ();
// Returns mailbox associated with this I/O thread.
mailbox_t *get_mailbox ();
// i_poll_events implementation.
void in_event ();
void out_event ();
void timer_event (int id_);
// Used by io_objects to retrieve the associated poller object.
poller_t *get_poller ();
// Command handlers.
void process_stop ();
// Returns load experienced by the I/O thread.
int get_load ();
private:
// I/O thread accesses incoming commands via this mailbox.
mailbox_t mailbox;
// Handle associated with mailbox' file descriptor.
poller_t::handle_t mailbox_handle;
// I/O multiplexing is performed using a poller object.
poller_t *poller;
io_thread_t (const io_thread_t&);
const io_thread_t &operator = (const io_thread_t&);
};
构造函数中把mailbox_t句柄加入poller中，让poller监听其读事件，所以，如果有信号发过来，poller会被唤醒，并调用io_thread_t的in_event：