[置顶] zeromq源代码分析2------线/进程间通信方式
本文我们讲一下zeromq的线/进程间通信方式。
在zeromq源代码分析1中我们分析了zeromq的基本工作流程。我们了解到了zeromq的线/进程间通信的方式为消息传递。
zeromq中的各线程间通信的消息称之为command_t类型:
// This structure defines the commands that can be sent between threads.
struct command_t
{
// Object to process the command.
class object_t *destination;
enum type_t
{
stop,
plug,
own,
attach,
bind,
activate_reader,
activate_writer,
pipe_term,
pipe_term_ack,
term_req,
term,
term_ack,
reap,
reaped,
done
} type;
union {
// Sent to I/O thread to let it know that it should
// terminate itself.
struct {
} stop;
// Sent to I/O object to make it register with its I/O thread.
struct {
} plug;
// Sent to socket to let it know about the newly created object.
struct {
class own_t *object;
} own;
// Attach the engine to the session. If engine is NULL, it informs
// session that the connection have failed.
struct {
struct i_engine *engine;
unsigned char peer_identity_size;
unsigned char *peer_identity;
} attach;
// Sent from session to socket to establish pipe(s) between them.
// Caller have used inc_seqnum beforehand sending the command.
struct {
class reader_t *in_pipe;
class writer_t *out_pipe;
unsigned char peer_identity_size;
unsigned char *peer_identity;
} bind;
// Sent by pipe writer to inform dormant pipe reader that there
// are messages in the pipe.
struct {
} activate_reader;
// Sent by pipe reader to inform pipe writer about how many
// messages it has read so far.
struct {
uint64_t msgs_read;
} activate_writer;
// Sent by pipe reader to pipe writer to ask it to terminate
// its end of the pipe.
struct {
} pipe_term;
// Pipe writer acknowledges pipe_term command.
struct {
} pipe_term_ack;
// Sent by I/O object ot the socket to request the shutdown of
// the I/O object.
struct {
class own_t *object;
} term_req;
// Sent by socket to I/O object to start its shutdown.
struct {
int linger;
} term;
// Sent by I/O object to the socket to acknowledge it has
// shut down.
struct {
} term_ack;
// Transfers the ownership of the closed socket
// to the reaper thread.
struct {
class socket_base_t *socket;
} reap;
// Closed socket notifies the reaper that it's already deallocated.
struct {
} reaped;
// Sent by reaper thread to the term thread when all the sockets
// are successfully deallocated.
struct {
} done;
} args;
};
由此可见一个command_t主要包括destination，type，args，其中destination表明要传递命令消息的目的对象，type和args则是命令消息的类型和参数。
而发送消息的代码如下:
// Derived object can use these functions to send commands
// to other objects.
void send_stop ();
void send_plug (class own_t *destination_,
bool inc_seqnum_ = true);
void send_own (class own_t *destination_,
class own_t *object_);
void send_attach (class session_t *destination_,
struct i_engine *engine_, const blob_t &peer_identity_,
bool inc_seqnum_ = true);
void send_bind (class own_t *destination_,
class reader_t *in_pipe_, class writer_t *out_pipe_,
const blob_t &peer_identity_, bool inc_seqnum_ = true);
void send_activate_reader (class reader_t *destination_);
void send_activate_writer (class writer_t *destination_,
uint64_t msgs_read_);
void send_pipe_term (class writer_t *destination_);
void send_pipe_term_ack (class reader_t *destination_);
void send_term_req (class own_t *destination_,
class own_t *object_);
void send_term (class own_t *destination_, int linger_);
void send_term_ack (class own_t *destination_);
void send_reap (class socket_base_t *socket_);
void send_reaped ();
void send_done ();
void zmq::object_t::send_command (command_t &cmd_)
{
ctx->send_command (cmd_.destination->get_tid (), cmd_);
}
void zmq::ctx_t::send_command (uint32_t tid_, const command_t &command_)
{
slots [tid_]->send (command_);
}
因为zeromq采用的模式基本上就是一个object和一个io_thread绑定起来，所以这里的意思就是我们将command发送给目的对象的io_thread的mailbox中去。
如果对ctx::slots不太懂，请回到zeromq源代码分析1并且结合源代码理解。
下面我们对mailbox来一个深入分析:
zmq::mailbox_t::mailbox_t ()
{
#ifdef PIPE_BUF
// Make sure that command can be written to the socket in atomic fashion.
// If this wasn't guaranteed, commands from different threads would be
// interleaved.
zmq_assert (sizeof (command_t) <= PIPE_BUF);
#endif
// Create the socketpair for signaling.
int rc = make_socketpair (&r, &w);
errno_assert (rc == 0);
// Set the writer to non-blocking mode.
int flags = fcntl (w, F_GETFL, 0);
errno_assert (flags >= 0);
rc = fcntl (w, F_SETFL, flags | O_NONBLOCK);
errno_assert (rc == 0);
#ifndef MSG_DONTWAIT
// Set the reader to non-blocking mode.
flags = fcntl (r, F_GETFL, 0);
errno_assert (flags >= 0);
rc = fcntl (r, F_SETFL, flags | O_NONBLOCK);
errno_assert (rc == 0);
#endif
}
该函数创建了socketpair，并且设置了writer和reader都为非阻塞模式。
而socketpair在UNIX上是有原生的实现的，在windows下面通过自己创建一个一对socket连接，客户端当reader，服务端当writer。
void zmq::mailbox_t::send (const command_t &cmd_)
{
// Attempt to write an entire command without blocking.
ssize_t nbytes;
do {
nbytes = ::send (w, &cmd_, sizeof (command_t), 0);
} while (nbytes == -1 && errno == EINTR);
// Attempt to increase mailbox SNDBUF if the send failed.
if (nbytes == -1 && errno == EAGAIN) {
int old_sndbuf, new_sndbuf;
socklen_t sndbuf_size = sizeof old_sndbuf;
// Retrieve current send buffer size.
int rc = getsockopt (w, SOL_SOCKET, SO_SNDBUF, &old_sndbuf,
&sndbuf_size);
errno_assert (rc == 0);
new_sndbuf = old_sndbuf * 2;
// Double the new send buffer size.
rc = setsockopt (w, SOL_SOCKET, SO_SNDBUF, &new_sndbuf, sndbuf_size);
errno_assert (rc == 0);
// Verify that the OS actually honored the request.
rc = getsockopt (w, SOL_SOCKET, SO_SNDBUF, &new_sndbuf, &sndbuf_size);
errno_assert (rc == 0);
zmq_assert (new_sndbuf > old_sndbuf);
// Retry the sending operation; at this point it must succeed.
do {
nbytes = ::send (w, &cmd_, sizeof (command_t), 0);
} while (nbytes == -1 && errno == EINTR);
}
errno_assert (nbytes != -1);
// This should never happen as we've already checked that command size is
// less than PIPE_BUF.
zmq_assert (nbytes == sizeof (command_t));
}
发送command，这边利用writer句柄调用send(4) syscall进行发送command。先尝试写，如果被中断则继续循环写。如果返回错误EAGAIN，由于是非阻塞模式写我们就会增加SND_BUF继续写，最后验证一下是否成功。
int zmq::mailbox_t::recv (command_t *cmd_, bool block_)
{
#ifdef MSG_DONTWAIT
// Attempt to read an entire command. Returns EAGAIN if non-blocking
// mode is requested and a command is not available.
ssize_t nbytes = ::recv (r, cmd_, sizeof (command_t),
block_ ? 0 : MSG_DONTWAIT);
if (nbytes == -1 && (errno == EAGAIN || errno == EINTR))
return -1;
#else
// If required, set the reader to blocking mode.
if (block_) {
int flags = fcntl (r, F_GETFL, 0);
errno_assert (flags >= 0);
int rc = fcntl (r, F_SETFL, flags & ~O_NONBLOCK);
errno_assert (rc == 0);
}
// Attempt to read an entire command. Returns EAGAIN if non-blocking
// and a command is not available. Save value of errno if we wish to pass
// it to caller.
int err = 0;
ssize_t nbytes = ::recv (r, cmd_, sizeof (command_t), 0);
if (nbytes == -1 && (errno == EAGAIN || errno == EINTR))
err = errno;
// Re-set the reader to non-blocking mode.
if (block_) {
int flags = fcntl (r, F_GETFL, 0);
errno_assert (flags >= 0);
int rc = fcntl (r, F_SETFL, flags | O_NONBLOCK);
errno_assert (rc == 0);
}
// If the recv failed, return with the saved errno if set.
if (err != 0) {
errno = err;
return -1;
}
#endif
// Sanity check for success.
errno_assert (nbytes != -1);
// Check whether we haven't got half of command.
zmq_assert (nbytes == sizeof (command_t));
return 0;
接收command则根据参数block_可以选择是否阻塞地读。这边之所以可以直接使用
ssize_t nbytes = ::recv (r, cmd_, sizeof (command_t), 0);是因为对于线程间的通信木有字节序不同的问题。
下面我们来讲一下接收的函数调用过程。
由于我们在zeromq源代码分析1中已经说明io_thread在构造函数中会将自己的mailbox的句柄加入到poller中，设置事件处理器就是自身的函数in_event()/out_event()，并且激活读事件。因此当poller轮询到mailbox的读事件，也就是有人发消息给该io_thread的mailbox时候，就能通过in_event()函数去recv消息。
void zmq::io_thread_t::in_event ()
{
// TODO: Do we want to limit number of commands I/O thread can
// process in a single go?
while (true) {
// Get the next command. If there is none, exit.
command_t cmd;
int rc = mailbox.recv (&cmd, false);
if (rc != 0 && errno == EINTR)
continue;
if (rc != 0 && errno == EAGAIN)
break;
errno_assert (rc == 0);
// Process the command.
cmd.destination->process_command (cmd);
}
}
这边收到command之后我们处理该command，即调用cmd的destination的process_command(1)函数，也就是与该io_thread绑定的object。
// These handlers can be overloaded by the derived objects. They are
// called when command arrives from another thread.
virtual void process_stop ();
virtual void process_plug ();
virtual void process_own (class own_t *object_);
virtual void process_attach (struct i_engine *engine_,
const blob_t &peer_identity_);
virtual void process_bind (class reader_t *in_pipe_,
class writer_t *out_pipe_, const blob_t &peer_identity_);
virtual void process_activate_reader ();
virtual void process_activate_writer (uint64_t msgs_read_);
virtual void process_pipe_term ();
virtual void process_pipe_term_ack ();
virtual void process_term_req (class own_t *object_);
virtual void process_term (int linger_);
virtual void process_term_ack ();
virtual void process_reap (class socket_base_t *socket_);
virtual void process_reaped ();
void zmq::object_t::process_command (command_t &cmd_)
{
switch (cmd_.type) {
case command_t::activate_reader:
process_activate_reader ();
break;
case command_t::activate_writer:
process_activate_writer (cmd_.args.activate_writer.msgs_read);
break;
case command_t::stop:
process_stop ();
break;
case command_t::plug:
process_plug ();
process_seqnum ();
return;
...
这些process_xxx()函数在子类中override，处理各种线程间发送的命令。
下一篇文章会介绍zmq_msg_t以及相关的函数，敬请期待。希望有兴趣的朋友可以和我联系，一起学习。 [email protected]