nginx 父子进程通信 channel

转载地址：ttp://blog.csdn.net/benbendy1984/article/details/6023991

版权声明：本文为博主原创文章，未经博主允许不得转载。

        nginx 中利用socketpair()函数，创建一对相互连接的socket，实现父子进程的通信。 下面分析下整个过程：

 

一  创建

       在子进程创建之前，需要创建这样一对连接：  ngx_spawn_process函数(ngx_process.c中)

//创建一对匿名的已经连接的套接字
if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1)
{
    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                  "socketpair() failed while spawning /"%s/"", name);
    return NGX_INVALID_PID;
}
ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,
               "channel %d:%d",
               ngx_processes[s].channel[0],
               ngx_processes[s].channel[1]);
//设置成非阻塞模式
if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) {
    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                  ngx_nonblocking_n " failed while spawning /"%s/"",
                  name);
    ngx_close_channel(ngx_processes[s].channel, cycle->log);
    return NGX_INVALID_PID;
}
//设置成非阻塞模式
if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) {
    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                  ngx_nonblocking_n " failed while spawning /"%s/"",
                  name);
    ngx_close_channel(ngx_processes[s].channel, cycle->log);
    return NGX_INVALID_PID;
}
//打开异步模式
on = 1;
if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) {
    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                  "ioctl(FIOASYNC) failed while spawning /"%s/"", name);
    ngx_close_channel(ngx_processes[s].channel, cycle->log);
    return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) {
    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                  "fcntl(F_SETOWN) failed while spawning /"%s/"", name);
    ngx_close_channel(ngx_processes[s].channel, cycle->log);
    return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {
    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                  "fcntl(FD_CLOEXEC) failed while spawning /"%s/"",
                   name);
    ngx_close_channel(ngx_processes[s].channel, cycle->log);
    return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                  "fcntl(FD_CLOEXEC) failed while spawning /"%s/"",
                   name);
    ngx_close_channel(ngx_processes[s].channel, cycle->log);
    return NGX_INVALID_PID;
}
//设置即将创建子进程的channel ，这个在后面会用到
ngx_channel = ngx_processes[s].channel[1];

 

二 创建子进程

     接着会通过fork创建子进程，在子进程中，会调用ngx_worker_process_init 函数(ngx_process_cycle.c)

    //关闭所有其它进程对应的 channel[1] 和 自己的 channel[0]
    for (n = 0; n < ngx_last_process; n++) {
        if (ngx_processes[n].pid == -1) {
            continue;
        }
        if (n == ngx_process_slot) {
            continue;
        }
        if (ngx_processes[n].channel[1] == -1) {
            continue;
        }

        if (close(ngx_processes[n].channel[1]) == -1) {
            ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                          "close() channel failed");
        }
    }
    if (close(ngx_processes[ngx_process_slot].channel[0]) == -1) {
        ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                      "close() channel failed");
    }
#if 0
    ngx_last_process = 0;
#endif
    //调用epoll add 把ngx_chanel 加入epoll 中
    if (ngx_add_channel_event(cycle, ngx_channel, NGX_READ_EVENT,
                              ngx_channel_handler)
        == NGX_ERROR)
    {
        /* fatal */
        exit(2);
    }

 

channel[0] 是用来发送信息的，channel[1]是用来接收信息的。那么对自己而言，它需要向其他进程发送信息，需要保留其它进程的channel[0], 关闭channel[1]; 对自己而言，则需要关闭channel[0]。 最后把ngx_channel放到epoll中，从第一部分中的介绍我们可以知道，这个ngx_channel实际就是自己的 channel[1]。这样有信息进来的时候就可以通知到了。

 

三 实现子进程自己的通信

       子进程创建的时候，父进程的东西都会被子进程继承，所以后面创建的进程能够得到前面进程的channel信息，直接可以和他们通信，那么前面创建的进程如何知道后面的进程信息呢？ 很简单，既然前面创建的进程能够接受消息，那么我就发个信息告诉他后面的进程的channel,并把信息保存在channel[0]中，这样就可以相互通信了。

  ngx_pass_open_channel函数中实现(ngx_process_cycle.c)

 

static void
ngx_pass_open_channel(ngx_cycle_t *cycle, ngx_channel_t *ch)
{
    ngx_int_t  i;
    //遍历所有的进程
    for (i = 0; i < ngx_last_process; i++) {
        //不能是当前channel
        if (i == ngx_process_slot
            || ngx_processes[i].pid == -1
            || ngx_processes[i].channel[0] == -1)
        {
            continue;
        }
        ngx_log_debug6(NGX_LOG_DEBUG_CORE, cycle->log, 0,
                      "pass channel s:%d pid:%P fd:%d to s:%i pid:%P fd:%d",
                      ch->slot, ch->pid, ch->fd,
                      i, ngx_processes[i].pid,
                      ngx_processes[i].channel[0]);
        /* TODO: NGX_AGAIN */
        //向 每个进程 channel[0]发送信息
        ngx_write_channel(ngx_processes[i].channel[0],
                          ch, sizeof(ngx_channel_t), cycle->log);
    }
}

 

我们来看看 ngx_write_channel

 

ngx_int_t
ngx_write_channel(ngx_socket_t s, ngx_channel_t *ch, size_t size,
    ngx_log_t *log)
{
    ssize_t             n;
    ngx_err_t           err;
    struct iovec        iov[1];
    struct msghdr       msg;
#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
    union {
        struct cmsghdr  cm;
        char            space[CMSG_SPACE(sizeof(int))];
    } cmsg;
    if (ch->fd == -1) {
        msg.msg_control = NULL;
        msg.msg_controllen = 0;
    } else {
        msg.msg_control = (caddr_t) &cmsg;
        msg.msg_controllen = sizeof(cmsg);
        cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int));
        cmsg.cm.cmsg_level = SOL_SOCKET;
        cmsg.cm.cmsg_type = SCM_RIGHTS;
        /*
         * We have to use ngx_memcpy() instead of simple
         *   *(int *) CMSG_DATA(&cmsg.cm) = ch->fd;
         * because some gcc 4.4 with -O2/3/s optimization issues the warning:
         *   dereferencing type-punned pointer will break strict-aliasing rules
         *
         * Fortunately, gcc with -O1 compiles this ngx_memcpy()
         * in the same simple assigment as in the code above
         */
        //赋值信息到 cmsg.cm
        ngx_memcpy(CMSG_DATA(&cmsg.cm), &ch->fd, sizeof(int));
    }
    msg.msg_flags = 0;
#else
    if (ch->fd == -1) {
        msg.msg_accrights = NULL;
        msg.msg_accrightslen = 0;
    } else {
        msg.msg_accrights = (caddr_t) &ch->fd;
        msg.msg_accrightslen = sizeof(int);
    }
#endif
    iov[0].iov_base = (char *) ch;
    iov[0].iov_len = size;
    msg.msg_name = NULL;
    msg.msg_namelen = 0;
    msg.msg_iov = iov;
    msg.msg_iovlen = 1;
    n = sendmsg(s, &msg, 0);
    if (n == -1) {
        err = ngx_errno;
        if (err == NGX_EAGAIN) {
            return NGX_AGAIN;
        }
        ngx_log_error(NGX_LOG_ALERT, log, err, "sendmsg() failed");
        return NGX_ERROR;
    }
    return NGX_OK;
}

 

其实就是复制信息，在发送。我们再看看接受到信息的那段如何处理：ngx_chanel_read 函数

ngx_int_t
ngx_read_channel(ngx_socket_t s, ngx_channel_t *ch, size_t size, ngx_log_t *log)
{

    ssize_t             n;
    ngx_err_t           err;
    struct iovec        iov[1];
    struct msghdr       msg;
#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
    union {
        struct cmsghdr  cm;
        char            space[CMSG_SPACE(sizeof(int))];
    } cmsg;
#else
    int                 fd;
#endif
    iov[0].iov_base = (char *) ch;
    iov[0].iov_len = size;
    msg.msg_name = NULL;
    msg.msg_namelen = 0;
    msg.msg_iov = iov;
    msg.msg_iovlen = 1;
#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
    msg.msg_control = (caddr_t) &cmsg;
    msg.msg_controllen = sizeof(cmsg);
#else
    msg.msg_accrights = (caddr_t) &fd;
    msg.msg_accrightslen = sizeof(int);
#endif
    //接收消息
    n = recvmsg(s, &msg, 0);
    if (n == -1) {
        err = ngx_errno;
        if (err == NGX_EAGAIN) {
            return NGX_AGAIN;
        }
        ngx_log_error(NGX_LOG_ALERT, log, err, "recvmsg() failed");
        return NGX_ERROR;
    }
    if (n == 0) {
        ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "recvmsg() returned zero");
        return NGX_ERROR;
    }
    if ((size_t) n < sizeof(ngx_channel_t)) {
        ngx_log_error(NGX_LOG_ALERT, log, 0,
                      "recvmsg() returned not enough data: %uz", n);
        return NGX_ERROR;
    }
#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
    //
    if (ch->command == NGX_CMD_OPEN_CHANNEL) {
        if (cmsg.cm.cmsg_len < (socklen_t) CMSG_LEN(sizeof(int))) {
            ngx_log_error(NGX_LOG_ALERT, log, 0,
                          "recvmsg() returned too small ancillary data");
            return NGX_ERROR;
        }
        if (cmsg.cm.cmsg_level != SOL_SOCKET || cmsg.cm.cmsg_type != SCM_RIGHTS)
        {
            ngx_log_error(NGX_LOG_ALERT, log, 0,
                          "recvmsg() returned invalid ancillary data "
                          "level %d or type %d",
                          cmsg.cm.cmsg_level, cmsg.cm.cmsg_type);
            return NGX_ERROR;
        }
        /* ch->fd = *(int *) CMSG_DATA(&cmsg.cm); */
        //复制 fd
        ngx_memcpy(&ch->fd, CMSG_DATA(&cmsg.cm), sizeof(int));
    }
    if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) {
        ngx_log_error(NGX_LOG_ALERT, log, 0,
                      "recvmsg() truncated data");
    }
#else
    if (ch->command == NGX_CMD_OPEN_CHANNEL) {
        if (msg.msg_accrightslen != sizeof(int)) {
            ngx_log_error(NGX_LOG_ALERT, log, 0,
                          "recvmsg() returned no ancillary data");
            return NGX_ERROR;
        }
        ch->fd = fd;
    }
#endif
    return n;
}

先接受信息，再复制信息，下面就是真正的处理了 ngx_channel_handler函数中(ngx_process_cycle.c)

 

            ngx_processes[ch.slot].pid = ch.pid;
            ngx_processes[ch.slot].channel[0] = ch.fd;

就是在对应的位置上复制pid和fd,下次向往哪个进程发信息的时候，直接发到 ngx_process[目标进程].channel[0]

 

 

四 父进程

     对于父进程而言，他知道所有进程的channel[0]， 直接可以向子进程发送命令