Nginx进程分析（master_process篇）
首先看Nginx的主进程，主进程从main函数开始运行：
int ngx_cdecl main(int argc, char *const *argv)
{
ngx_int_t i;
ngx_log_t *log;
ngx_cycle_t *cycle, init_cycle;
ngx_core_conf_t *ccf;
ngx_max_sockets = -1;
// 初始化时钟模块，更新时钟（详见ngx_time_update）
ngx_time_init();
// 获取进程号
ngx_pid = ngx_getpid();
// 初始化日志
log = ngx_log_init();
if (log == NULL) { return 1;}
// 初始化Openssl
ngx_ssl_init(log);
ngx_memzero(&init_cycle, sizeof(ngx_cycle_t));
init_cycle.log = log;
ngx_cycle = &init_cycle;
// 创建内存池
init_cycle.pool = ngx_create_pool(1024, log);
if (init_cycle.pool == NULL) { return 1; }
// 保存argc和argv到ngx_argc和ngx_argv中
if (ngx_save_argv(&init_cycle, argc, argv) != NGX_OK) { return 1; }
// 解析argv中的参数，部分作用于init_cycle
if (ngx_getopt(&init_cycle, argc, ngx_argv) != NGX_OK) { return 1; }
// 这里的ngx_show_version、ngx_show_configure和ngx_test_config都是由ngx_getopt确定的
if (ngx_show_version) {
ngx_write_fd(ngx_stderr_fileno, "nginx version: " NGINX_VER CRLF,
sizeof("nginx version: " NGINX_VER CRLF) - 1);
if (ngx_show_configure) { …… }
if (!ngx_test_config) { return 0; }
}
if (ngx_test_config) {
log->log_level = NGX_LOG_INFO;
}
// 获取系统相关参数，例如pagesize和cpu个数；初始化随机种子
if (ngx_os_init(log) != NGX_OK) { return 1; }
if (ngx_crc32_init() != NGX_OK) { return 1; }
// 如果获得了环境变量，就会初始化cycle. Listening，否则直接返回
if (ngx_add_inherited_sockets(&init_cycle) != NGX_OK) { return 1; }
ngx_max_module = 0;
for (i = 0; ngx_modules[i]; i++) {
ngx_modules[i]->index = ngx_max_module++;
}
// 超级大的一个函数
cycle = ngx_init_cycle(&init_cycle);
if (cycle == NULL) {
if (ngx_test_config) {
ngx_log_error(NGX_LOG_EMERG, log, 0,
"the configuration file %s test failed",
init_cycle.conf_file.data);
}
return 1;
}
if (ngx_test_config) {
ngx_log_error(NGX_LOG_INFO, log, 0,
"the configuration file %s was tested successfully",
cycle->conf_file.data);
return 0;
}
// 打印日志
ngx_os_status(cycle->log);
ngx_cycle = cycle;
// 宏展开为ccf = cycle->conf_ctx[ngx_core_module.index];
ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
// ccf->master在ngx_core_module_init_conf中被设置为1
ngx_process = ccf->master ? NGX_PROCESS_MASTER : NGX_PROCESS_SINGLE;
// 设置软中断（信号）处理函数ngx_signal_handler，该函数完成了对11种信号的处理
if (ngx_init_signals(cycle->log) != NGX_OK) { return 1; }
if (!ngx_inherited && ccf->daemon) {
if (ngx_daemon(cycle->log) != NGX_OK) {
return 1;
}
ngx_daemonized = 1;
}
// 创建进程记录文件
if (ngx_create_pidfile(&ccf->pid, cycle->log) != NGX_OK) {
return 1;
}
if (ngx_process == NGX_PROCESS_MASTER) {
ngx_master_process_cycle(cycle);
} else {
ngx_single_process_cycle(cycle);
}
return 0;
}main函数主要完成一系列初始化（详见注释），然后进入进程循环函数。根据输入参数的设置，可以进入主进程循环（ngx_master_process_cycle）或者单进程循环（ngx_single_process_cycle）。一般都是进入主进程循环。在该模式下，主进程会启动若干个（等于CPU核心数）工作进程用以完成相关计算。
下面是主进程真正进入主循环之前所做的工作：
void ngx_master_process_cycle(ngx_cycle_t *cycle)
{
char *title;
u_char *p;
size_t size;
ngx_int_t i;
ngx_uint_t n;
sigset_t set;
struct itimerval itv;
ngx_uint_t live;
ngx_msec_t delay;
ngx_listening_t *ls;
ngx_core_conf_t *ccf;
// 屏蔽前述11种信号之10种（SIGPIPE除外，因为信号处理函数唯独对SIGPIPE没处理）
sigemptyset(&set);
sigaddset(&set, SIGCHLD);
sigaddset(&set, SIGALRM);
sigaddset(&set, SIGIO);
sigaddset(&set, SIGINT);
sigaddset(&set, ngx_signal_value(NGX_RECONFIGURE_SIGNAL));
sigaddset(&set, ngx_signal_value(NGX_REOPEN_SIGNAL));
sigaddset(&set, ngx_signal_value(NGX_NOACCEPT_SIGNAL));
sigaddset(&set, ngx_signal_value(NGX_TERMINATE_SIGNAL));
sigaddset(&set, ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
sigaddset(&set, ngx_signal_value(NGX_CHANGEBIN_SIGNAL));
if (sigprocmask(SIG_BLOCK, &set, NULL) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"sigprocmask() failed");
}
sigemptyset(&set);
size = sizeof(master_process);
for (i = 0; i < ngx_argc; i++) {
size += ngx_strlen(ngx_argv[i]) + 1;
}
title = ngx_palloc(cycle->pool, size);
p = ngx_cpymem(title, master_process, sizeof(master_process) - 1);
for (i = 0; i < ngx_argc; i++) {
*p++ = ' ';
p = ngx_cpystrn(p, (u_char *) ngx_argv[i], size);
}
ngx_setproctitle(title);
ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
ngx_start_worker_processes(cycle, ccf->worker_processes, NGX_PROCESS_RESPAWN);
ngx_start_garbage_collector(cycle, NGX_PROCESS_RESPAWN);
下面是master进程的主循环。进程每次等待信号到来，信号到来后首先被ngx_signal_handler处理，设置标志变量，然后进程被唤醒，根据标志变量执行相关操作，然后再次等待信号。注意主循环之前已屏蔽ngx_singal_handler中会被处理的信号，只在sigsuspend时解除屏蔽，保证了标志变量访问的互斥性。
ngx_new_binary = 0;
delay = 0;
live = 1;
for ( ;; ) {
// delay只在ngx_terminate被置位后才被赋为非零值，然后在此基础上每次循环增加一倍，并添加一个定时器，在delay毫秒后唤醒该进程
if (delay) {
delay *= 2;
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"temination cycle: %d", delay);
itv.it_interval.tv_sec = 0;
itv.it_interval.tv_usec = 0;
itv.it_value.tv_sec = delay / 1000;
itv.it_value.tv_usec = (delay % 1000 ) * 1000;
if (setitimer(ITIMER_REAL, &itv, NULL) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"setitimer() failed");
}
}
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "sigsuspend");
sigsuspend(&set);
ngx_time_update(0, 0);
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "wake up");
if (ngx_reap) {
ngx_reap = 0;
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "reap children");
// 如果还有运行着的工作进程则返回1，否则返回0
live = ngx_reap_children(cycle);
}
// 如果工作进程已死并且主进程终端或退出标志置位，则退出主进程
if (!live && (ngx_terminate || ngx_quit)) {
ngx_master_process_exit(cycle);
}
if (ngx_terminate) {
if (delay == 0) {
delay = 50;
}
// 运行到这里说明工作进程没有全部退出，从delay=0开始依次间隔50ms、100ms、200ms、400ms、800ms向所有工作进程发送SIGTERM信号，最后一次直接发送SIGKILL强制终止工作进程。
if (delay > 1000) {
ngx_signal_worker_processes(cycle, SIGKILL);
} else {
ngx_signal_worker_processes(cycle,
ngx_signal_value(NGX_TERMINATE_SIGNAL));
}
continue;
}
if (ngx_quit) {
// 向所有工作进程发送SIGQUIT信号
ngx_signal_worker_processes(cycle,
ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
// 关闭所有监听套接字
ls = cycle->listening.elts;
for (n = 0; n < cycle->listening.nelts; n++) {
if (ngx_close_socket(ls[n].fd) == -1) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
ngx_close_socket_n " %V failed",
&ls[n].addr_text);
}
}
cycle->listening.nelts = 0;
continue;
}
if (ngx_reconfigure) {
ngx_reconfigure = 0;
// 另外再创建ccf->worker_processes个工作进程
if (ngx_new_binary) {
ngx_start_worker_processes(cycle, ccf->worker_processes,
NGX_PROCESS_RESPAWN);
ngx_start_garbage_collector(cycle, NGX_PROCESS_RESPAWN);
ngx_noaccepting = 0;
continue;
}
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reconfiguring");
// 重新创建ccf->worker_processes个工作进程
cycle = ngx_init_cycle(cycle);
if (cycle == NULL) {
cycle = (ngx_cycle_t *) ngx_cycle;
continue;
}
ngx_cycle = cycle;
ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
ngx_start_worker_processes(cycle, ccf->worker_processes,
NGX_PROCESS_JUST_RESPAWN);
ngx_start_garbage_collector(cycle, NGX_PROCESS_JUST_RESPAWN);
live = 1;
ngx_signal_worker_processes(cycle, ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
}
if (ngx_restart) {
ngx_restart = 0;
ngx_start_worker_processes(cycle, ccf->worker_processes,
NGX_PROCESS_RESPAWN);
ngx_start_garbage_collector(cycle, NGX_PROCESS_RESPAWN);
live = 1;
}
if (ngx_reopen) {
ngx_reopen = 0;
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs");
ngx_reopen_files(cycle, ccf->user);
ngx_signal_worker_processes(cycle,
ngx_signal_value(NGX_REOPEN_SIGNAL));
}
if (ngx_change_binary) {
ngx_change_binary = 0;
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "changing binary");
ngx_new_binary = ngx_exec_new_binary(cycle, ngx_argv);
}
if (ngx_noaccept) {
ngx_noaccept = 0;
ngx_noaccepting = 1;
ngx_signal_worker_processes(cycle,
ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
}
}
}
之前提到主进程首先会屏蔽信号处理函数中会被处理的信号，那么信号处理函数究竟会处理哪些信号，有时怎么处理，信号和主循环中的标志变量之间的对应关系是什么？所以我们看一看信号处理函数（由于我们暂时并不关心工作进程，因此下面只有主进程相关的信号处理）：
void ngx_signal_handler(int signo)
{
char *action;
ngx_int_t ignore;
ngx_err_t err;
ngx_signal_t *sig;
ignore = 0;
err = ngx_errno;
for (sig = signals; sig->signo != 0; sig++) {
if (sig->signo == signo) {
break;
}
}
ngx_time_update(0, 0);
action = "";
switch (ngx_process) {
case NGX_PROCESS_MASTER:
case NGX_PROCESS_SINGLE:
switch (signo) {
case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
ngx_quit = 1;
action = ", shutting down";
break;
case ngx_signal_value(NGX_TERMINATE_SIGNAL):
case SIGINT:
ngx_terminate = 1;
action = ", exiting";
break;
case ngx_signal_value(NGX_NOACCEPT_SIGNAL):
ngx_noaccept = 1;
action = ", stop accepting connections";
break;
case ngx_signal_value(NGX_RECONFIGURE_SIGNAL):
ngx_reconfigure = 1;
action = ", reconfiguring";
break;
case ngx_signal_value(NGX_REOPEN_SIGNAL):
ngx_reopen = 1;
action = ", reopening logs";
break;
case ngx_signal_value(NGX_CHANGEBIN_SIGNAL):
if (getppid() > 1 || ngx_new_binary > 0) {
/*
* Ignore the signal in the new binary if its parent is
* not the init process, i.e. the old binary's process
* is still running. Or ignore the signal in the old binary's
* process if the new binary's process is already running.
*/
action = ", ignoring";
ignore = 1;
break;
}
ngx_change_binary = 1;
action = ", changing binary";
break;
case SIGALRM:
break;
case SIGIO:
ngx_sigio = 1;
break;
case SIGCHLD:
ngx_reap = 1;
break;
}
break;
case NGX_PROCESS_WORKER:
...
break;
}
...
if (signo == SIGCHLD) {
ngx_process_get_status();
}
ngx_set_errno(err);
}我们首先来看SIGCHLD信号，对应着ngx_reap，在主循环中如果ngx_reap置一则调用ngx_reap_children函数。子进程结束时，父进程就会收到SIGCHLD信号。所以ngx_reap主要做的是：关闭与该子进程（工作进程）通信的管道；通知其他所有（工作）进程该工作进程退出的消息；对于需要重启的则重启该工作进程（守护功能），同时也要通知其他进程；从进程数组中移除该进程。
static ngx_uint_t ngx_reap_children(ngx_cycle_t *cycle)
{
ngx_int_t i, n;
ngx_uint_t live;
ngx_channel_t ch;
ngx_core_conf_t *ccf;
ch.command = NGX_CMD_CLOSE_CHANNEL;
ch.fd = -1;
live = 0;
for (i = 0; i < ngx_last_process; i++) {
if (ngx_processes[i].pid == -1) {
continue;
}
if (ngx_processes[i].exited) {
if (!ngx_processes[i].detached) {
// 关闭读管道
ngx_close_channel(ngx_processes[i].channel, cycle->log);
ngx_processes[i].channel[0] = -1;
ngx_processes[i].channel[1] = -1;
ch.pid = ngx_processes[i].pid;
ch.slot = i;
// 通知其他所有进程该进程已退出，因为工作进程都是通过fork产生的，所以它们都有和主进程一样的该进程管道句柄，需要关闭
for (n = 0; n < ngx_last_process; n++) {
if (ngx_processes[n].exited
|| ngx_processes[n].pid == -1
|| ngx_processes[n].channel[0] == -1)
{
continue;
}
ngx_write_channel(ngx_processes[n].channel[0],
&ch, sizeof(ngx_channel_t), cycle->log);
}
}
// 需要重启的进程则调用ngx_spawn_process重启。
if (ngx_processes[i].respawn
&& !ngx_processes[i].exiting
&& !ngx_terminate
&& !ngx_quit)
{
if (ngx_spawn_process(cycle, ngx_processes[i].proc,
ngx_processes[i].data,
ngx_processes[i].name, i)
== NGX_INVALID_PID)
{
continue;
}
ch.command = NGX_CMD_OPEN_CHANNEL;
ch.pid = ngx_processes[ngx_process_slot].pid;
ch.slot = ngx_process_slot;
ch.fd = ngx_processes[ngx_process_slot].channel[0];
// 通知其他所有进程，新进程已创建，与上面的类似，也需要设置新进程ID和管道描述符
for (n = 0; n < ngx_last_process; n++) {
if (n == ngx_process_slot
|| ngx_processes[n].pid == -1
|| ngx_processes[n].channel[0] == -1)
{
continue;
}
ngx_write_channel(ngx_processes[n].channel[0],
&ch, sizeof(ngx_channel_t), cycle->log);
}
live = 1;
continue;
}
if (ngx_processes[i].pid == ngx_new_binary) {
ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,
ngx_core_module);
if (ngx_rename_file((char *) ccf->oldpid.data,
(char *) ccf->pid.data)
!= NGX_OK)
{}
ngx_new_binary = 0;
if (ngx_noaccepting) {
ngx_restart = 1;
ngx_noaccepting = 0;
}
}
// 移除该进程
if (i == ngx_last_process - 1) {
ngx_last_process--;
} else {
ngx_processes[i].pid = -1;
}
} else if (ngx_processes[i].exiting || !ngx_processes[i].detached) {
live = 1;
}
}
return live;
}
在主循环中，如果ngx_terminate或ngx_quit变量被置一，并且没有工作进程仍在运行，则会调用ngx_master_process_exit函数负责退出主进程。
static void ngx_master_process_exit(ngx_cycle_t *cycle)
{
ngx_uint_t i;
ngx_delete_pidfile(cycle);
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exit");
for (i = 0; ngx_modules[i]; i++) {
if (ngx_modules[i]->exit_master) {
ngx_modules[i]->exit_master(cycle);
}
}
/*
* Copy ngx_cycle->log related data to the special static exit cycle,
* log, and log file structures enough to allow a signal handler to log.
* The handler may be called when standard ngx_cycle->log allocated from
* ngx_cycle->pool is already destroyed.
*/
ngx_exit_log_file.fd = ngx_cycle->log->file->fd;
ngx_exit_log = *ngx_cycle->log;
ngx_exit_log.file = &ngx_exit_log_file;
ngx_exit_cycle.log = &ngx_exit_log;
ngx_cycle = &ngx_exit_cycle;
ngx_destroy_pool(cycle->pool);
exit(0);
}