Ceph: 异步信号处理实现

异步信号处理函数

原理：

创建一个独立的线程，根据信号调用对应的信号处理函数
利用管道进行线程之间的通信，在产生信号时，通知信号处理线程，调用信号处理函数
主线程中，所有注册的信号的信号处理函数都相同，主要是通知信号处理线程有信号产生
信号处理线程收到信号发生的通知时，调用此信号对应的处理函数

数据结构

struct SignalHandler : public Thread {
  /*
  * 监听pipefd[0], 用于处理 shutdown, 注册signal handler等.
  * pipefd[1]，主线程写，通知信号处理线程有新注册的signal
  */
  int pipefd[2];  // write to [1], read from [0]

  /* 
  * shutdown信号处理线程
  */
  bool stop = false;

  /*
  * 每一个信号，都对应一个safe_handler
  * info_t: siginfo_t，信号产生时，填充的信号相关信息，如信号值，错误码，状态等
  * pipefd[2]: 信号处理线程监听在pipefd[0]，主线程中产生信号时，会向pipefd[1]写，通知信号处理线程，有信号产生
  * handler: 信号处理函数，业务层面注册的信号处理函数存储在此
  */
  struct safe_handler {

    safe_handler() {
      memset(pipefd, 0, sizeof(pipefd));
      memset(&handler, 0, sizeof(handler));
      memset(&info_t, 0, sizeof(info_t));    
    }

    siginfo_t info_t;
    int pipefd[2];  // write to [1], read from [0]
    signal_handler_t handler;
  };

  /*
  * 所有的信号对应的safe_handler数组，index: signum
  */
  safe_handler *handlers[32] = {nullptr};

  /*
  * 操作handlers的锁
  * make_mutex:
      //Args不起作用，用于调试之用
      template 
      std::mutex make_mutex(Args&& ...args) {
        return {};
      }
  */
  std::mutex lock = make_mutex("SignalHandler::lock");

构造函数

初始化pipefd，设置close-on-exec: pipe2(pipefd, O_CLOEXEC | flags)
设置pipefd[0]非阻塞
开启一个新线程(信号处理线程)，入口函数为entry()

SignalHandler() {
  // create signal pipe
  int r = pipe_cloexec(pipefd, 0);
  ceph_assert(r == 0);
  r = fcntl(pipefd[0], F_SETFL, O_NONBLOCK);
  ceph_assert(r == 0);

  // create thread
  create("signal_handler");
}

析构函数：

signal_thread: 写pipefd[1]，唤醒信号处理线程
shutdown: 停止信号处理线程，设置stop标记，调用signal_thread唤醒线程
join: 等待信号处理线程退出

~SignalHandler() override {
  shutdown();
}

void signal_thread() {
  int r = write(pipefd[1], "\0", 1);
  ceph_assert(r == 1);
}

void shutdown() {
  stop = true;
  signal_thread();
  join();
}

信号处理线程

入口函数：entry

功能逻辑：

poll实现IO复用，主要监听主线程及所有信号(safe_handler)中的pipefd[0]
当有IO发生时，轮询read，确定发生的信号(signum)
根据信号(signum)，调用safe_handler->handler: handlers[signum]->handler(signum);

entry函数处理逻辑如下：

  void *entry() override {
    while (!stop) {
      // build fd list
      struct pollfd fds[33];

      lock.lock();
      int num_fds = 0;
      fds[num_fds].fd = pipefd[0];
      fds[num_fds].events = POLLIN | POLLERR;
      fds[num_fds].revents = 0;
      ++num_fds;
      for (unsigned i=0; i<32; i++) {
    if (handlers[i]) {
      fds[num_fds].fd = handlers[i]->pipefd[0];
      fds[num_fds].events = POLLIN | POLLERR;
      fds[num_fds].revents = 0;
      ++num_fds;
    }
      }
      lock.unlock();

      // wait for data on any of those pipes
      int r = poll(fds, num_fds, -1);
      if (stop)
    break;
      if (r > 0) {
    char v;

    // consume byte from signal socket, if any.
    TEMP_FAILURE_RETRY(read(pipefd[0], &v, 1));

    lock.lock();
    for (unsigned signum=0; signum<32; signum++) {
      if (handlers[signum]) {
        r = read(handlers[signum]->pipefd[0], &v, 1);
        if (r == 1) {
          siginfo_t * siginfo = &handlers[signum]->info_t;
              ostringstream message;
              message << "received  signal: " << sig_str(signum);
              switch (siginfo->si_code) {
                case SI_USER:
                  message << " from " << get_name_by_pid(siginfo->si_pid);
                  // If PID is undefined, it doesn't have a meaning to be displayed
                  if (siginfo->si_pid) {
                    message << " (PID: " << siginfo->si_pid << ")";
                  } else {
                    message << " ( Could be generated by pthread_kill(), raise(), abort(), alarm() )";
                  }
                  message << " UID: " << siginfo->si_uid;
                  break;
                default:
                  /* As we have a not expected signal, let's report the structure to help debugging */
                  message << ", si_code : " << siginfo->si_code;
                  message << ", si_value (int): " << siginfo->si_value.sival_int;
                  message << ", si_value (ptr): " << siginfo->si_value.sival_ptr;
                  message << ", si_errno: " << siginfo->si_errno;
                  message << ", si_pid : " << siginfo->si_pid;
                  message << ", si_uid : " << siginfo->si_uid;
                  message << ", si_addr" << siginfo->si_addr;
                  message << ", si_status" << siginfo->si_status;
                  break;
              }
              derr << message.str() << dendl;
              handlers[signum]->handler(signum);
        }
      }
    }
    lock.unlock();
      } 
    }
    return NULL;
  }

产生信号时，通知信号处理线程接口函数

逻辑：

注册时的信号处理函数调用此函数接口，通知信号处理线程
写信号对应的pipefd[1]

  void queue_signal(int signum) {
    ceph_assert(handlers[signum]);
    int r = write(handlers[signum]->pipefd[1], " ", 1);
    ceph_assert(r == 1);
  }

  void queue_signal_info(int signum, siginfo_t *siginfo, void * content) {
    ceph_assert(handlers[signum]);
    memcpy(&handlers[signum]->info_t, siginfo, sizeof(siginfo_t));
    int r = write(handlers[signum]->pipefd[1], " ", 1);
    ceph_assert(r == 1);
  }

注册信号处理函数

主要逻辑如下：

信号: signum，生成new safe_handler，将此safe_handler与signum对应： handlers[signum] = new safe_handler
存储信号处理函数: safe_handler->handler = handler
signal_thread: 通知信号处理线程，有新的信号注册，信号处理线程会被唤醒，添加针对此信号的监听
设置实际的信号处理函数: handler_signal_hook
handler_signal_hook: 实际信号处理函数，产生信号时，进入此函数。此函数调用queue_signal_info，通知信号处理线程

注册代码：

void SignalHandler::register_handler(int signum, signal_handler_t handler, bool oneshot)
{
  int r;

  ceph_assert(signum >= 0 && signum < 32);

  safe_handler *h = new safe_handler;

  r = pipe_cloexec(h->pipefd, 0);
  ceph_assert(r == 0);
  r = fcntl(h->pipefd[0], F_SETFL, O_NONBLOCK);
  ceph_assert(r == 0);

  h->handler = handler;
  lock.lock();
  handlers[signum] = h;
  lock.unlock();

  // signal thread so that it sees our new handler
  signal_thread();
  
  // install our handler
  struct sigaction oldact;
  struct sigaction act;
  memset(&act, 0, sizeof(act));

  act.sa_handler = (signal_handler_t)handler_signal_hook;
  sigfillset(&act.sa_mask);  // mask all signals in the handler
  act.sa_flags = SA_SIGINFO | (oneshot ? SA_RESETHAND : 0);
  int ret = sigaction(signum, &act, &oldact);
  ceph_assert(ret == 0);
}

handler_signal_hook，其调用的是queue_signal_info：

static void handler_signal_hook(int signum, siginfo_t * siginfo, void * content) {
  g_signal_handler->queue_signal_info(signum, siginfo, content);
}

unregister_handler

恢复信号的默认处理啊方式: signal(signum, SIG_DFL)
删除handler[signum]
关闭signum对应的pipefd,关闭pipefd[0]，会唤醒信号处理线程，信号处理线程会清理监听的pipefd[0]

代码如下：

void SignalHandler::unregister_handler(int signum, signal_handler_t handler)
{
  ceph_assert(signum >= 0 && signum < 32);
  safe_handler *h = handlers[signum];
  ceph_assert(h);
  ceph_assert(h->handler == handler);

  // restore to default
  signal(signum, SIG_DFL);

  // _then_ remove our handlers entry
  lock.lock();
  handlers[signum] = NULL;
  lock.unlock();

  // this will wake up select() so that worker thread sees our handler is gone
  close(h->pipefd[0]);
  close(h->pipefd[1]);
  delete h;
}

上述是整个异步信号处理程序的实现。使用如下：

g_signal_handler = new SignalHandler;
g_signal_handler->register_handler(int signum, signal_handler_t handler)

Ceph: 异步信号处理实现

异步信号处理函数

你可能感兴趣的:(Ceph: 异步信号处理实现)