ceph rgw CephContext的_admin_socket属性及功能扩展模式

ceph rgw CephContext的_admin_socket属性及功能扩展模式

_admin_socket是CephContext类的一个属性，在实例化CephContext时，初始化该变量。在rgw中他代表一个线程，提供了一些额外的command line API。
他定义了一些命令行指令：

//在CephContext构造函数中注册以下指令
_admin_hook = new CephContextHook(this);
_admin_socket->register_command("perfcounters_dump", "perfcounters_dump", _admin_hook, "");
  _admin_socket->register_command("1", "1", _admin_hook, "");
  _admin_socket->register_command("perf dump", "perf dump name=logger,type=CephString,req=false name=counter,type=CephString,req=false", _admin_hook, "dump perfcounters value");
  _admin_socket->register_command("perfcounters_schema", "perfcounters_schema", _admin_hook, "");
  _admin_socket->register_command("2", "2", _admin_hook, "");
  _admin_socket->register_command("perf schema", "perf schema", _admin_hook, "dump perfcounters schema");
  _admin_socket->register_command("perf reset", "perf reset name=var,type=CephString", _admin_hook, "perf reset : perf reset all or one perfcounter name");
  _admin_socket->register_command("config show", "config show", _admin_hook, "dump current config settings");
  _admin_socket->register_command("config set", "config set name=var,type=CephString name=val,type=CephString,n=N",  _admin_hook, "config set   [ ...]: set a config variable");
  _admin_socket->register_command("config get", "config get name=var,type=CephString", _admin_hook, "config get : get the config value");
  _admin_socket->register_command("config diff",
      "config diff", _admin_hook,
      "dump diff of current config and default config");
  _admin_socket->register_command("log flush", "log flush", _admin_hook, "flush log entries to log file");
  _admin_socket->register_command("log dump", "log dump", _admin_hook, "dump recent log entries to log file");
  _admin_socket->register_command("log reopen", "log reopen", _admin_hook, "reopen log file");
//以下是在adminSocket::init()中注册：
 m_version_hook = new VersionHook;
  register_command("0", "0", m_version_hook, "");
  register_command("version", "version", m_version_hook, "get ceph version");
  register_command("git_version", "git_version", m_version_hook, "get git sha1");
  m_help_hook = new HelpHook(this);
  register_command("help", "help", m_help_hook, "list available commands");
  m_getdescs_hook = new GetdescsHook(this);
  register_command("get_command_descriptions", "get_command_descriptions",
           m_getdescs_hook, "list available commands");

以上注册的方法在CephContextHook类中有实现，通过注册回调的方式提供。

_admin_socket是类AdminSocket类的实例，该线程的构建过程如下：
1. 为admin 线程的创建做一些初始化工作：

bool AdminSocket::init(const std::string &path)
{
  ldout(m_cct, 5) << "init " << path << dendl;

  /* Set up things for the new thread */
  std::string err;
  int pipe_rd = -1, pipe_wr = -1;
  //为线程管理（关闭该线程）创建入口管道
  err = create_shutdown_pipe(&pipe_rd, &pipe_wr);
  if (!err.empty()) {
    lderr(m_cct) << "AdminSocketConfigObs::init: error: " << err << dendl;
    return false;
  }
  int sock_fd;
  //创建admin socket并监听指定端口
  err = bind_and_listen(path, &sock_fd);
  if (!err.empty()) {
    lderr(m_cct) << "AdminSocketConfigObs::init: failed: " << err << dendl;
    close(pipe_rd);
    close(pipe_wr);
    return false;
  }

  /* Create new thread */
  m_sock_fd = sock_fd;
  m_shutdown_rd_fd = pipe_rd;
  m_shutdown_wr_fd = pipe_wr;
  m_path = path;

  m_version_hook = new VersionHook;
  register_command("0", "0", m_version_hook, "");
  register_command("version", "version", m_version_hook, "get ceph version");
  register_command("git_version", "git_version", m_version_hook, "get git sha1");
  m_help_hook = new HelpHook(this);
  register_command("help", "help", m_help_hook, "list available commands");
  m_getdescs_hook = new GetdescsHook(this);
  register_command("get_command_descriptions", "get_command_descriptions",
           m_getdescs_hook, "list available commands");

  create();
  add_cleanup_file(m_path.c_str());
  return true;
}

2.创建、绑定、监听admin socket

std::string AdminSocket::bind_and_listen(const std::string &sock_path, int *fd)
{
  ldout(m_cct, 5) << "bind_and_listen " << sock_path << dendl;

  //1.创建sockaddr_un对象。
  struct sockaddr_un address;
  if (sock_path.size() > sizeof(address.sun_path) - 1) {
    ostringstream oss;
    oss << "AdminSocket::bind_and_listen: "
    << "The UNIX domain socket path " << sock_path << " is too long! The "
    << "maximum length on this system is "
    << (sizeof(address.sun_path) - 1);
    return oss.str();
  }
  //2.创建unix domain socket
  int sock_fd = socket(PF_UNIX, SOCK_STREAM, 0);
  if (sock_fd < 0) {
    int err = errno;
    ostringstream oss;
    oss << "AdminSocket::bind_and_listen: "
    << "failed to create socket: " << cpp_strerror(err);
    return oss.str();
  }
  //2.设置socket的属性
  int r = fcntl(sock_fd, F_SETFD, FD_CLOEXEC);
  if (r < 0) {
    r = errno;
    VOID_TEMP_FAILURE_RETRY(::close(sock_fd));
    ostringstream oss;
    oss << "AdminSocket::bind_and_listen: failed to fcntl on socket: " << cpp_strerror(r);
    return oss.str();
  }
  //清空address实例，并初始化sum_family属性
  memset(&address, 0, sizeof(struct sockaddr_un));
  address.sun_family = AF_UNIX;
  //把sock_path按照指定格式赋值给address.sum_path。
  snprintf(address.sun_path, sizeof(address.sun_path),
       "%s", sock_path.c_str());
  //3.把sock_fd与address绑定。
  if (::bind(sock_fd, (struct sockaddr*)&address,
       sizeof(struct sockaddr_un)) != 0) {
    int err = errno;
    //判断该sokcet path是否已被使用。
    if (err == EADDRINUSE) {
      AdminSocketClient client(sock_path);
      bool ok;
      client.ping(&ok);
      if (ok) {
    ldout(m_cct, 20) << "socket " << sock_path << " is in use" << dendl;
    err = EEXIST;
      } else {
    ldout(m_cct, 20) << "unlink stale file " << sock_path << dendl;
    VOID_TEMP_FAILURE_RETRY(unlink(sock_path.c_str()));
    if (::bind(sock_fd, (struct sockaddr*)&address,
         sizeof(struct sockaddr_un)) == 0) {
      err = 0;
    } else {
      err = errno;
    }
      }
    }
    if (err != 0) {
      ostringstream oss;
      oss << "AdminSocket::bind_and_listen: "
      << "failed to bind the UNIX domain socket to '" << sock_path
      << "': " << cpp_strerror(err);
      close(sock_fd);
      return oss.str();
    }
  }
  //4.socket监听指定端口5
  if (listen(sock_fd, 5) != 0) {
    int err = errno;
    ostringstream oss;
    oss << "AdminSocket::bind_and_listen: "
      << "failed to listen to socket: " << cpp_strerror(err);
    close(sock_fd);
    VOID_TEMP_FAILURE_RETRY(unlink(sock_path.c_str()));
    return oss.str();
  }
  *fd = sock_fd;
  return "";
}

该线程的主要逻辑

void* AdminSocket::entry()
{
  ldout(m_cct, 5) << "entry start" << dendl;
  while (true) {
    struct pollfd fds[2];
    memset(fds, 0, sizeof(fds));
    fds[0].fd = m_sock_fd;
    fds[0].events = POLLIN | POLLRDBAND;
    fds[1].fd = m_shutdown_rd_fd;
    fds[1].events = POLLIN | POLLRDBAND;

    int ret = poll(fds, 2, -1);
    if (ret < 0) {
      int err = errno;
      if (err == EINTR) {
    continue;
      }
      lderr(m_cct) << "AdminSocket: poll(2) error: '"
           << cpp_strerror(err) << dendl;
      return PFL_FAIL;
    }
    //接收指令，并找到与之对对应的函数，执行。
    if (fds[0].revents & POLLIN) {
      // Send out some data
      do_accept();
    }
    //关闭当前线程
    if (fds[1].revents & POLLIN) {
      // Parent wants us to shut down
      return PFL_SUCCESS;
    }
  }
  ldout(m_cct, 5) << "entry exit" << dendl;
}

3.接收请求并处理

bool AdminSocket::do_accept()
{
  struct sockaddr_un address;
  socklen_t address_length = sizeof(address);
  ldout(m_cct, 30) << "AdminSocket: calling accept" << dendl;
  int connection_fd = accept(m_sock_fd, (struct sockaddr*) &address,
                 &address_length);
  ldout(m_cct, 30) << "AdminSocket: finished accept" << dendl;
  if (connection_fd < 0) {
    int err = errno;
    lderr(m_cct) << "AdminSocket: do_accept error: '"
               << cpp_strerror(err) << dendl;
    return false;
  }

  char cmd[1024];
  int pos = 0;
  string c;
  while (1) {
    int ret = safe_read(connection_fd, &cmd[pos], 1);
    if (ret <= 0) {
      lderr(m_cct) << "AdminSocket: error reading request code: "
           << cpp_strerror(ret) << dendl;
      close(connection_fd);
      return false;
    }
    //ldout(m_cct, 0) << "AdminSocket read byte " << (int)cmd[pos] << " pos " << pos << dendl;
    if (cmd[0] == '\0') {
      // old protocol: __be32
      if (pos == 3 && cmd[0] == '\0') {
    switch (cmd[3]) {
    case 0:
      c = "0";
      break;
    case 1:
      c = "perfcounters_dump";
      break;
    case 2:
      c = "perfcounters_schema";
      break;
    default:
      c = "foo";
      break;
    }
    break;
      }
    } else {
      // new protocol: null or \n terminated string
      if (cmd[pos] == '\n' || cmd[pos] == '\0') {
    cmd[pos] = '\0';
    c = cmd;
    break;
      }
    }
    pos++;
  }

  bool rval = false;

  map<string, cmd_vartype> cmdmap;
  string format;
  vector<string> cmdvec;
  stringstream errss;
  cmdvec.push_back(cmd);
  if (!cmdmap_from_json(cmdvec, &cmdmap, errss)) {
    ldout(m_cct, 0) << "AdminSocket: " << errss.rdbuf() << dendl;
    return false;
  }
  cmd_getval(m_cct, cmdmap, "format", format);
  if (format != "json" && format != "json-pretty" &&
      format != "xml" && format != "xml-pretty")
    format = "json-pretty";
  cmd_getval(m_cct, cmdmap, "prefix", c);

  m_lock.Lock();
  map<string,AdminSocketHook*>::iterator p;
  string match = c;
  while (match.size()) {
    p = m_hooks.find(match);
    if (p != m_hooks.end())
      break;

    // drop right-most word
    size_t pos = match.rfind(' ');
    if (pos == std::string::npos) {
      match.clear();  // we fail
      break;
    } else {
      match.resize(pos);
    }
  }

  bufferlist out;
  if (p == m_hooks.end()) {
    lderr(m_cct) << "AdminSocket: request '" << c << "' not defined" << dendl;
  } else {
    string args;
    if (match != c)
      args = c.substr(match.length() + 1);
    bool success = p->second->call(match, cmdmap, format, out);
    if (!success) {
      ldout(m_cct, 0) << "AdminSocket: request '" << match << "' args '" << args
              << "' to " << p->second << " failed" << dendl;
      out.append("failed");
    } else {
      ldout(m_cct, 5) << "AdminSocket: request '" << match << "' '" << args
               << "' to " << p->second
               << " returned " << out.length() << " bytes" << dendl;
    }
    uint32_t len = htonl(out.length());
    int ret = safe_write(connection_fd, &len, sizeof(len));
    if (ret < 0) {
      lderr(m_cct) << "AdminSocket: error writing response length "
           << cpp_strerror(ret) << dendl;
    } else {
      if (out.write_fd(connection_fd) >= 0)
    rval = true;
    }
  }
  m_lock.Unlock();

  VOID_TEMP_FAILURE_RETRY(close(connection_fd));
  return rval;
}

回调被注册的回调函数，AdminSocketHook是一个类，其中定义了一个钩子，如果要提供在admin 线程中扩展新的方法，也可以通过派生AdminSocketHook的子类，来实现，其中一定要实现父类中的call（…）方法即可。
AdminSocketHook类：

class AdminSocketHook {
public:
  virtual bool call(std::string command, cmdmap_t &cmdmap, std::string format,
            bufferlist& out) = 0;
  virtual ~AdminSocketHook() {}
};

派生的子类CephContextHook：

class CephContextHook : public AdminSocketHook {
  CephContext *m_cct;

public:
  CephContextHook(CephContext *cct) : m_cct(cct) {}

  bool call(std::string command, cmdmap_t& cmdmap, std::string format,
        bufferlist& out) {
    m_cct->do_command(command, cmdmap, format, &out);
    return true;
  }
};

void CephContext::do_command(std::string command, cmdmap_t& cmdmap,
                 std::string format, bufferlist *out)
{
  Formatter *f = Formatter::create(format, "json-pretty", "json-pretty");
  stringstream ss;
  for (cmdmap_t::iterator it = cmdmap.begin(); it != cmdmap.end(); ++it) {
    if (it->first != "prefix") {
      ss << it->first  << ":" << cmd_vartype_stringify(it->second) << " ";
    }
  }
  lgeneric_dout(this, 1) << "do_command '" << command << "' '"
             << ss.str() << dendl;
  if (command == "perfcounters_dump" || command == "1" ||
      command == "perf dump") {
    std::string logger;
    std::string counter;
    cmd_getval(this, cmdmap, "logger", logger);
    cmd_getval(this, cmdmap, "counter", counter);
    _perf_counters_collection->dump_formatted(f, false, logger, counter);
  }
  else if (command == "perfcounters_schema" || command == "2" ||
    command == "perf schema") {
    _perf_counters_collection->dump_formatted(f, true);
  }
  else if (command == "perf reset") {
    std::string var;
    if (!cmd_getval(this, cmdmap, "var", var)) {
      f->dump_string("error", "syntax error: 'perf reset '");
    } else {
     if(!_perf_counters_collection->reset(var))
        f->dump_stream("error") << "Not find: " << var;
    }
  }
  else {
    string section = command;
    boost::replace_all(section, " ", "_");
    f->open_object_section(section.c_str());
    if (command == "config show") {
      _conf->show_config(f);
    }
    else if (command == "config set") {
      std::string var;
      std::vector<std::string> val;

      if (!(cmd_getval(this, cmdmap, "var", var)) ||
          !(cmd_getval(this, cmdmap, "val", val))) {
        f->dump_string("error", "syntax error: 'config set  '");
      } else {
    // val may be multiple words
    string valstr = str_join(val, " ");
        int r = _conf->set_val(var.c_str(), valstr.c_str());
        if (r < 0) {
          f->dump_stream("error") << "error setting '" << var << "' to '" << valstr << "': " << cpp_strerror(r);
        } else {
          ostringstream ss;
          _conf->apply_changes(&ss);
          f->dump_string("success", ss.str());
        }
      }
    } else if (command == "config get") {
      std::string var;
      if (!cmd_getval(this, cmdmap, "var", var)) {
    f->dump_string("error", "syntax error: 'config get '");
      } else {
    char buf[4096];
    memset(buf, 0, sizeof(buf));
    char *tmp = buf;
    int r = _conf->get_val(var.c_str(), &tmp, sizeof(buf));
    if (r < 0) {
        f->dump_stream("error") << "error getting '" << var << "': " << cpp_strerror(r);
    } else {
        f->dump_string(var.c_str(), buf);
    }
      }
    } else if (command == "config diff") {
      md_config_t def_conf;
      def_conf.set_val("cluster", _conf->cluster);
      def_conf.name = _conf->name;
      def_conf.set_val("host", _conf->host);
      def_conf.apply_changes(NULL);

      map<string,pair<string,string> > diff;
      set<string> unknown;
      def_conf.diff(_conf, &diff, &unknown);
      f->open_object_section("diff");

      f->open_object_section("current");
      for (map<string,pair<string,string> >::iterator p = diff.begin();
           p != diff.end(); ++p) {
        f->dump_string(p->first.c_str(), p->second.second);
      }
      f->close_section(); // current
      f->open_object_section("defaults");
      for (map<string,pair<string,string> >::iterator p = diff.begin();
           p != diff.end(); ++p) {
        f->dump_string(p->first.c_str(), p->second.first);
      }
      f->close_section(); // defaults
      f->close_section(); // diff

      f->open_array_section("unknown");
      for (set<string>::iterator p = unknown.begin();
           p != unknown.end(); ++p) {
        f->dump_string("option", *p);
      }
      f->close_section(); // unknown
    } else if (command == "log flush") {
      _log->flush();
    }
    else if (command == "log dump") {
      _log->dump_recent();
    }
    else if (command == "log reopen") {
      _log->reopen_log_file();
    }
    else {
      assert(0 == "registered under wrong command?");    
    }
    f->close_section();
  }
  f->flush(*out);
  delete f;
  lgeneric_dout(this, 1) << "do_command '" << command << "' '" << ss.str()
                 << "result is " << out->length() << " bytes" << dendl;
}

派生的子类HelpHook：

class HelpHook : public AdminSocketHook {
  AdminSocket *m_as;
public:
  HelpHook(AdminSocket *as) : m_as(as) {}
  bool call(string command, cmdmap_t &cmdmap, string format, bufferlist& out) {
    Formatter *f = Formatter::create(format, "json-pretty", "json-pretty");
    f->open_object_section("help");
    for (map<string,string>::iterator p = m_as->m_help.begin();
     p != m_as->m_help.end();
     ++p) {
      if (p->second.length())
    f->dump_string(p->first.c_str(), p->second);
    }
    f->close_section();
    ostringstream ss;
    f->flush(ss);
    out.append(ss.str());
    delete f;
    return true;
  }
};

派生类GetdescsHook ：

class GetdescsHook : public AdminSocketHook {
  AdminSocket *m_as;
public:
  GetdescsHook(AdminSocket *as) : m_as(as) {}
  bool call(string command, cmdmap_t &cmdmap, string format, bufferlist& out) {
    int cmdnum = 0;
    JSONFormatter jf(false);
    jf.open_object_section("command_descriptions");
    for (map<string,string>::iterator p = m_as->m_descs.begin();
     p != m_as->m_descs.end();
     ++p) {
      ostringstream secname;
      secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
      dump_cmd_and_help_to_json(&jf,
                secname.str().c_str(),
                p->second.c_str(),
                m_as->m_help[p->first]);
      cmdnum++;
    }
    jf.close_section(); // command_descriptions
    ostringstream ss;
    jf.flush(ss);
    out.append(ss.str());
    return true;
  }
};