MonClient中的函数
MonClient类位于mon目录下，其功能是负责与monitor交互。
//在消息层组件中注册MonClient实例，指定认证方式，初始化keyring，启动定时器
int MonClient::init()
{
ldout(cct, 10) << "init" << dendl;
//将monclient实例注册到消息层，用于处理与其相关的消息
messenger->add_dispatcher_head(this);
entity_name = cct->_conf->name;
Mutex::Locker l(monc_lock);
//获取认证类型，用于实例化服务之间的认证方式，ceph支持的认证方式有cephx和none（无认证）两种。
string method;
if (!cct->_conf->auth_supported.empty())
method = cct->_conf->auth_supported;
else if (entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
entity_name.get_type() == CEPH_ENTITY_TYPE_MDS ||
entity_name.get_type() == CEPH_ENTITY_TYPE_MON)
method = cct->_conf->auth_cluster_required;
else
method = cct->_conf->auth_client_required;
auth_supported = new AuthMethodList(cct, method);
ldout(cct, 10) << "auth_supported " << auth_supported->get_supported_set() << " method " << method << dendl;
int r = 0;
//初始化keyring，其中保存的都是类似用户名和秘钥对。
keyring = new KeyRing; // initializing keyring anyway
//如果认证方式为cephx，则必须初始化keyring
if (auth_supported->is_supported_auth(CEPH_AUTH_CEPHX)) {
r = keyring->from_ceph_context(cct);
if (r == -ENOENT) {
auth_supported->remove_supported_auth(CEPH_AUTH_CEPHX);
if (!auth_supported->get_supported_set().empty()) {
r = 0;
no_keyring_disabled_cephx = true;
} else {
lderr(cct) << "ERROR: missing keyring, cannot use cephx for authentication" << dendl;
}
}
}
if (r < 0) {
return r;
}
//初始化rotating_secrets
rotating_secrets = new RotatingKeyRing(cct, cct->get_module_type(), keyring);
initialized = true;
//启动定时器
timer.init();
finisher.start();
//向定时器中注册定时事件
schedule_tick();
return 0;
}
如果MonClient.status的状态为MC_STATE_HAVE_SESSION，说明已经认证通过；MonClient.timer其主要功能是执行C_Tick回调对象，其主要执行函数是
MonClient.tick()
int MonClient::authenticate(double timeout)
{
Mutex::Locker lock(monc_lock);
if (state == MC_STATE_HAVE_SESSION) {
ldout(cct, 5) << "already authenticated" << dendl;
return 0;
}
//monmap的请求放入sub_new中
_sub_want("monmap", monmap.get_epoch() ? monmap.get_epoch() + 1 : 0, 0);
if (cur_mon.empty())
_reopen_session(); //重新建立会话连接
//等待monclient与monitor的认证过程完成，异步认证，由auth_cond来触发是否认证成功。
utime_t until = ceph_clock_now(cct);
until += timeout;
if (timeout > 0.0)
ldout(cct, 10) << "authenticate will time out at " << until << dendl;
while (state != MC_STATE_HAVE_SESSION && !authenticate_err) {
if (timeout > 0.0) {
int r = auth_cond.WaitUntil(monc_lock, until);
if (r == ETIMEDOUT) {
ldout(cct, 0) << "authenticate timed out after " << timeout << dendl;
authenticate_err = -r;
}
} else {
auth_cond.Wait(monc_lock);
}
}
if (state == MC_STATE_HAVE_SESSION) {
ldout(cct, 5) << "authenticate success, global_id " << global_id << dendl;
}
if (authenticate_err < 0 && no_keyring_disabled_cephx) {
lderr(cct) << "authenticate NOTE: no keyring found; disabled cephx authentication" << dendl;
}
return authenticate_err;
}
//处理认证响应请求
void MonClient::handle_auth(MAuthReply *m)
{
Context *cb = NULL;
bufferlist::iterator p = m->result_bl.begin();
//解析认证响应的认证类型，重置auth
if (state == MC_STATE_NEGOTIATING) {
if (!auth || (int)m->protocol != auth->get_protocol()) {
delete auth;
auth = get_auth_client_handler(cct, m->protocol, rotating_secrets);
if (!auth) {
ldout(cct, 10) << "no handler for protocol " << m->protocol << dendl;
if (m->result == -ENOTSUP) {
ldout(cct, 10) << "none of our auth protocols are supported by the server"
<< dendl;
authenticate_err = m->result;
auth_cond.SignalAll();
}
m->put();
return;
}
auth->set_want_keys(want_keys);
auth->init(entity_name);
auth->set_global_id(global_id);
} else {
auth->reset();
}
state = MC_STATE_AUTHENTICATING;
}
assert(auth);
if (m->global_id && m->global_id != global_id) {
global_id = m->global_id;
auth->set_global_id(global_id);
ldout(cct, 10) << "my global_id is " << m->global_id << dendl;
}
//处理认证响应的请求
int ret = auth->handle_response(m->result, p);
m->put();
//如果返回值为EGGAIN,则重新启动认证流程
if (ret == -EAGAIN) {
MAuth *ma = new MAuth;
ma->protocol = auth->get_protocol();
auth->prepare_build_request();
ret = auth->build_request(ma->auth_payload);
_send_mon_message(ma, true);
return;
}
_finish_hunting();
//认证成功，更改当前实例的状态为MC_STATE_HAVE_SESSION，然后将为发送出去的消息，通过该会话连接发送至monitor
authenticate_err = ret;
if (ret == 0) {
if (state != MC_STATE_HAVE_SESSION) {
state = MC_STATE_HAVE_SESSION;
last_rotating_renew_sent = utime_t();
while (!waiting_for_session.empty()) {
_send_mon_message(waiting_for_session.front());
waiting_for_session.pop_front();
}
_resend_mon_commands();
send_log(true);
if (session_established_context) {
cb = session_established_context;
session_established_context = NULL;
}
}
//从monitor，获取新的tickets
_check_auth_tickets();
}
//触发所以阻塞在auth_cond上的线程
auth_cond.SignalAll();
if (cb) {
monc_lock.Unlock();
cb->complete(0);
monc_lock.Lock();
}
}
//mon_client_hunt_interval间隔时间触发的回调函数，检查ticket是否过期，并重新注册定时事件。
void MonClient::tick()
{
ldout(cct, 10) << "tick" << dendl;
_check_auth_tickets();
if (hunting) {
ldout(cct, 1) << "continuing hunt" << dendl;
_reopen_session();
} else if (!cur_mon.empty()) {
// just renew as needed
utime_t now = ceph_clock_now(cct);
if (!cur_con->has_feature(CEPH_FEATURE_MON_STATEFUL_SUB)) {
ldout(cct, 10) << "renew subs? (now: " << now
<< "; renew after: " << sub_renew_after << ") -- "
<< (now > sub_renew_after ? "yes" : "no")
<< dendl;
if (now > sub_renew_after)
_renew_subs();
}
cur_con->send_keepalive();
if (state == MC_STATE_HAVE_SESSION) {
if (cct->_conf->mon_client_ping_timeout > 0 &&
cur_con->has_feature(CEPH_FEATURE_MSGR_KEEPALIVE2)) {
utime_t lk = cur_con->get_last_keepalive_ack();
utime_t interval = now - lk;
if (interval > cct->_conf->mon_client_ping_timeout) {
ldout(cct, 1) << "no keepalive since " << lk << " (" << interval
<< " seconds), reconnecting" << dendl;
_reopen_session();
}
}
send_log();
}
}
schedule_tick();
}
//用于注册定时事件
void MonClient::schedule_tick()
{
if (hunting)
timer.add_event_after(cct->_conf->mon_client_hunt_interval
* reopen_interval_multiplier, new C_Tick(this));
else
timer.add_event_after(cct->_conf->mon_client_ping_interval, new C_Tick(this));
}
//与monitor重新建立会话连接，清除之前未发送的消息，重启与monitor的认证过程，重新发送map的订阅请求
void MonClient::_reopen_session(int rank, string name)
{
assert(monc_lock.is_locked());
ldout(cct, 10) << "_reopen_session rank " << rank << " name " << name << dendl;
if (rank < 0 && name.length() == 0) {
cur_mon = _pick_random_mon();
} else if (name.length()) {
cur_mon = name;
} else {
cur_mon = monmap.get_name(rank);
}
if (cur_con) {
cur_con->mark_down();
}
//与monitor重新建立会话连接
cur_con = messenger->get_connection(monmap.get_inst(cur_mon));
ldout(cct, 10) << "picked mon." << cur_mon << " con " << cur_con
<< " addr " << cur_con->get_peer_addr()
<< dendl;
//清除之前未发送的消息
// throw out old queued messages
while (!waiting_for_session.empty()) {
waiting_for_session.front()->put();
waiting_for_session.pop_front();
}
// throw out version check requests
while (!version_requests.empty()) {
finisher.queue(version_requests.begin()->second->context, -EAGAIN);
delete version_requests.begin()->second;
version_requests.erase(version_requests.begin());
}
// adjust timeouts if necessary
if (had_a_connection) {
reopen_interval_multiplier *= cct->_conf->mon_client_hunt_interval_backoff;
if (reopen_interval_multiplier >
cct->_conf->mon_client_hunt_interval_max_multiple)
reopen_interval_multiplier =
cct->_conf->mon_client_hunt_interval_max_multiple;
}
//重启与monitor的认证过程
// restart authentication handshake
state = MC_STATE_NEGOTIATING;
hunting = true;
// send an initial keepalive to ensure our timestamp is valid by the
// time we are in an OPENED state (by sequencing this before
// authentication)
cur_con->send_keepalive();
MAuth *m = new MAuth;
m->protocol = 0;
m->monmap_epoch = monmap.get_epoch();
__u8 struct_v = 1;
::encode(struct_v, m->auth_payload);
::encode(auth_supported->get_supported_set(), m->auth_payload);
::encode(entity_name, m->auth_payload);
::encode(global_id, m->auth_payload);
_send_mon_message(m, true);
for (map::iterator p = sub_sent.begin();
p != sub_sent.end();
++p) {
if (sub_new.count(p->first) == 0)
sub_new[p->first] = p->second;
}
//重新向monitor发送map订阅的请求
if (!sub_new.empty())
_renew_subs();
}
MonClient处理的消息类型：
CEPH_MSG_MON_MAP:
CEPH_MSG_AUTH_REPLY:
CEPH_MSG_MON_SUBSCRIBE_ACK:
CEPH_MSG_MON_GET_VERSION_REPLY:
MSG_MON_COMMAND_ACK:
MSG_LOGACK:
bool MonClient::ms_dispatch(Message *m)
{
if (my_addr == entity_addr_t())
my_addr = messenger->get_myaddr();
// we only care about these message types
switch (m->get_type()) {
case CEPH_MSG_MON_MAP:
case CEPH_MSG_AUTH_REPLY:
case CEPH_MSG_MON_SUBSCRIBE_ACK:
case CEPH_MSG_MON_GET_VERSION_REPLY:
case MSG_MON_COMMAND_ACK:
case MSG_LOGACK:
break;
default:
return false;
}
Mutex::Locker lock(monc_lock);
// ignore any messages outside our current session
if (m->get_connection() != cur_con) {
ldout(cct, 10) << "discarding stray monitor message " << *m << dendl;
m->put();
return true;
}
switch (m->get_type()) {
case CEPH_MSG_MON_MAP:
handle_monmap(static_cast(m));
break;
case CEPH_MSG_AUTH_REPLY:
handle_auth(static_cast(m));
break;
case CEPH_MSG_MON_SUBSCRIBE_ACK:
handle_subscribe_ack(static_cast(m));
break;
case CEPH_MSG_MON_GET_VERSION_REPLY:
handle_get_version_reply(static_cast(m));
break;
case MSG_LOGACK:
if (log_client) {
log_client->handle_log_ack(static_cast(m));
m->put();
if (more_log_pending) {
send_log();
}
} else {
m->put();
}
break;
}
return true;
}
//处理monmap的消息，反序列化新好的的monmap，更新本地的monmap
void MonClient::handle_monmap(MMonMap *m)
{
ldout(cct, 10) << "handle_monmap " << *m << dendl;
bufferlist::iterator p = m->monmapbl.begin();
::decode(monmap, p);
assert(!cur_mon.empty());
ldout(cct, 10) << " got monmap " << monmap.epoch
<< ", mon." << cur_mon << " is now rank " << monmap.get_rank(cur_mon)
<< dendl;
ldout(cct, 10) << "dump:\n";
monmap.print(*_dout);
*_dout << dendl;
_sub_got("monmap", monmap.get_epoch());
if (!monmap.get_addr_name(cur_con->get_peer_addr(), cur_mon)) {
ldout(cct, 10) << "mon." << cur_mon << " went away" << dendl;
_reopen_session(); // can't find the mon we were talking to (above)
}
map_cond.Signal();
want_monmap = false;
m->put();
}
//处理subscribe响应消息，更新下一次发送订阅map的时间点
void MonClient::handle_subscribe_ack(MMonSubscribeAck *m)
{
if (sub_renew_sent != utime_t()) {
// NOTE: this is only needed for legacy (infernalis or older)
// mons; see tick().
sub_renew_after = sub_renew_sent;
sub_renew_after += m->interval / 2.0;
ldout(cct, 10) << "handle_subscribe_ack sent " << sub_renew_sent << " renew after " << sub_renew_after << dendl;
sub_renew_sent = utime_t();
} else {
ldout(cct, 10) << "handle_subscribe_ack sent " << sub_renew_sent << ", ignoring" << dendl;
}
m->put();
}
//处理返送给monclient的command响应消息，通过finisher线程处理对应的回调方法，并将对应的消息从 mon_command中清除
void MonClient::handle_mon_command_ack(MMonCommandAck *ack)
{
MonCommand *r = NULL;
uint64_t tid = ack->get_tid();
if (tid == 0 && !mon_commands.empty()) {
r = mon_commands.begin()->second;
ldout(cct, 10) << "handle_mon_command_ack has tid 0, assuming it is " << r->tid << dendl;
} else {
map::iterator p = mon_commands.find(tid);
if (p == mon_commands.end()) {
ldout(cct, 10) << "handle_mon_command_ack " << ack->get_tid() << " not found" << dendl;
ack->put();
return;
}
r = p->second;
}
ldout(cct, 10) << "handle_mon_command_ack " << r->tid << " " << r->cmd << dendl;
if (r->poutbl)
r->poutbl->claim(ack->get_data());
_finish_command(r, ack->r, ack->rs);
ack->put();
}
//处理响应请求version的响应消息，将与其想的消息消息从version_requests中清除，并由finisher来执行对应的回调函数
void MonClient::handle_get_version_reply(MMonGetVersionReply* m)
{
assert(monc_lock.is_locked());
map::iterator iter = version_requests.find(m->handle);
if (iter == version_requests.end()) {
ldout(cct, 0) << __func__ << " version request with handle " << m->handle
<< " not found" << dendl;
} else {
version_req_d *req = iter->second;
ldout(cct, 10) << __func__ << " finishing " << req << " version " << m->version << dendl;
version_requests.erase(iter);
if (req->newest)
*req->newest = m->version;
if (req->oldest)
*req->oldest = m->oldest_version;
finisher.queue(req->context, 0);
delete req;
}
m->put();
}
总结：
MonClient是一个与monitor交互模块，主要负责与monitor的交互，实现服务认证，monmap的拉取，以及其他发送给monitor的请求。