mongodb源码分析(二十五)mongos writeback
这里的writeback也许可以翻译成回写,是指发生如下情况,来自mongos对mongod的数据请求,但是请求时发现版本不对了(发生了chunk的迁移)那么这里的请求将得不到响应,这里的请求需要以某种方式回到mongos,然后再次发往正确的mongod,这就是所谓的writeback.下面直接来看代码.先来看一份简化了的插入操作代码.
void receivedInsert(Message& m, CurOp& op) {
if ( handlePossibleShardedMessage( m , 0 ) )//这里判断插入的消息数据是否因为chunk的迁移而不应该在该服务上执行了
return;
checkAndInsert(ns, first);
}
inline bool handlePossibleShardedMessage( Message &m, DbResponse * dbresponse ) {
if( !shardingState.enabled() ) //只有在开启了shard时才有意义
return false;
return _handlePossibleShardedMessage(m, dbresponse);
}
bool _handlePossibleShardedMessage( Message &m, DbResponse* dbresponse ) {
int op = m.operation();
if ( op < 2000 || op >= 3000|| op == dbGetMore )
return false;
DbMessage d(m);
const char *ns = d.getns();
string errmsg;
// We don't care about the version here, since we're returning it later in the writeback
ConfigVersion received, wanted;//判断ns对应版本是否变化了,没有变化返回true,则正常执行操作
if ( shardVersionOk( ns , errmsg, received, wanted ) ) {
return false;
}
//下面已经是因为版本改变之类的信息,使得操作不能继续了
bool getsAResponse = doesOpGetAResponse( op );//查询操作
if( getsAResponse ){//这里查询操作,但是chunk版本已改变,所以返回错误消息
BufBuilder b( 32768 );
b.skip( sizeof( QueryResult ) );
{
BSONObjBuilder bob;
bob.append( "$err", errmsg );
bob.append( "ns", ns );
wanted.addToBSON( bob, "vWanted" );
received.addToBSON( bob, "vReceived" );
BSONObj obj = bob.obj();
b.appendBuf( obj.objdata() , obj.objsize() );
}
QueryResult *qr = (QueryResult*)b.buf();
qr->_resultFlags() = ResultFlag_ErrSet | ResultFlag_ShardConfigStale;
qr->len = b.len();
qr->setOperation( opReply );
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
b.decouple();
Message * resp = new Message();
resp->setData( qr , true );
dbresponse->response = resp;
dbresponse->responseTo = m.header()->id;
return true;
}
OID writebackID;//下面是修改操作,先将这些操作保存起来
writebackID.initSequential();
const OID& clientID = ShardedConnectionInfo::get(false)->getID();
BSONObjBuilder b;
b.appendBool( "writeBack" , true );
b.append( "ns" , ns );
b.append( "id" , writebackID );
b.append( "connectionId" , cc().getConnectionId() );
b.append( "instanceIdent" , prettyHostName() );
wanted.addToBSON( b );
received.addToBSON( b, "yourVersion" );
b.appendBinData( "msg" , m.header()->len , bdtCustom , (char*)(m.singleData()) );
// Don't register the writeback until immediately before we queue it -
// after this line, mongos will wait for an hour if we don't queue correctly
lastError.getSafe()->writeback( writebackID );
writeBackManager.queueWriteBack( clientID.str() , b.obj() );//操作保持到本地queue中
return true;
}
继续这里的shardVersionOk函数.
bool shardVersionOk( const string& ns , string& errmsg, ConfigVersion& received, ConfigVersion& wanted ) {
if ( ! shardingState.enabled() )
return true;
if ( ! isMasterNs( ns.c_str() ) ) {
// right now connections to secondaries aren't versioned at all
return true;
}
ShardedConnectionInfo* info = ShardedConnectionInfo::get( false );
if ( ! info ) {//不存在本地的shard信息,允许该操作继续在本地执行
// this means the client has nothing sharded
// so this allows direct connections to do whatever they want
// which i think is the correct behavior
return true;
}
if ( info->inForceVersionOkMode() ) //版本信息一定是正确的,允许操作继续
return true;
// TODO : all collections at some point, be sharded or not, will have a version
// (and a ShardChunkManager)
received = info->getVersion( ns );//得到当前ns版本信息
wanted = shardingState.getVersion( ns );//从shard中得到信息
//writeCompatible比较major版本,这个版本只会在chunk发生迁移时增长.
if( received.isWriteCompatibleWith( wanted ) ) return true;
// Figure out exactly why not compatible, send appropriate error message
// The versions themselves are returned in the error, so not needed in messages here
// Check epoch first, to send more meaningful message, since other parameters probably
// won't match either
if( ! wanted.hasCompatibleEpoch( received ) )//Epoch版本不匹配,需要缓存操作,以后发回mogos
return false;
if( ! wanted.isSet() && received.isSet() )
return false;
if( wanted.isSet() && ! received.isSet() )
return false;
if( wanted.majorVersion() != received.majorVersion() ){
// Could be > or < - wanted is > if this is the source of a migration,
// wanted < if this is the target of a migration
return false;
}
return false;
}
继续_handlePossibleShardedMessage->queueWriteBack.
void WriteBackManager::queueWriteBack( const string& remote , const BSONObj& o ) {
static mongo::mutex xxx( "WriteBackManager::queueWriteBack tmp" );
static OID lastOID;
scoped_lock lk( xxx );
const BSONElement& e = o["id"];
lastOID = e.OID();
getWritebackQueue( remote )->queue.push( o );//记录到queue中
}
下面我们来看mongos对于这些操作数据的取回过程.前面分析mongos初始化 mongodb源码分析(二十五)mongos分片的配置时就提到过当从pool中得到一个连接时,其回调函数onCreate->initShardVersionCB
bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){
WriteBackListener::init( *conn_in );
这里writeBackListener初始化.继续看这里的init函数.
void WriteBackListener::init( DBClientBase& conn ) {
if ( conn.type() == ConnectionString::SYNC ) {
// don't want write back listeners for config servers
return;
}
if ( conn.type() != ConnectionString::SET ) {//单服务器之间初始化
init( conn.getServerAddress() );
return;
}
{
scoped_lock lk( _cacheLock );
if ( _seenSets.count( conn.getServerAddress() ) )
return;
}
// we want to do writebacks on all rs nodes
string errmsg;
ConnectionString cs = ConnectionString::parse( conn.getServerAddress() , errmsg );
vector<HostAndPort> hosts = cs.getServers();
for ( unsigned i=0; i<hosts.size(); i++ )//多服务器一台一台初始化
init( hosts[i].toString() );
}
void WriteBackListener::init( const string& host ) {
scoped_lock lk( _cacheLock );
WriteBackListener*& l = _cache[host];//建立了的连接缓存起来
if ( l )
return;
l = new WriteBackListener( host );//新建一个writeBackListener
l->go();//这里开启一个线程专门负责从对端host中取出相应的未正确发往目的地的操作,然后发往正确的host
}
下面来看这里的线程,其函数为:
void WriteBackListener::run() {
int secsToSleep = 0;
scoped_ptr<ShardChunkVersion> lastNeededVersion;
int lastNeededCount = 0;
while ( ! inShutdown() ) {
if ( ! Shard::isAShardNode( _addr ) ) {//不是一个shard节点.
sleepsecs( 60 );
continue;
}
try {
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getInternalScopedDbConnection( _addr ) );
BSONObj result;
{
BSONObjBuilder cmd;//发往shard的命令,取回操作
cmd.appendOID( "writebacklisten" , &serverID ); // Command will block for data
if ( ! conn->get()->runCommand( "admin" , cmd.obj() , result ) ) {
result = result.getOwned();
conn->done();
continue;
}
}
conn->done();
BSONObj data = result.getObjectField( "data" );
if ( data.getBoolField( "writeBack" ) ) {//实际取得数据
string ns = data["ns"].valuestrsafe();
ConnectionIdent cid( "" , 0 );
OID wid;
if ( data["connectionId"].isNumber() && data["id"].type() == jstOID ) {
string s = "";
if ( data["instanceIdent"].type() == String )
s = data["instanceIdent"].String();
cid = ConnectionIdent( s , data["connectionId"].numberLong() );
wid = data["id"].OID();
}
int len; // not used, but needed for next call
Message msg( (void*)data["msg"].binData( len ) , false );
DBConfigPtr db = grid.getDBConfig( ns );
ShardChunkVersion needVersion = ShardChunkVersion::fromBSON( data, "version" );
// TODO: Refactor the sharded strategy to correctly handle all sharding state changes itself,
// we can't rely on WBL to do this for us b/c anything could reset our state in-between.
// We should always reload here for efficiency when possible, but staleness is also caught in the
// loop below.
ChunkManagerPtr manager;
ShardPtr primary;
db->getChunkManagerOrPrimary( ns, manager, primary );
ShardChunkVersion currVersion;
if( manager ) currVersion = manager->getVersion();
// We should reload only if we need to update our version to be compatible *and* we
// haven't already done so. This avoids lots of reloading when we remove/add a sharded collection
bool alreadyReloaded = lastNeededVersion &&
lastNeededVersion->isEquivalentTo( needVersion );
if( alreadyReloaded ){}
else if( lastNeededVersion )
lastNeededCount = 0;
// Set our lastNeededVersion for next time
lastNeededVersion.reset( new ShardChunkVersion( needVersion ) );
lastNeededCount++;
// Determine if we should reload, if so, reload
//版本是否兼容,是否需要重新加载配置
bool shouldReload = ! needVersion.isWriteCompatibleWith( currVersion ) &&
! alreadyReloaded;
if( shouldReload && currVersion.isSet()
&& needVersion.isSet()
&& currVersion.hasCompatibleEpoch( needVersion ) )
{
// If we disagree about versions only, reload the chunk manager
db->getChunkManagerIfExists( ns, true );
}
else if( shouldReload ){
// If we disagree about anything else, reload the full db
db->reload();
}
// do request and then call getLastError
// we have to call getLastError so we can return the right fields to the user if they decide to call getLastError
BSONObj gle;
int attempts = 0;
while ( true ) {
attempts++;
try {//这里再次将来操作发往正确的shard中
Request r( msg , 0 );
r.init();
r.d().reservedField() |= Reserved_FromWriteback;
ClientInfo * ci = r.getClientInfo();
if (!noauth) {
// TODO: Figure out why this is 'admin' instead of 'local'.
ci->getAuthenticationInfo()->authorize("admin", internalSecurity.user);
}
ci->noAutoSplit();
r.process( attempts );//再次将该操作发向应该被操作的shard
ci->newRequest(); // this so we flip prev and cur shards
BSONObjBuilder b;
string errmsg;
if ( ! ci->getLastError( "admin",
BSON( "getLastError" << 1 ),
b,
errmsg,
true ) )
{
b.appendBool( "commandFailed" , true );
if( ! b.hasField( "errmsg" ) ){
b.append( "errmsg", errmsg );
gle = b.obj();
}
else if( errmsg.size() > 0 ){
// Rebuild GLE object with errmsg
// TODO: Make this less clumsy by improving GLE interface
gle = b.obj();
if( gle["errmsg"].type() == String ){
BSONObj gleNoErrmsg =
gle.filterFieldsUndotted( BSON( "errmsg" << 1 ),
false );
BSONObjBuilder bb;
bb.appendElements( gleNoErrmsg );
bb.append( "errmsg", gle["errmsg"].String() +
" ::and:: " +
errmsg );
gle = bb.obj().getOwned();
}
}
}
else{
gle = b.obj();
}//9517表示chunk version再次发生了变化,然后这次的发送又失败了,只能取回来
//再次找到正确的shard发往之
if ( gle["code"].numberInt() == 9517 ) {
lastNeededVersion.reset();
lastNeededCount = 1;
// Bringing this in line with the similar retry logic elsewhere
// TODO: Reloading the chunk manager may not help if we dropped a
// collection, but we don't actually have that info in the writeback
// error
if( attempts <= 2 ){
db->getChunkManagerIfExists( ns, true );
}
else{
versionManager.forceRemoteCheckShardVersionCB( ns );
sleepsecs( attempts - 1 );
}
continue;
}
ci->clearSinceLastGetError();
}
break;
}
{
scoped_lock lk( _seenWritebacksLock );
WBStatus& s = _seenWritebacks[cid];
s.id = wid;
s.gle = gle;
}
}
secsToSleep = 0;
continue;
}
secsToSleep++;
sleepsecs(secsToSleep);
if ( secsToSleep > 10 )
secsToSleep = 0;
}
}简单而言就是读取shard端的writeback的操作,然后将其发往正确的shard.
为什么会发生writeback这种状态呢,我猜测如网络延时这种情况发生导致了有些操作发往了错误的shard.
原文链接:mongodb源码分析(二十五)mongos writeback
作者: yhjj0108,杨浩