HM编码器代码阅读(17)——帧间预测之一merge模式

入口函数：TEncCu::xCheckRDCostMerge2Nx2N

主要流程：

（1）获取可用merge候选MV数量以及merge候选MV
（2）判断是否为无损模式，如果是无损模式，那么下面的for循环只需迭代一次，如果不是武勋模式，那么下面的for循环要迭代两次
（3）一个for循环，遍历所有的merge模式候选者
     1）设置相关参数
     2）调用motionCompensation进行运动补偿
     3）调用encodeResAndCalcRdInterCU，进行量化变换等操作
     4）调用xCheckDQP，检测量化步长
     5）调用xCheckBestMode，选出最优的模式
     6）调用initEstData，初始化预测数据

// merge模式
Void TEncCu::xCheckRDCostMerge2Nx2N( TComDataCU*& rpcBestCU, TComDataCU*& rpcTempCU, Bool *earlyDetectionSkipMode )
{
	assert( rpcTempCU->getSlice()->getSliceType() != I_SLICE );
	TComMvField  cMvFieldNeighbours[ 2 * MRG_MAX_NUM_CANDS ]; // double length for mv of both lists
	UChar uhInterDirNeighbours[MRG_MAX_NUM_CANDS];
	Int numValidMergeCand = 0;
	const Bool bTransquantBypassFlag = rpcTempCU->getCUTransquantBypass(0);

	for( UInt ui = 0; ui < rpcTempCU->getSlice()->getMaxNumMergeCand(); ++ui )
	{
		uhInterDirNeighbours[ui] = 0;
	}
	UChar uhDepth = rpcTempCU->getDepth( 0 );
	rpcTempCU->setPartSizeSubParts( SIZE_2Nx2N, 0, uhDepth ); // interprets depth relative to LCU level
	// 取出merge候选列表，这是一个重要的函数
	rpcTempCU->getInterMergeCandidates( 0, 0, cMvFieldNeighbours,uhInterDirNeighbours, numValidMergeCand );

	Int mergeCandBuffer[MRG_MAX_NUM_CANDS];
	for( UInt ui = 0; ui < numValidMergeCand; ++ui )
	{
		mergeCandBuffer[ui] = 0;
	}

	Bool bestIsSkip = false;

	UInt iteration;
	if ( rpcTempCU->isLosslessCoded(0))
	{
		iteration = 1;
	}
	else 
	{
		iteration = 2;
	}

	for( UInt uiNoResidual = 0; uiNoResidual < iteration; ++uiNoResidual )
	{
		// 遍历每一个merge候选者
		for( UInt uiMergeCand = 0; uiMergeCand < numValidMergeCand; ++uiMergeCand )
		{
			if(!(uiNoResidual==1 && mergeCandBuffer[uiMergeCand]==1))
			{
				if( !(bestIsSkip && uiNoResidual == 0) )
				{
					// set MC parameters
					// 设置MC参数
					rpcTempCU->setPredModeSubParts( MODE_INTER, 0, uhDepth ); // interprets depth relative to LCU level
					rpcTempCU->setCUTransquantBypassSubParts( bTransquantBypassFlag,     0, uhDepth );
					rpcTempCU->setPartSizeSubParts( SIZE_2Nx2N, 0, uhDepth ); // interprets depth relative to LCU level
					rpcTempCU->setMergeFlagSubParts( true, 0, 0, uhDepth ); // interprets depth relative to LCU level
					rpcTempCU->setMergeIndexSubParts( uiMergeCand, 0, 0, uhDepth ); // interprets depth relative to LCU level
					rpcTempCU->setInterDirSubParts( uhInterDirNeighbours[uiMergeCand], 0, 0, uhDepth ); // interprets depth relative to LCU level
					rpcTempCU->getCUMvField( REF_PIC_LIST_0 )->setAllMvField( cMvFieldNeighbours[0 + 2*uiMergeCand], SIZE_2Nx2N, 0, 0 ); // interprets depth relative to rpcTempCU level
					rpcTempCU->getCUMvField( REF_PIC_LIST_1 )->setAllMvField( cMvFieldNeighbours[1 + 2*uiMergeCand], SIZE_2Nx2N, 0, 0 ); // interprets depth relative to rpcTempCU level

					// do MC
					// 运动估计
					m_pcPredSearch->motionCompensation ( rpcTempCU, m_ppcPredYuvTemp[uhDepth] );
					// estimate residual and encode everything
					// 变换量化等等操作
					m_pcPredSearch->encodeResAndCalcRdInterCU( rpcTempCU,
						m_ppcOrigYuv    [uhDepth],
						m_ppcPredYuvTemp[uhDepth],
						m_ppcResiYuvTemp[uhDepth],
						m_ppcResiYuvBest[uhDepth],
						m_ppcRecoYuvTemp[uhDepth],
						(uiNoResidual? true:false));


					if ( uiNoResidual == 0 && rpcTempCU->getQtRootCbf(0) == 0 )
					{
						// If no residual when allowing for one, then set mark to not try case where residual is forced to 0
						mergeCandBuffer[uiMergeCand] = 1;
					}

					rpcTempCU->setSkipFlagSubParts( rpcTempCU->getQtRootCbf(0) == 0, 0, uhDepth );
					Int orgQP = rpcTempCU->getQP( 0 );
					xCheckDQP( rpcTempCU );
					// 选择最优模式	
					xCheckBestMode(rpcBestCU, rpcTempCU, uhDepth);
					rpcTempCU->initEstData( uhDepth, orgQP, bTransquantBypassFlag );

					if( m_pcEncCfg->getUseFastDecisionForMerge() && !bestIsSkip )
					{
						bestIsSkip = rpcBestCU->getQtRootCbf(0) == 0;
					}
				}
			}
		}

		if(uiNoResidual == 0 && m_pcEncCfg->getUseEarlySkipDetection())
		{
			if(rpcBestCU->getQtRootCbf( 0 ) == 0)
			{
				if( rpcBestCU->getMergeFlag( 0 ))
				{
					*earlyDetectionSkipMode = true;
				}
				else
				{
					Int absoulte_MV=0;
					for ( UInt uiRefListIdx = 0; uiRefListIdx < 2; uiRefListIdx++ )
					{
						if ( rpcBestCU->getSlice()->getNumRefIdx( RefPicList( uiRefListIdx ) ) > 0 )
						{
							TComCUMvField* pcCUMvField = rpcBestCU->getCUMvField(RefPicList( uiRefListIdx ));
							Int iHor = pcCUMvField->getMvd( 0 ).getAbsHor();
							Int iVer = pcCUMvField->getMvd( 0 ).getAbsVer();
							absoulte_MV+=iHor+iVer;
						}
					}

					if(absoulte_MV == 0)
					{
						*earlyDetectionSkipMode = true;
					}
				}
			}
		}
	}
}

merge候选MV的选取，按照左边——>上边——>右上角——>左下角——>左上角的顺序遍历空域上相邻的PU的MV，然后处理时域上参考的预测MV，最后整理

// 选出merge候选列表
Void TComDataCU::getInterMergeCandidates( UInt uiAbsPartIdx, UInt uiPUIdx, TComMvField* pcMvFieldNeighbours, UChar* puhInterDirNeighbours, Int& numValidMergeCand, Int mrgCandIdx )
{
	UInt uiAbsPartAddr = m_uiAbsIdxInLCU + uiAbsPartIdx;
	// 候选MV对应的标记
	Bool abCandIsInter[ MRG_MAX_NUM_CANDS ];

	// 先清空列表
	for( UInt ui = 0; ui < getSlice()->getMaxNumMergeCand(); ++ui )
	{
		abCandIsInter[ui] = false;
		pcMvFieldNeighbours[ ( ui << 1 )     ].setRefIdx(NOT_VALID);
		pcMvFieldNeighbours[ ( ui << 1 ) + 1 ].setRefIdx(NOT_VALID);
	}

	// 获取有效的候选数量
	numValidMergeCand = getSlice()->getMaxNumMergeCand();
	// compute the location of the current PU
	Int xP, yP, nPSW, nPSH;
	this->getPartPosition(uiPUIdx, xP, yP, nPSW, nPSH);

	Int iCount = 0;

	UInt uiPartIdxLT, uiPartIdxRT, uiPartIdxLB;
	PartSize cCurPS = getPartitionSize( uiAbsPartIdx );
	// 转换成索引
	deriveLeftRightTopIdxGeneral( uiAbsPartIdx, uiPUIdx, uiPartIdxLT, uiPartIdxRT );
	deriveLeftBottomIdxGeneral  ( uiAbsPartIdx, uiPUIdx, uiPartIdxLB );

	// 开始进行空域MV预测，处理的顺序是 左边——>上边——>右上角——>左下角——>左上角

	// 左边
	//left
	UInt uiLeftPartIdx = 0;
	TComDataCU* pcCULeft = 0;
	pcCULeft = getPULeft( uiLeftPartIdx, uiPartIdxLB );
	Bool isAvailableA1 = pcCULeft &&
		pcCULeft->isDiffMER(xP -1, yP+nPSH-1, xP, yP) &&
		!( uiPUIdx == 1 && (cCurPS == SIZE_Nx2N || cCurPS == SIZE_nLx2N || cCurPS == SIZE_nRx2N) ) &&
		!pcCULeft->isIntra( uiLeftPartIdx ) ;
	if ( isAvailableA1 )
	{
		abCandIsInter[iCount] = true;
		// get Inter Dir
		puhInterDirNeighbours[iCount] = pcCULeft->getInterDir( uiLeftPartIdx );
		// get Mv from Left
		pcCULeft->getMvField( pcCULeft, uiLeftPartIdx, REF_PIC_LIST_0, pcMvFieldNeighbours[iCount<<1] );
		if ( getSlice()->isInterB() )
		{
			pcCULeft->getMvField( pcCULeft, uiLeftPartIdx, REF_PIC_LIST_1, pcMvFieldNeighbours[(iCount<<1)+1] );
		}
		if ( mrgCandIdx == iCount )
		{
			return;
		}
		iCount ++;
	}

	// early termination
	if (iCount == getSlice()->getMaxNumMergeCand()) 
	{
		return;
	}
	
	// 上方
	// above
	UInt uiAbovePartIdx = 0;
	TComDataCU* pcCUAbove = 0;
	pcCUAbove = getPUAbove( uiAbovePartIdx, uiPartIdxRT );
	Bool isAvailableB1 = pcCUAbove &&
		pcCUAbove->isDiffMER(xP+nPSW-1, yP-1, xP, yP) &&
		!( uiPUIdx == 1 && (cCurPS == SIZE_2NxN || cCurPS == SIZE_2NxnU || cCurPS == SIZE_2NxnD) ) &&
		!pcCUAbove->isIntra( uiAbovePartIdx );
	if ( isAvailableB1 && (!isAvailableA1 || !pcCULeft->hasEqualMotion( uiLeftPartIdx, pcCUAbove, uiAbovePartIdx ) ) )
	{
		abCandIsInter[iCount] = true;
		// get Inter Dir
		puhInterDirNeighbours[iCount] = pcCUAbove->getInterDir( uiAbovePartIdx );
		// get Mv from Left
		pcCUAbove->getMvField( pcCUAbove, uiAbovePartIdx, REF_PIC_LIST_0, pcMvFieldNeighbours[iCount<<1] );
		if ( getSlice()->isInterB() )
		{
			pcCUAbove->getMvField( pcCUAbove, uiAbovePartIdx, REF_PIC_LIST_1, pcMvFieldNeighbours[(iCount<<1)+1] );
		}
		if ( mrgCandIdx == iCount )
		{
			return;
		}
		iCount ++;
	}
	// early termination
	if (iCount == getSlice()->getMaxNumMergeCand()) 
	{
		return;
	}

	// above right
	// 右上角
	UInt uiAboveRightPartIdx = 0;
	TComDataCU* pcCUAboveRight = 0;
	pcCUAboveRight = getPUAboveRight( uiAboveRightPartIdx, uiPartIdxRT );
	Bool isAvailableB0 = pcCUAboveRight &&
		pcCUAboveRight->isDiffMER(xP+nPSW, yP-1, xP, yP) &&
		!pcCUAboveRight->isIntra( uiAboveRightPartIdx );
	if ( isAvailableB0 && ( !isAvailableB1 || !pcCUAbove->hasEqualMotion( uiAbovePartIdx, pcCUAboveRight, uiAboveRightPartIdx ) ) )
	{
		abCandIsInter[iCount] = true;
		// get Inter Dir
		puhInterDirNeighbours[iCount] = pcCUAboveRight->getInterDir( uiAboveRightPartIdx );
		// get Mv from Left
		pcCUAboveRight->getMvField( pcCUAboveRight, uiAboveRightPartIdx, REF_PIC_LIST_0, pcMvFieldNeighbours[iCount<<1] );
		if ( getSlice()->isInterB() )
		{
			pcCUAboveRight->getMvField( pcCUAboveRight, uiAboveRightPartIdx, REF_PIC_LIST_1, pcMvFieldNeighbours[(iCount<<1)+1] );
		}
		if ( mrgCandIdx == iCount )
		{
			return;
		}
		iCount ++;
	}
	// early termination
	if (iCount == getSlice()->getMaxNumMergeCand()) 
	{
		return;
	}

	// 左下角
	//left bottom
	UInt uiLeftBottomPartIdx = 0;
	TComDataCU* pcCULeftBottom = 0;
	pcCULeftBottom = this->getPUBelowLeft( uiLeftBottomPartIdx, uiPartIdxLB );
	Bool isAvailableA0 = pcCULeftBottom &&
		pcCULeftBottom->isDiffMER(xP-1, yP+nPSH, xP, yP) &&
		!pcCULeftBottom->isIntra( uiLeftBottomPartIdx ) ;
	if ( isAvailableA0 && ( !isAvailableA1 || !pcCULeft->hasEqualMotion( uiLeftPartIdx, pcCULeftBottom, uiLeftBottomPartIdx ) ) )
	{
		abCandIsInter[iCount] = true;
		// get Inter Dir
		puhInterDirNeighbours[iCount] = pcCULeftBottom->getInterDir( uiLeftBottomPartIdx );
		// get Mv from Left
		pcCULeftBottom->getMvField( pcCULeftBottom, uiLeftBottomPartIdx, REF_PIC_LIST_0, pcMvFieldNeighbours[iCount<<1] );
		if ( getSlice()->isInterB() )
		{
			pcCULeftBottom->getMvField( pcCULeftBottom, uiLeftBottomPartIdx, REF_PIC_LIST_1, pcMvFieldNeighbours[(iCount<<1)+1] );
		}
		if ( mrgCandIdx == iCount )
		{
			return;
		}
		iCount ++;
	}
	// early termination
	if (iCount == getSlice()->getMaxNumMergeCand()) 
	{
		return;
	}
	// 左上角
	// above left 
	if( iCount < 4 )
	{
		UInt uiAboveLeftPartIdx = 0;
		TComDataCU* pcCUAboveLeft = 0;
		pcCUAboveLeft = getPUAboveLeft( uiAboveLeftPartIdx, uiAbsPartAddr );
		Bool isAvailableB2 = pcCUAboveLeft &&
			pcCUAboveLeft->isDiffMER(xP-1, yP-1, xP, yP) &&
			!pcCUAboveLeft->isIntra( uiAboveLeftPartIdx );
		if ( isAvailableB2 && ( !isAvailableA1 || !pcCULeft->hasEqualMotion( uiLeftPartIdx, pcCUAboveLeft, uiAboveLeftPartIdx ) )
			&& ( !isAvailableB1 || !pcCUAbove->hasEqualMotion( uiAbovePartIdx, pcCUAboveLeft, uiAboveLeftPartIdx ) ) )
		{
			abCandIsInter[iCount] = true;
			// get Inter Dir
			puhInterDirNeighbours[iCount] = pcCUAboveLeft->getInterDir( uiAboveLeftPartIdx );
			// get Mv from Left
			pcCUAboveLeft->getMvField( pcCUAboveLeft, uiAboveLeftPartIdx, REF_PIC_LIST_0, pcMvFieldNeighbours[iCount<<1] );
			if ( getSlice()->isInterB() )
			{
				pcCUAboveLeft->getMvField( pcCUAboveLeft, uiAboveLeftPartIdx, REF_PIC_LIST_1, pcMvFieldNeighbours[(iCount<<1)+1] );
			}
			if ( mrgCandIdx == iCount )
			{
				return;
			}
			iCount ++;
		}
	}
	// early termination
	if (iCount == getSlice()->getMaxNumMergeCand()) 
	{
		return;
	}

	// 如果使用了TMVP（就是时域的MVP（MVP是MV预测）），上面的都是空域MV预测
	if ( getSlice()->getEnableTMVPFlag())
	{
		//>> MTK colocated-RightBottom
		UInt uiPartIdxRB;

		deriveRightBottomIdx( uiPUIdx, uiPartIdxRB );  

		UInt uiAbsPartIdxTmp = g_auiZscanToRaster[uiPartIdxRB];
		UInt uiNumPartInCUWidth = m_pcPic->getNumPartInWidth();

		TComMv cColMv;
		Int iRefIdx;
		Int uiLCUIdx = -1;

		if      ( ( m_pcPic->getCU(m_uiCUAddr)->getCUPelX() + g_auiRasterToPelX[uiAbsPartIdxTmp] + m_pcPic->getMinCUWidth() ) >= m_pcSlice->getSPS()->getPicWidthInLumaSamples() )  // image boundary check
		{
		}
		else if ( ( m_pcPic->getCU(m_uiCUAddr)->getCUPelY() + g_auiRasterToPelY[uiAbsPartIdxTmp] + m_pcPic->getMinCUHeight() ) >= m_pcSlice->getSPS()->getPicHeightInLumaSamples() )
		{
		}
		else
		{
			if ( ( uiAbsPartIdxTmp % uiNumPartInCUWidth < uiNumPartInCUWidth - 1 ) &&           // is not at the last column of LCU 
				( uiAbsPartIdxTmp / uiNumPartInCUWidth < m_pcPic->getNumPartInHeight() - 1 ) ) // is not at the last row    of LCU
			{
				uiAbsPartAddr = g_auiRasterToZscan[ uiAbsPartIdxTmp + uiNumPartInCUWidth + 1 ];
				uiLCUIdx = getAddr();
			}
			else if ( uiAbsPartIdxTmp % uiNumPartInCUWidth < uiNumPartInCUWidth - 1 )           // is not at the last column of LCU But is last row of LCU
			{
				uiAbsPartAddr = g_auiRasterToZscan[ (uiAbsPartIdxTmp + uiNumPartInCUWidth + 1) % m_pcPic->getNumPartInCU() ];
			}
			else if ( uiAbsPartIdxTmp / uiNumPartInCUWidth < m_pcPic->getNumPartInHeight() - 1 ) // is not at the last row of LCU But is last column of LCU
			{
				uiAbsPartAddr = g_auiRasterToZscan[ uiAbsPartIdxTmp + 1 ];
				uiLCUIdx = getAddr() + 1;
			}
			else //is the right bottom corner of LCU                       
			{
				uiAbsPartAddr = 0;
			}
		}


		iRefIdx = 0;
		Bool bExistMV = false;
		UInt uiPartIdxCenter;
		UInt uiCurLCUIdx = getAddr();
		Int dir = 0;
		UInt uiArrayAddr = iCount;
		xDeriveCenterIdx( uiPUIdx, uiPartIdxCenter );
		// 从参考列表中找到预测的MV（xGetColMVP是个比较重要的函数，它的作用是从参考帧中找打MVP）
		bExistMV = uiLCUIdx >= 0 && xGetColMVP( REF_PIC_LIST_0, uiLCUIdx, uiAbsPartAddr, cColMv, iRefIdx );
		if( bExistMV == false )
		{
			bExistMV = xGetColMVP( REF_PIC_LIST_0, uiCurLCUIdx, uiPartIdxCenter, cColMv, iRefIdx );
		}
		if( bExistMV )
		{
			dir |= 1;
			pcMvFieldNeighbours[ 2 * uiArrayAddr ].setMvField( cColMv, iRefIdx );
		}

		// 如果是B帧，那么还需要在list中查找
		if ( getSlice()->isInterB() )
		{
			bExistMV = uiLCUIdx >= 0 && xGetColMVP( REF_PIC_LIST_1, uiLCUIdx, uiAbsPartAddr, cColMv, iRefIdx);
			if( bExistMV == false )
			{
				bExistMV = xGetColMVP( REF_PIC_LIST_1, uiCurLCUIdx, uiPartIdxCenter, cColMv, iRefIdx );
			}
			if( bExistMV )
			{
				dir |= 2;
				pcMvFieldNeighbours[ 2 * uiArrayAddr + 1 ].setMvField( cColMv, iRefIdx );
			}
		}

		if (dir != 0)
		{
			puhInterDirNeighbours[uiArrayAddr] = dir;
			abCandIsInter[uiArrayAddr] = true;

			if ( mrgCandIdx == iCount )
			{
				return;
			}
			iCount++;
		}
	}
	// early termination
	if (iCount == getSlice()->getMaxNumMergeCand()) 
	{
		return;
	}
	UInt uiArrayAddr = iCount;
	UInt uiCutoff = uiArrayAddr;

	// 如果是B帧
	if ( getSlice()->isInterB())
	{
		UInt uiPriorityList0[12] = {0 , 1, 0, 2, 1, 2, 0, 3, 1, 3, 2, 3};
		UInt uiPriorityList1[12] = {1 , 0, 2, 0, 2, 1, 3, 0, 3, 1, 3, 2};

		for (Int idx=0; idx<uiCutoff*(uiCutoff-1) && uiArrayAddr!= getSlice()->getMaxNumMergeCand(); idx++)
		{
			Int i = uiPriorityList0[idx]; Int j = uiPriorityList1[idx];
			if (abCandIsInter[i] && abCandIsInter[j]&& (puhInterDirNeighbours[i]&0x1)&&(puhInterDirNeighbours[j]&0x2))
			{
				abCandIsInter[uiArrayAddr] = true;
				puhInterDirNeighbours[uiArrayAddr] = 3;

				// get Mv from cand[i] and cand[j]
				pcMvFieldNeighbours[uiArrayAddr << 1].setMvField(pcMvFieldNeighbours[i<<1].getMv(), pcMvFieldNeighbours[i<<1].getRefIdx());
				pcMvFieldNeighbours[( uiArrayAddr << 1 ) + 1].setMvField(pcMvFieldNeighbours[(j<<1)+1].getMv(), pcMvFieldNeighbours[(j<<1)+1].getRefIdx());

				Int iRefPOCL0 = m_pcSlice->getRefPOC( REF_PIC_LIST_0, pcMvFieldNeighbours[(uiArrayAddr<<1)].getRefIdx() );
				Int iRefPOCL1 = m_pcSlice->getRefPOC( REF_PIC_LIST_1, pcMvFieldNeighbours[(uiArrayAddr<<1)+1].getRefIdx() );
				if (iRefPOCL0 == iRefPOCL1 && pcMvFieldNeighbours[(uiArrayAddr<<1)].getMv() == pcMvFieldNeighbours[(uiArrayAddr<<1)+1].getMv())
				{
					abCandIsInter[uiArrayAddr] = false;
				}
				else
				{
					uiArrayAddr++;
				}
			}
		}
	}
	// early termination
	if (uiArrayAddr == getSlice()->getMaxNumMergeCand()) 
	{
		return;
	}
	Int iNumRefIdx = (getSlice()->isInterB()) ? min(m_pcSlice->getNumRefIdx(REF_PIC_LIST_0), m_pcSlice->getNumRefIdx(REF_PIC_LIST_1)) : m_pcSlice->getNumRefIdx(REF_PIC_LIST_0);
	Int r = 0;
	Int refcnt = 0;

	// 填充候选的MV
	while (uiArrayAddr < getSlice()->getMaxNumMergeCand())
	{
		abCandIsInter[uiArrayAddr] = true;
		puhInterDirNeighbours[uiArrayAddr] = 1;
		pcMvFieldNeighbours[uiArrayAddr << 1].setMvField( TComMv(0, 0), r);

		if ( getSlice()->isInterB() )
		{
			puhInterDirNeighbours[uiArrayAddr] = 3;
			pcMvFieldNeighbours[(uiArrayAddr << 1) + 1].setMvField(TComMv(0, 0), r);
		}
		uiArrayAddr++;
		if ( refcnt == iNumRefIdx - 1 )
		{
			r = 0;
		}
		else
		{
			++r;
			++refcnt;
		}
	}

	numValidMergeCand = uiArrayAddr;
}