H.266/VVC VTM阅读7-帧内搜索过程
帧内搜索主要是对预测模式的搜索，此处记录下相关函数。执行帧内模式搜索的函数是 IntraSearch::estIntraPredLumaQT()和IntraSearch::estIntraPredChromaQT()，它们都在EncCu::xCheckRDCostIntra()中被调用，用于决策是否使用帧内相关的模式。下面以IntraSearch::estIntraPredLumaQT()为例记录下帧内搜索过程。
1、建立列表
首先定了四个列表，列表长度均为FAST_UDI_MAX_RDMODE_NUM，FAST_UDI_MAX_RDMODE_NUM是常规帧内模式(67种)和MIP模式(32种)之和，值为99。。uiHadModeList储存了预测误差（sad和satd的较小值）最小的几种模式的顺序排列，CandHadList储存对应误差值。uiRdModeList储存了rd cost（估计值，未进行rd计算）最小的几种模式的顺序排列，CandCostList储存对应cost值。
static_vector uiHadModeList;
static_vector CandHadList;
static_vector uiRdModeList;
static_vector CandCostList;
模式信息ModeInfo结构拥有以下几种变量，是否是mip模式，是否是mip转置模式，是否使用多参考行模式MRL，是否使用帧内子块变化ISP，以及预测模式索引。
struct ModeInfo
{
bool mipFlg; // CU::mipFlag
bool mipTrFlg; // PU::mipTransposedFlag
int mRefId; // PU::multiRefIdx
uint8_t ispMod; // CU::ispMode
uint32_t modeId; // PU::intraDir[CHANNEL_TYPE_LUMA]
ModeInfo() : mipFlg(false), mipTrFlg(false), mRefId(0), ispMod(NOT_INTRA_SUBPARTITIONS), modeId(0) {}
ModeInfo(const bool mipf, const bool miptf, const int mrid, const uint8_t ispm, const uint32_t mode) : mipFlg(mipf), mipTrFlg(miptf), mRefId(mrid), ispMod(ispm), modeId(mode) {}
bool operator==(const ModeInfo cmp) const { return (mipFlg == cmp.mipFlg && mipTrFlg == cmp.mipTrFlg && mRefId == cmp.mRefId && ispMod == cmp.ispMod && modeId == cmp.modeId); }
};
struct M
2、通过预测值决策候选模式
决策候选模式顺序是常规角度模式、扩展角度模式、MRL模式、MIP模式、MPM模式、ISP模式，以常规模式为例记录下计算过程。遍历67种亮度模式，并跳过扩展角度模式。调用predIntraAng()函数计算对应模式的角度预测值，然后然后调用对应函数计算SAD、HAD、rd cost并更新列表。最后如果需要储存候选列表在二次变换中使用，那么将当前的候选列表储存下来。
if (!LFNSTLoadFlag)
{
for (int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++)
{
uint32_t uiMode = modeIdx;
Distortion minSadHad = 0;
// Skip checking extended Angular modes in the first round of SATD
if (uiMode > DC_IDX && (uiMode & 1))
{
continue;
}
bSatdChecked[uiMode] = true;
pu.intraDir[0] = modeIdx;
initPredIntraParams(pu, pu.Y(), sps);
predIntraAng(COMPONENT_Y, piPred, pu);
// Use the min between SAD and HAD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx( Ctx::MipFlag, ctxStartMipFlag );
m_CABACEstimator->getCtx() = SubCtx( Ctx::ISPMode, ctxStartIspMode );
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
CandCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), double(minSadHad),
uiHadModeList, CandHadList, numHadCand);
}
if (!sps.getUseMIP() && LFNSTSaveFlag)
{
// save found best modes
m_uiSavedNumRdModesLFNST = numModesForFullRD;
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = CandCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
m_dSavedHadListLFNST = CandHadList;
LFNSTSaveFlag = false;
}
}
3、通过rdo决策最佳模式
根据模式调用xIntraCodingLumaISP()等不同函数完成rd cost计算，cost值储存在csTemp->cost种，据此选择出最佳模式。
bool tmpValidReturn = false;
if( cu.ispMode )
{
if ( m_pcEncCfg->getUseFastISP() )
{
m_modeCtrl->setISPWasTested(true);
}
tmpValidReturn = xIntraCodingLumaISP(*csTemp, subTuPartitioner, bestCurrentCost);
if (csTemp->tus.size() == 0)
{
// no TUs were coded
csTemp->cost = MAX_DOUBLE;
continue;
}
// we save the data for future tests
m_ispTestedModes[m_curIspLfnstIdx].setModeResults((ISPType)cu.ispMode, (int)uiOrgMode.modeId, (int)csTemp->tus.size(), csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] ? csTemp->cost : MAX_DOUBLE, csBest->cost);
csTemp->cost = !tmpValidReturn ? MAX_DOUBLE : csTemp->cost;
}
else
{
if (cu.colorTransform)
{
tmpValidReturn = xRecurIntraCodingACTQT(*csTemp, partitioner, mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst);
}
else
{
tmpValidReturn = xRecurIntraCodingLumaQT(
*csTemp, partitioner, uiBestPUMode.ispMod ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP,
uiBestPUMode.ispMod, mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst);
}
}
...
if( tmpValidReturn )
{
...
// check r-d cost
if( csTemp->cost < csBest->cost )
{
std::swap( csTemp, csBest );
uiBestPUMode = uiOrgMode;
bestBDPCMMode = cu.bdpcmMode;
if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
{
m_bestModeCostStore[ lfnstIdx ] = csBest->cost; //cs.cost;
m_bestModeCostValid[ lfnstIdx ] = true;
}
if( csBest->cost < bestCurrentCost )
{
bestCurrentCost = csBest->cost;
}
if ( cu.ispMode )
{
m_modeCtrl->setIspCost(csBest->cost);
bestLfnstIdx = cu.lfnstIdx;
}
else if ( testISP )
{
m_modeCtrl->setMtsFirstPassNoIspCost(csBest->cost);
}
}
if( !cu.ispMode && !cu.bdpcmMode && csBest->cost < bestCostNonBDPCM )
{
bestCostNonBDPCM = csBest->cost;
}
}