帧内预测之前,首先要获取帧内预测参考像素,VTM获取帧内参考像素的函数initIntraPatternChType()
,分成两个步骤,首先获取参考像素,然后对参考像素进行滤波(可选,由参数 bFilterRefSamples
决定),bFilterRefSamples
表示是否满足滤波条件,通过调用函数useFilteredIntraRefSamples()
得到。
/**
* \param bFilterRefSamples: whether to filter the referecne samples
* This function derives the reference samples for intra coding.
*/
void IntraPrediction::initIntraPatternChType(const CodingUnit &cu, const CompArea &area, const bool bFilterRefSamples)
{
const CodingStructure& cs = *cu.cs;
Pel *refBufUnfiltered = m_piYuvExt[area.compID][PRED_BUF_UNFILTERED]; // 未滤波像素首地址
Pel *refBufFiltered = m_piYuvExt[area.compID][PRED_BUF_FILTERED]; // 已滤波像素首地址
// 设置左、上参考像素长度
setReferenceArrayLengths( cu.ispMode && isLuma( area.compID ) ? cu.blocks[area.compID] : area );
// ----- Step 1: unfiltered reference samples -----填充参考像素
xFillReferenceSamples( cs.picture->getRecoBuf( area ), refBufUnfiltered, area, cu );
// ----- Step 2: filtered reference samples ----如果满足滤波条件,对参考像素进行滤波
if( bFilterRefSamples )
{
xFilterReferenceSamples( refBufUnfiltered, refBufFiltered, area, *cs.sps
, cu.firstPU->multiRefIdx
);
}
}
首先,解读一下参考像素的获取函数xFillReferenceSamples()
,VTM参考图像获取需要注意多行预测和ISP预测的区别。在VTM帧内预测工具详解中介绍了VTM帧内预测可使用最多三个参考样本行(0,1,3),如下图
当时说道,为了保证45°到-135°的预测角度,需要获得Segment A
和Segment F
的像素做参考,且Segment A
和Segment F
并不是相邻的已重建像素,而是由临近的Segment B
和Segment E
填充得到。
基于上述基础,xFillReferenceSamples()
填充参考像素主要分为以下几个步骤:
segment D
,segment E
,segment C
,segment B
位置的参考像素
segment E
填充segment F
区域参考像素,使用segment B
填充segment A
区域参考像素详细代码及解释如下
void IntraPrediction::xFillReferenceSamples( const CPelBuf &recoBuf, Pel* refBufUnfiltered, const CompArea &area, const CodingUnit &cu )
{
const ChannelType chType = toChannelType( area.compID );
const CodingStructure &cs = *cu.cs;
const SPS &sps = *cs.sps;
const PreCalcValues &pcv = *cs.pcv;
const int multiRefIdx = (area.compID == COMPONENT_Y) ? cu.firstPU->multiRefIdx : 0; // MRL参考行索引
const int tuWidth = area.width; // 待处理tu宽(ISP模式下,tuWidth为sub block的尺寸)
const int tuHeight = area.height; // 待处理tu高
const int predSize = m_topRefLength; // 上参考像素长度
const int predHSize = m_leftRefLength; // 左参考像素长度
const int cuWidth = cu.blocks[area.compID].width; // tu所属cu宽
const int cuHeight = cu.blocks[area.compID].height; // tu所属cu高
// 参考像素宽高比? isp预测时,为cuWidth/cuHeight,否则为(2*tuWidth) / (2*tuHeight),即tuWidth / tuHeight
const int whRatio = cu.ispMode && isLuma(area.compID) ? std::max(1, cuWidth / cuHeight) : std::max(1, tuWidth / tuHeight);
const int hwRatio = cu.ispMode && isLuma(area.compID) ? std::max(1, cuHeight / cuWidth) : std::max(1, tuHeight / tuWidth);
// predstride =
const int predStride = predSize + 1 + (whRatio + 1) * multiRefIdx;
// 填充帧内参考像素以Unit为基本单元,unit的尺寸计算如下
const bool noShift = pcv.noChroma2x2 && area.width == 4; // don't shift on the lowest level (chroma not-split)
const int unitWidth = tuWidth <= 2 && cu.ispMode && isLuma(area.compID) ? tuWidth : pcv.minCUWidth >> (noShift ? 0 : getComponentScaleX(area.compID, sps.getChromaFormatIdc()));
const int unitHeight = tuHeight <= 2 && cu.ispMode && isLuma(area.compID) ? tuHeight : pcv.minCUHeight >> (noShift ? 0 : getComponentScaleY(area.compID, sps.getChromaFormatIdc()));
const int totalAboveUnits = (predSize + (unitWidth - 1)) / unitWidth; // 上方参考像素全部unit个数
const int totalLeftUnits = (predHSize + (unitHeight - 1)) / unitHeight; // 左侧参考像素全部unit个数
const int totalUnits = totalAboveUnits + totalLeftUnits + 1; // 参考像素全部unit个数,包含左上角一个unit
const int numAboveUnits = std::max<int>( tuWidth / unitWidth, 1 ); // 待处理tu的上边界unit个数
const int numLeftUnits = std::max<int>( tuHeight / unitHeight, 1 ); // 待处理tu的左边界unit个数
const int numAboveRightUnits = totalAboveUnits - numAboveUnits; // 右上unit个数
const int numLeftBelowUnits = totalLeftUnits - numLeftUnits; // 左下unit个数
CHECK( numAboveUnits <= 0 || numLeftUnits <= 0 || numAboveRightUnits <= 0 || numLeftBelowUnits <= 0, "Size not supported" );
// ----- Step 1: analyze neighborhood -----相邻参考像素有效性分析。
const Position posLT = area; // 左上坐标
const Position posRT = area.topRight(); // 右上坐标
const Position posLB = area.bottomLeft(); // 左下坐标
bool neighborFlags[4 * MAX_NUM_PART_IDXS_IN_CTU_WIDTH + 1]; // 参考像素有效性标志,顺序:左下->左上->右上
int numIntraNeighbor = 0; // 有效参考像素unit个数
memset( neighborFlags, 0, totalUnits ); // 初始化为false
/**
* is***Available 相应位置参考像素的有效性判别,其最后一个参数为相应标志位在neighborFlags中所处的位置
* 左侧参考像素扫描顺序为从上到下,而相应标志在neighborFlags的顺序为从下到上(先belowleft,然后left),因此注意is**Avaiable最优一个参数
* 右侧参考像素扫描顺序为从左至右,相应标志在neighbourFlags的顺序也是从左至右(先above,然后aboveright)
**/
neighborFlags[totalLeftUnits] = isAboveLeftAvailable( cu, chType, posLT ); //左上unit是否有效,其在neighborFlags的位置为totalLeftUnits
numIntraNeighbor += neighborFlags[totalLeftUnits] ? 1 : 0; //更新numIntraNeighbor
numIntraNeighbor += isAboveAvailable ( cu, chType, posLT, numAboveUnits, unitWidth, (neighborFlags + totalLeftUnits + 1) );
numIntraNeighbor += isAboveRightAvailable( cu, chType, posRT, numAboveRightUnits, unitWidth, (neighborFlags + totalLeftUnits + 1 + numAboveUnits) );
numIntraNeighbor += isLeftAvailable ( cu, chType, posLT, numLeftUnits, unitHeight, (neighborFlags + totalLeftUnits - 1) );
numIntraNeighbor += isBelowLeftAvailable ( cu, chType, posLB, numLeftBelowUnits, unitHeight, (neighborFlags + totalLeftUnits - 1 - numLeftUnits) );
// ----- Step 2: fill reference samples (depending on neighborhood) -----参考样本填充
CHECK((predHSize + 1) * predStride > m_iYuvExtSize, "Reference sample area not supported");
const Pel* srcBuf = recoBuf.buf; // 当前编码块(pu)重建信号左上第一个元素首地址
const int srcStride = recoBuf.stride;
Pel* ptrDst = refBufUnfiltered; //参考像素首地址,存储顺序:左上、上、右上、左、左下
const Pel* ptrSrc;
const Pel valueDC = 1 << (sps.getBitDepth( chType ) - 1);
// ********** 参考像素全部不可用,全部填充默认值
if( numIntraNeighbor == 0 )
{
// Fill border with DC value
for (int j = 0; j <= predSize + multiRefIdx; j++) { ptrDst[j] = valueDC; } // 用DC填充上方参考
for (int i = 1; i <= predHSize + multiRefIdx; i++) { ptrDst[i*predStride] = valueDC; } // 用DC填充左侧参考
}
// ********** 参考像素全部可用,全部用相邻重建像素填充
else if( numIntraNeighbor == totalUnits )
{
// Fill top-left border and top and top right with rec. samples
ptrSrc = srcBuf - (1 + multiRefIdx) * srcStride - (1 + multiRefIdx); // 指向左上角重建像素
for (int j = 0; j <= predSize + multiRefIdx; j++) { ptrDst[j] = ptrSrc[j]; } // 填充左上、上方参考像素
ptrSrc = srcBuf - multiRefIdx * srcStride - (1 + multiRefIdx); // 指向左上角下方第一个重建像素(左边第一个浅灰,见图)
for (int i = 1; i <= predHSize + multiRefIdx; i++) { ptrDst[i*predStride] = *(ptrSrc); ptrSrc += srcStride; } // 填充左侧参考像素
}
// **********参考像素部分可用
else // reference samples are partially available
{
// Fill top-left sample(s) if available填充左上区域
ptrSrc = srcBuf - (1 + multiRefIdx) * srcStride - (1 + multiRefIdx); // 指向左上重建像素
ptrDst = refBufUnfiltered; // 参考像素首地址
if (neighborFlags[totalLeftUnits]) // 左上的有效性标志索引为totalLeftUnits
{
ptrDst[0] = ptrSrc[0]; // 如果左上重建像素存在,用之填充左上参考像素
for (int i = 1; i <= multiRefIdx; i++) // 多行预测,填充左上浅灰区域参考像素(见上图)
{
ptrDst[i] = ptrSrc[i];
ptrDst[i*predStride] = ptrSrc[i*srcStride];
}
}
// Fill left & below-left samples if available (downwards)正左方,和左下方,由上至下依次填充
ptrSrc += (1 + multiRefIdx) * srcStride; // 重建信号指针移到当前pu最左侧参考
ptrDst += (1 + multiRefIdx) * predStride; // 参考像素指针移到左侧参考像素
for (int unitIdx = totalLeftUnits - 1; unitIdx > 0; unitIdx--) // 以unit为基本单位,遍历左重建像素有效性
//(注意unitIdx > = 0, 少了左下最后一个unit)
{
if (neighborFlags[unitIdx]) // 如果重建像素有效,填充左侧unitHeight个参考像素
{
for (int i = 0; i < unitHeight; i++)
{
ptrDst[i*predStride] = ptrSrc[i*srcStride];
}
}
ptrSrc += unitHeight * srcStride;
ptrDst += unitHeight * predStride;
}
// Fill last below-left sample(s) 左下最后一个unit
if (neighborFlags[0])
{
int lastSample = (predHSize % unitHeight == 0) ? unitHeight : predHSize % unitHeight;
// 剩余参考样点个数(可能不足一个unit)
for (int i = 0; i < lastSample; i++)
{
ptrDst[i*predStride] = ptrSrc[i*srcStride];
}
}
// Fill above & above-right samples if available (left-to-right) 填充上、右上参考像素
ptrSrc = srcBuf - srcStride * (1 + multiRefIdx); // 指向正上方左边第一个重建像素(考虑MRL)
ptrDst = refBufUnfiltered + 1 + multiRefIdx; // 指向上方参考像素位置
for (int unitIdx = totalLeftUnits + 1; unitIdx < totalUnits - 1; unitIdx++) // 从上方第一个,到右上倒数第二个
{
if (neighborFlags[unitIdx])
{
for (int j = 0; j < unitWidth; j++)
{
ptrDst[j] = ptrSrc[j];
}
}
ptrSrc += unitWidth;
ptrDst += unitWidth;
}
// Fill last above-right sample(s)
if (neighborFlags[totalUnits - 1]) // 右上倒数第一个unit剩余参考样点数
{
int lastSample = (predSize % unitWidth == 0) ? unitWidth : predSize % unitWidth;
for (int j = 0; j < lastSample; j++)
{
ptrDst[j] = ptrSrc[j];
}
}
/**
* 至此,使用有效重建像素填充参考像素完毕,下面对剩余未填充的参考像素填充。
**/
// pad from first available down to the last below-left
ptrDst = refBufUnfiltered; // 指向帧内参考首地址
int lastAvailUnit = 0; // 上一个有效的相邻参考单元
if (!neighborFlags[0]) // 如果相邻左下最后一个unit重建值不存在
{
int firstAvailUnit = 1; // 第一个有效unit初始化为1
while (firstAvailUnit < totalUnits && !neighborFlags[firstAvailUnit]) // 找到第一个有效的unit
{
firstAvailUnit++;
}
// first available sample
int firstAvailRow = 0; // 第一个有效的相邻参考行索引
int firstAvailCol = 0; // 第一个有效的相邻参考列索引
if (firstAvailUnit < totalLeftUnits) // 左侧有有效参考像素
{
firstAvailRow = (totalLeftUnits - firstAvailUnit) * unitHeight + multiRefIdx;
}
else if (firstAvailUnit == totalLeftUnits) // 左侧没有,左上角参考像素有效
{
firstAvailRow = multiRefIdx;
}
else // 只有上方有有效参考
{
firstAvailCol = (firstAvailUnit - totalLeftUnits - 1) * unitWidth + 1 + multiRefIdx;
}
const Pel firstAvailSample = ptrDst[firstAvailCol + firstAvailRow * predStride]; // 第一个有效填充的参考样本
// last sample below-left (n.a.)
int lastRow = predHSize + multiRefIdx; // 最左下样本的行索引
// fill left column
for (int i = lastRow; i > firstAvailRow; i--)
{
ptrDst[i*predStride] = firstAvailSample; // 最邻近有效参考填充
}
// fill top row
if (firstAvailCol > 0)
{
for (int j = 0; j < firstAvailCol; j++)
{
ptrDst[j] = firstAvailSample;
}
}
lastAvailUnit = firstAvailUnit; // 更新lastAvailUnit
}
// pad all other reference samples. 填充剩余的空参考像素
int currUnit = lastAvailUnit + 1;
while (currUnit < totalUnits)
{
if (!neighborFlags[currUnit]) // samples not available
{
// last available sample
int lastAvailRow = 0;
int lastAvailCol = 0;
if (lastAvailUnit < totalLeftUnits)
{
lastAvailRow = (totalLeftUnits - lastAvailUnit - 1) * unitHeight + multiRefIdx + 1;
}
else if (lastAvailUnit == totalLeftUnits)
{
lastAvailCol = multiRefIdx;
}
else
{
lastAvailCol = (lastAvailUnit - totalLeftUnits) * unitWidth + multiRefIdx;
}
const Pel lastAvailSample = ptrDst[lastAvailCol + lastAvailRow * predStride];
// fill current unit with last available sample
if (currUnit < totalLeftUnits)
{
for (int i = lastAvailRow - 1; i >= lastAvailRow - unitHeight; i--)
{
ptrDst[i*predStride] = lastAvailSample;
}
}
else if (currUnit == totalLeftUnits)
{
for (int i = 1; i < multiRefIdx + 1; i++)
{
ptrDst[i*predStride] = lastAvailSample;
}
for (int j = 0; j < multiRefIdx + 1; j++)
{
ptrDst[j] = lastAvailSample;
}
}
else
{
int numSamplesInUnit = (currUnit == totalUnits - 1) ? ((predSize % unitWidth == 0) ? unitWidth : predSize % unitWidth) : unitWidth;
for (int j = lastAvailCol + 1; j <= lastAvailCol + numSamplesInUnit; j++)
{
ptrDst[j] = lastAvailSample;
}
}
}
lastAvailUnit = currUnit;
currUnit++;
}
}
// padding of extended samples above right with the last sample
// segment F 用segment E填充
int lastSample = multiRefIdx + predSize;
for (int j = 1; j <= whRatio * multiRefIdx; j++) { ptrDst[lastSample + j] = ptrDst[lastSample]; }
// padding of extended samples below left with the last sample
// segment B 用segment A填充
lastSample = multiRefIdx + predHSize;
for (int i = 1; i <= hwRatio * multiRefIdx; i++) { ptrDst[(lastSample + i)*predStride] = ptrDst[lastSample*predStride]; }
}