webrtc视频帧率控制算法机制（二）--目标帧率丢帧

前言

上一篇文章的丢帧是依据编码后的码率和目标码率来决定丢帧，

http://www.jianshu.com/p/e2a9740b9877

而本文介绍的丢帧依据是目标帧率。

由此可对丢帧策略分类如下：

• 编码后的码率和目标码率来决定丢帧
• 目标帧率决定丢帧

调用流程

前处理模块调用丢帧处理

ViEEncoder::DeliverFrame
->int32_t VideoProcessingModuleImpl::PreprocessFrame
->int32_t VPMFramePreprocessor::PreprocessFrame

PreprocessFrame前处理先进行丢帧处理，然后进行重采样处理。丢帧处理中，每来一帧先进行输入帧率统计，然后进行丢帧判断。输入帧率统计和上一篇码率决定丢帧输入丢帧统计算法一样。

int32_t VPMFramePreprocessor::PreprocessFrame(const I420VideoFrame& frame,
    I420VideoFrame** processed_frame) {
  if (frame.IsZeroSize()) {
    return VPM_PARAMETER_ERROR;
  }

  vd_->UpdateIncomingframe_rate(); //更新输入帧率

  if (vd_->DropFrame()) {//判断是否丢帧
    return 1;  // drop 1 frame
  }

  // Resizing incoming frame if needed. Otherwise, remains NULL.
  // We are not allowed to resample the input frame (must make a copy of it).
  *processed_frame = NULL;
  if (spatial_resampler_->ApplyResample(frame.width(), frame.height()))  {
    int32_t ret = spatial_resampler_->ResampleFrame(frame, &resampled_frame_);
    if (ret != VPM_OK) return ret;
    *processed_frame = &resampled_frame_;
  }

  // Perform content analysis on the frame to be encoded.
  if (enable_ca_) {
    // Compute new metrics every |kSkipFramesCA| frames, starting with
    // the first frame.
    if (frame_cnt_ % kSkipFrameCA == 0) {
      if (*processed_frame == NULL)  {
        content_metrics_ = ca_->ComputeContentMetrics(frame);
      } else {
        content_metrics_ = ca_->ComputeContentMetrics(resampled_frame_);
      }
    }
    ++frame_cnt_;
  }
  return VPM_OK;
}```

##输入帧率统计
每来一帧都将此时的时间作为样本，记录在滑动窗口为kFrameCountHistory_size的incoming_frame_times_数组中。并进行ProcessIncomingframe_rate输入帧率的计算。

void VPMVideoDecimator::UpdateIncomingframe_rate() {
int64_t now = TickTime::MillisecondTimestamp();
if (incoming_frame_times_[0] == 0) {
// First no shift.
} else {
// Shift.
for (int i = kFrameCountHistory_size - 2; i >= 0; i--) {
incoming_frame_times_[i+1] = incoming_frame_times_[i];
}
}
incoming_frame_times_[0] = now;
ProcessIncomingframe_rate(now);
}```

统计最多不超过2秒钟的样本，计算输入帧率。

void VPMVideoDecimator::ProcessIncomingframe_rate(int64_t now) {
  int32_t num = 0;
  int32_t nrOfFrames = 0;
  for (num = 1; num < (kFrameCountHistory_size - 1); num++) {
    // Don't use data older than 2sec.
    if (incoming_frame_times_[num] <= 0 ||
        now - incoming_frame_times_[num] > kFrameHistoryWindowMs) {
      break;
    } else {
      nrOfFrames++;
    }
  }
  if (num > 1) {
    int64_t diff = now - incoming_frame_times_[num-1];
    incoming_frame_rate_ = 1.0;
    if (diff > 0) {
      incoming_frame_rate_ = nrOfFrames * 1000.0f / static_cast(diff);
    }
  } else {
    incoming_frame_rate_ = static_cast(nrOfFrames);
  }
}```

##目标帧率丢帧核心
一、当2 * overshoot < (int32_t) incomingframe_rate，即输入帧率大于目标帧率，小于2倍目标帧率的情况下。具体细节看注释，这里假设incomingframe_rate=20，target_frame_rate_=15。具体就是实现均匀丢帧。

bool VPMVideoDecimator::DropFrame() {
if (!enable_temporal_decimation_) return false;

if (incoming_frame_rate_ <= 0) return false;

const uint32_t incomingframe_rate =
static_cast(incoming_frame_rate_ + 0.5f);

if (target_frame_rate_ == 0) return true;

bool drop = false;
if (incomingframe_rate > target_frame_rate_) {//输入帧率大于目标帧率
int32_t overshoot =
overshoot_modifier_ + (incomingframe_rate - target_frame_rate_);//20-15=5，超出帧率
if (overshoot < 0) {
overshoot = 0;
overshoot_modifier_ = 0;
}

if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) {//2*5<20,即20<2*15，输入帧率小于2倍目标帧率
  if (drop_count_) {  // Just got here so drop to be sure.
      drop_count_ = 0;
      return true;
  }
  const uint32_t dropVar = incomingframe_rate / overshoot;//20/5=4，丢帧比率

  if (keep_count_ >= dropVar) {//均匀丢帧
      drop = true;
      overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3;//-（20%5）/3=0，修正overshoot
      keep_count_ = 1;//重置保留帧数
  } else {
      keep_count_++;//保留帧数
  }
} else {
  keep_count_ = 0;
  const uint32_t dropVar = overshoot / target_frame_rate_;
  if (drop_count_ < dropVar) {
      drop = true;
      drop_count_++;
  } else {
      overshoot_modifier_ = overshoot % target_frame_rate_;
      drop = false;
      drop_count_ = 0;
  }
}

}
return drop;
}```

二、当2 * overshoot >=(int32_t) incomingframe_rate时，即输入帧率大于等于2倍目标帧率，此时，均匀丢帧每次丢1帧以上，具体丢帧方法和上一部分略有区别。

bool VPMVideoDecimator::DropFrame() {
  if (!enable_temporal_decimation_) return false;

  if (incoming_frame_rate_ <= 0) return false;

  const uint32_t incomingframe_rate =
      static_cast(incoming_frame_rate_ + 0.5f);

  if (target_frame_rate_ == 0) return true;

  bool drop = false;
  if (incomingframe_rate > target_frame_rate_) {//输入帧率大于目标帧率
    int32_t overshoot =
        overshoot_modifier_ + (incomingframe_rate - target_frame_rate_); //30-10=20，超出帧率
    if (overshoot < 0) {
      overshoot = 0;
      overshoot_modifier_ = 0;
    }

    if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) {  //2*20>30，即30>2*10，输入帧率大于2倍目标帧率
      if (drop_count_) {  // Just got here so drop to be sure.
          drop_count_ = 0;
          return true;
      }
      const uint32_t dropVar = incomingframe_rate / overshoot;

      if (keep_count_ >= dropVar) {
          drop = true;
          overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3;
          keep_count_ = 1;
      } else {
          keep_count_++;
      }
    } else {
      keep_count_ = 0;
      const uint32_t dropVar = overshoot / target_frame_rate_; //20/10=2，丢帧比率
      if (drop_count_ < dropVar) {//一次丢1帧以上
          drop = true;
          drop_count_++;
      } else {
          overshoot_modifier_ = overshoot % target_frame_rate_;  //20%10=0，overshoot修正
          drop = false;
          drop_count_ = 0;
      }
    }
  }
  return drop;
}```