webrtc 代码学习（二十二） audio neteq，待续

audio neteq
作者：LanPZzzz

1. modules\audio_coding\neteq 中

neteq 的发起点有2个，neteq_impl.cc

NetEqImpl::InsertPacket  // 接收数据
NetEqImpl::GetAudio     // 发送数据

2. InsertPacket 的流程，在学习十三中，我们已经简单的看到，这里就不介绍了。我们看下NetEqImpl::InsertPacketInternal 都作了哪些动作

int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header,
                                    rtc::ArrayView payload,
                                    uint32_t receive_timestamp) {
  if (payload.empty()) {
    RTC_LOG_F(LS_ERROR) << "payload is empty";
    return kInvalidPointer;
  }

  生成Packet 放入PacketList 中
  PacketList packet_list;
  // Insert packet in a packet list.
  packet_list.push_back([&rtp_header, &payload] {
    // Convert to Packet.
    Packet packet;
    packet.payload_type = rtp_header.payloadType;
    packet.sequence_number = rtp_header.sequenceNumber;
    packet.timestamp = rtp_header.timestamp;
    packet.payload.SetData(payload.data(), payload.size());
    // Waiting time will be set upon inserting the packet in the buffer.
    RTC_DCHECK(!packet.waiting_time);
    return packet;
  }());

  判断第一次发送，要下面Reset
  bool update_sample_rate_and_channels =
      first_packet_ || (rtp_header.ssrc != ssrc_);

  if (update_sample_rate_and_channels) {
    // Reset timestamp scaling.
    timestamp_scaler_->Reset();
  }

  if (!decoder_database_->IsRed(rtp_header.payloadType)) {
    // Scale timestamp to internal domain (only for some codecs).
    timestamp_scaler_->ToInternal(&packet_list);
  }

  // Store these for later use, since the first packet may very well disappear
  // before we need these values.
  uint32_t main_timestamp = packet_list.front().timestamp;
  uint8_t main_payload_type = packet_list.front().payload_type;
  uint16_t main_sequence_number = packet_list.front().sequence_number;

  // Reinitialize NetEq if it's needed (changed SSRC or first call).
  if (update_sample_rate_and_channels) {
    // Note: |first_packet_| will be cleared further down in this method, once
    // the packet has been successfully inserted into the packet buffer.

    rtcp_.Init(rtp_header.sequenceNumber);

    // Flush the packet buffer and DTMF buffer.
    packet_buffer_->Flush();
    dtmf_buffer_->Flush();

    // Store new SSRC.
    ssrc_ = rtp_header.ssrc;

    // Update audio buffer timestamp.
    sync_buffer_->IncreaseEndTimestamp(main_timestamp - timestamp_);

    // Update codecs.
    timestamp_ = main_timestamp;
  }

  // Update RTCP statistics, only for regular packets.
  rtcp_.Update(rtp_header, receive_timestamp);

  如果nack 打开，就统计nack 内容，如果有丢包情况，就发送nack 重传
  if (nack_enabled_) {
    RTC_DCHECK(nack_);
    if (update_sample_rate_and_channels) {
      nack_->Reset();
    }
    nack_->UpdateLastReceivedPacket(rtp_header.sequenceNumber,
                                    rtp_header.timestamp);
  }

  // Check for RED payload type, and separate payloads into several packets.
  if (decoder_database_->IsRed(rtp_header.payloadType)) {
    if (!red_payload_splitter_->SplitRed(&packet_list)) {
      return kRedundancySplitError;
    }
    // Only accept a few RED payloads of the same type as the main data,
    // DTMF events and CNG.
    red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
    if (packet_list.empty()) {
      return kRedundancySplitError;
    }
  }

  // Check payload types.
  if (decoder_database_->CheckPayloadTypes(packet_list) ==
      DecoderDatabase::kDecoderNotFound) {
    return kUnknownRtpPayloadType;
  }

  RTC_DCHECK(!packet_list.empty());

  // Update main_timestamp, if new packets appear in the list
  // after RED splitting.
  if (decoder_database_->IsRed(rtp_header.payloadType)) {
    timestamp_scaler_->ToInternal(&packet_list);
    main_timestamp = packet_list.front().timestamp;
    main_payload_type = packet_list.front().payload_type;
    main_sequence_number = packet_list.front().sequence_number;
  }

  // Process DTMF payloads. Cycle through the list of packets, and pick out any
  // DTMF payloads found.
  PacketList::iterator it = packet_list.begin();
  while (it != packet_list.end()) {
    const Packet& current_packet = (*it);
    RTC_DCHECK(!current_packet.payload.empty());
    if (decoder_database_->IsDtmf(current_packet.payload_type)) {
      DtmfEvent event;
      int ret = DtmfBuffer::ParseEvent(current_packet.timestamp,
                                       current_packet.payload.data(),
                                       current_packet.payload.size(), &event);
      if (ret != DtmfBuffer::kOK) {
        return kDtmfParsingError;
      }
      if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) {
        return kDtmfInsertError;
      }
      it = packet_list.erase(it);
    } else {
      ++it;
    }
  }

  // Update bandwidth estimate, if the packet is not comfort noise.
  if (!packet_list.empty() &&
      !decoder_database_->IsComfortNoise(main_payload_type)) {
    // The list can be empty here if we got nothing but DTMF payloads.
    AudioDecoder* decoder = decoder_database_->GetDecoder(main_payload_type);
    RTC_DCHECK(decoder);  // Should always get a valid object, since we have
                          // already checked that the payload types are known.
    decoder->IncomingPacket(packet_list.front().payload.data(),
                            packet_list.front().payload.size(),
                            packet_list.front().sequence_number,
                            packet_list.front().timestamp, receive_timestamp);   ==== 这里的IncomingPacket 方法在opus 中是空方法
  }

  可能存在多个解码数据在1个packet 中，这里进行分离，保存到parsed_packet_list 中
  PacketList parsed_packet_list;
  while (!packet_list.empty()) {
    Packet& packet = packet_list.front();
    const DecoderDatabase::DecoderInfo* info =
        decoder_database_->GetDecoderInfo(packet.payload_type);
    if (!info) {
      RTC_LOG(LS_WARNING) << "SplitAudio unknown payload type";
      return kUnknownRtpPayloadType;
    }

    if (info->IsComfortNoise()) {
      // Carry comfort noise packets along.
      parsed_packet_list.splice(parsed_packet_list.end(), packet_list,
                                packet_list.begin());
    } else {
      const auto sequence_number = packet.sequence_number;
      const auto payload_type = packet.payload_type;
      const Packet::Priority original_priority = packet.priority;
      auto packet_from_result = [&](AudioDecoder::ParseResult& result) {
        Packet new_packet;
        new_packet.sequence_number = sequence_number;
        new_packet.payload_type = payload_type;
        new_packet.timestamp = result.timestamp;
        new_packet.priority.codec_level = result.priority;
        new_packet.priority.red_level = original_priority.red_level;
        new_packet.frame = std::move(result.frame);
        return new_packet;
      };

      std::vector results =
          info->GetDecoder()->ParsePayload(std::move(packet.payload),
                                           packet.timestamp);
      if (results.empty()) {
        packet_list.pop_front();
      } else {
        bool first = true;
        for (auto& result : results) {
          RTC_DCHECK(result.frame);
          RTC_DCHECK_GE(result.priority, 0);
          if (first) {
            // Re-use the node and move it to parsed_packet_list.
            packet_list.front() = packet_from_result(result);
            parsed_packet_list.splice(parsed_packet_list.end(), packet_list,
                                      packet_list.begin());
            first = false;
          } else {
            parsed_packet_list.push_back(packet_from_result(result));
          }
        }
      }
    }
  }

  // Calculate the number of primary (non-FEC/RED) packets.
  const int number_of_primary_packets = std::count_if(
      parsed_packet_list.begin(), parsed_packet_list.end(),
      [](const Packet& in) { return in.priority.codec_level == 0; });

  把parsed_packet_list 插入到packet_buffer_ 的 buffer_ 中
  // Insert packets in buffer.
  const int ret = packet_buffer_->InsertPacketList(
      &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_,
      ¤t_cng_rtp_payload_type_, &stats_);
  if (ret == PacketBuffer::kFlushed) {
    // Reset DSP timestamp etc. if packet buffer flushed.
    new_codec_ = true;
    update_sample_rate_and_channels = true;
  } else if (ret != PacketBuffer::kOK) {
    return kOtherError;
  }

  if (first_packet_) {
    first_packet_ = false;
    // Update the codec on the next GetAudio call.
    new_codec_ = true;
  }

  if (current_rtp_payload_type_) {
    RTC_DCHECK(decoder_database_->GetDecoderInfo(*current_rtp_payload_type_))
        << "Payload type " << static_cast(*current_rtp_payload_type_)
        << " is unknown where it shouldn't be";
  }

  if (update_sample_rate_and_channels && !packet_buffer_->Empty()) {
    // We do not use |current_rtp_payload_type_| to |set payload_type|, but
    // get the next RTP header from |packet_buffer_| to obtain the payload type.
    // The reason for it is the following corner case. If NetEq receives a
    // CNG packet with a sample rate different than the current CNG then it
    // flushes its buffer, assuming send codec must have been changed. However,
    // payload type of the hypothetically new send codec is not known.
    const Packet* next_packet = packet_buffer_->PeekNextPacket();
    RTC_DCHECK(next_packet);
    const int payload_type = next_packet->payload_type;
    size_t channels = 1;
    if (!decoder_database_->IsComfortNoise(payload_type)) {
      AudioDecoder* decoder = decoder_database_->GetDecoder(payload_type);
      assert(decoder);  // Payloads are already checked to be valid.
      channels = decoder->Channels();
    }
    const DecoderDatabase::DecoderInfo* decoder_info =
        decoder_database_->GetDecoderInfo(payload_type);
    assert(decoder_info);
    if (decoder_info->SampleRateHz() != fs_hz_ ||
        channels != algorithm_buffer_->Channels()) {
      SetSampleRateAndChannels(decoder_info->SampleRateHz(), channels);
    }
    if (nack_enabled_) {
      RTC_DCHECK(nack_);
      // Update the sample rate even if the rate is not new, because of Reset().
      nack_->UpdateSampleRate(fs_hz_);
    }
  }

  // TODO(hlundin): Move this code to DelayManager class.
  const DecoderDatabase::DecoderInfo* dec_info =
      decoder_database_->GetDecoderInfo(main_payload_type);
  assert(dec_info);  // Already checked that the payload type is known.
  delay_manager_->LastDecodedWasCngOrDtmf(dec_info->IsComfortNoise() ||
                                          dec_info->IsDtmf());
  if (delay_manager_->last_pack_cng_or_dtmf() == 0) {
    // Calculate the total speech length carried in each packet.
    if (number_of_primary_packets > 0) {
      const size_t packet_length_samples =
          number_of_primary_packets * decoder_frame_length_;
      if (packet_length_samples != decision_logic_->packet_length_samples()) {
        decision_logic_->set_packet_length_samples(packet_length_samples);
        delay_manager_->SetPacketAudioLength(
            rtc::dchecked_cast((1000 * packet_length_samples) / fs_hz_));
      }
    }

    // Update statistics.
    if ((int32_t)(main_timestamp - timestamp_) >= 0 && !new_codec_) {
      // Only update statistics if incoming packet is not older than last played
      // out packet, and if new codec flag is not set.
      delay_manager_->Update(main_sequence_number, main_timestamp, fs_hz_);
    }
  } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) {
    // This is first "normal" packet after CNG or DTMF.
    // Reset packet time counter and measure time until next packet,
    // but don't update statistics.
    delay_manager_->set_last_pack_cng_or_dtmf(0);
    delay_manager_->ResetPacketIatCount();
  }
  return 0;
}

3. NetEqImpl::GetAudioInternal 获取数据jitter 控制其播放，应该是用10ms 的速度控制的，复杂的一笔。。。 待续

-> NetEqImpl::GetDecision
-> NetEqImpl::ExtractPackets
-> NetEqImpl::Decode

int NetEqSrcImpl::ExtractPackets(size_t required_samples,
                              PacketList* packet_list) {
  bool first_packet = true;
  uint8_t prev_payload_type = 0;
  uint32_t prev_timestamp = 0;
  uint16_t prev_sequence_number = 0;
  bool next_packet_available = false;

  获取下一个包的
  const Packet* next_packet = packet_buffer_->PeekNextPacket();
  RTC_DCHECK(next_packet);
  if (!next_packet) {
    RTC_LOG(LS_ERROR) << "Packet buffer unexpectedly empty.";
    return -1;
  }
  
  获取下一个包的时间戳，设置为第一个包的时间，通过这个进行计算
  uint32_t first_timestamp = next_packet->timestamp;
  size_t extracted_samples = 0;

  // Packet extraction loop.
  do {
    timestamp_ = next_packet->timestamp;
    拿出下一个包
    absl::optional packet = packet_buffer_->GetNextPacket();
    // |next_packet| may be invalid after the |packet_buffer_| operation.
    next_packet = nullptr;
    if (!packet) {
      RTC_LOG(LS_ERROR) << "Should always be able to extract a packet here";
      assert(false);  // Should always be able to extract a packet here.
      return -1;
    }
    
    计算出等待解码的时间
    const uint64_t waiting_time_ms = packet->waiting_time->ElapsedMs();
    stats_.StoreWaitingTime(waiting_time_ms);
    RTC_DCHECK(!packet->empty());

    if (first_packet) {
      first_packet = false;
      if (nack_enabled_) {
        RTC_DCHECK(nack_);
        // TODO(henrik.lundin): Should we update this for all decoded packets?
        nack_->UpdateLastDecodedPacket(packet->sequence_number,
                                       packet->timestamp);
      }
      prev_sequence_number = packet->sequence_number;
      prev_timestamp = packet->timestamp;
      prev_payload_type = packet->payload_type;
    }

    const bool has_cng_packet =
        decoder_database_->IsComfortNoise(packet->payload_type);
    // Store number of extracted samples.
    size_t packet_duration = 0;
    if (packet->frame) {
      packet_duration = packet->frame->Duration();
      // TODO(ossu): Is this the correct way to track Opus FEC packets?
      if (packet->priority.codec_level > 0) {
        stats_.SecondaryDecodedSamples(
            rtc::dchecked_cast(packet_duration));
      }
    } else if (!has_cng_packet) {
      RTC_LOG(LS_WARNING) << "Unknown payload type "
                          << static_cast(packet->payload_type);
      RTC_NOTREACHED();
    }

    if (packet_duration == 0) {
      // Decoder did not return a packet duration. Assume that the packet
      // contains the same number of samples as the previous one.
      packet_duration = decoder_frame_length_;
    }
    
    计算出 要解码的采样值（能够播放多长字节）
    extracted_samples = packet->timestamp - first_timestamp + packet_duration;

   设置jitterBufferDelay， （
   lifetime_stats_.jitter_buffer_delay_ms += waiting_time_ms * num_samples;
  ） 能够播放多长时间。
  根据audio 采样，播放多长时间的字节数是固定的，所以audio 主要计算输出到音频设备的字节数就知道过去多长时间
    stats_.JitterBufferDelay(extracted_samples, waiting_time_ms);

   把包保存到packet_list 中
    packet_list->push_back(std::move(*packet));  // Store packet in list.
    packet = absl::nullopt;  // Ensure it's never used after the move.

    // Check what packet is available next.
    next_packet = packet_buffer_->PeekNextPacket();
    next_packet_available = false;
    如果不是连续的，就退出
    if (next_packet && prev_payload_type == next_packet->payload_type &&
        !has_cng_packet) {
      int16_t seq_no_diff = next_packet->sequence_number - prev_sequence_number;
      size_t ts_diff = next_packet->timestamp - prev_timestamp;
      if (seq_no_diff == 1 ||
          (seq_no_diff == 0 && ts_diff == decoder_frame_length_)) {
        // The next sequence number is available, or the next part of a packet
        // that was split into pieces upon insertion.
        next_packet_available = true;
      }
      prev_sequence_number = next_packet->sequence_number;
    }
    超过需要的字节数 或者 不是连续的就退出
  } while (extracted_samples < required_samples && next_packet_available);

  if (extracted_samples > 0) {
    // Delete old packets only when we are going to decode something. Otherwise,
    // we could end up in the situation where we never decode anything, since
    // all incoming packets are considered too old but the buffer will also
    // never be flooded and flushed.
    packet_buffer_->DiscardAllOldPackets(timestamp_, &stats_);
  }

  return rtc::dchecked_cast(extracted_samples);
}

4. 参照，感谢

webRTC中音频相关的netEQ（一）：概述
webRTC中音频相关的netEQ（二）：数据结构
webRTC中音频相关的netEQ（三）：存取包和延时计算
webRTC中音频相关的netEQ（四）：控制命令决策
webRTC中音频相关的netEQ（五）：DSP处理