webrtc-m79-视频采集器到视频编码器流水线的建立

1 问题

视频采集器采集到像素格式的VideoFrame经过编码后才能通过P2PtransportChannel传递给对端,这其中就包括了两个主要环节:

第一个环节:采集器到编码器之间流水线的建立;

第二个环节:采集到的像素格式的VideoFrame沿着流水线送到编码器,并发送到对端;

2 采集器到编码器之间流水线的建立

webrtc-m79-视频采集器到视频编码器流水线的建立_第1张图片

 

 

3 相关代码

3.1 采集器到编码器之间流水线的建立


 VideoTrack 到 编码器 流水线的建立


void Conductor::AddTracks()
===>
  rtc::scoped_refptr video_device = CapturerTrackSource::Create();
  if (video_device) {
    rtc::scoped_refptr video_track_( // video_track_ 实际指向的是 webrtc::VideoTrack
        peer_connection_factory_->CreateVideoTrack(kVideoLabel, video_device));
    main_wnd_->StartLocalRenderer(video_track_);  建立了 VideoTrack 到本地渲染的流水线

    result_or_error = peer_connection_->AddTrack(video_track_, {kStreamId});  建立了 VideoTrack 到编码器的流水线
    if (!result_or_error.ok()) {
      RTC_LOG(LS_ERROR) << "Failed to add video track to PeerConnection: "
                        << result_or_error.error().message();
    }
  } else {
    RTC_LOG(LS_ERROR) << "OpenVideoCaptureDevice failed";
  }
  
  

/// 将 audio 或者 video track 添加到 PC
RTCErrorOr> PeerConnection::AddTrack(
    rtc::scoped_refptr track,
    const std::vector& stream_ids) {
  RTC_DCHECK_RUN_ON(signaling_thread());
  TRACE_EVENT0("webrtc", "PeerConnection::AddTrack");
  if (!track) {
    LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_PARAMETER, "Track is null.");
  }
  if (!(track->kind() == MediaStreamTrackInterface::kAudioKind ||
        track->kind() == MediaStreamTrackInterface::kVideoKind)) {
    LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_PARAMETER,
                         "Track has invalid kind: " + track->kind());
  }
  if (IsClosed()) {
    LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_STATE,
                         "PeerConnection is closed.");
  }
  if (FindSenderForTrack(track)) {
    LOG_AND_RETURN_ERROR(
        RTCErrorType::INVALID_PARAMETER,
        "Sender already exists for track " + track->id() + ".");
  }
  auto sender_or_error =
      (IsUnifiedPlan() ? AddTrackUnifiedPlan(track, stream_ids) ///针对视频, AddTrackUnifiedPlan 返回的实际上就是 VideoRtpSender 
                       : AddTrackPlanB(track, stream_ids)); ///
  if (sender_or_error.ok()) {
    UpdateNegotiationNeeded();
    stats_->AddTrack(track);
  }
  return sender_or_error;
}


RTCErrorOr>
PeerConnection::AddTrackUnifiedPlan(
    rtc::scoped_refptr track,
    const std::vector& stream_ids) {
  auto transceiver = FindFirstTransceiverForAddedTrack(track);
  if (transceiver) {
    RTC_LOG(LS_INFO) << "Reusing an existing "
                     << cricket::MediaTypeToString(transceiver->media_type())
                     << " transceiver for AddTrack.";
    if (transceiver->direction() == RtpTransceiverDirection::kRecvOnly) {
      transceiver->internal()->set_direction(
          RtpTransceiverDirection::kSendRecv);
    } else if (transceiver->direction() == RtpTransceiverDirection::kInactive) {
      transceiver->internal()->set_direction(
          RtpTransceiverDirection::kSendOnly);
    }
    transceiver->sender()->SetTrack(track);
    transceiver->internal()->sender_internal()->set_stream_ids(stream_ids);
  } else { / 运行到这个分支当中 
    cricket::MediaType media_type =
        (track->kind() == MediaStreamTrackInterface::kAudioKind
             ? cricket::MEDIA_TYPE_AUDIO
             : cricket::MEDIA_TYPE_VIDEO);
    RTC_LOG(LS_INFO) << "Adding " << cricket::MediaTypeToString(media_type)
                     << " transceiver in response to a call to AddTrack.";
    std::string sender_id = track->id();
    // Avoid creating a sender with an existing ID by generating a random ID.
    // This can happen if this is the second time AddTrack has created a sender
    // for this track.
    if (FindSenderById(sender_id)) {
      sender_id = rtc::CreateRandomUuid();
    }
    auto sender = CreateSender(media_type, sender_id, track, stream_ids, {}); ///
    auto receiver = CreateReceiver(media_type, rtc::CreateRandomUuid());
    transceiver = CreateAndAddTransceiver(sender, receiver); /// 
    transceiver->internal()->set_created_by_addtrack(true);
    transceiver->internal()->set_direction(RtpTransceiverDirection::kSendRecv);
  }
  return transceiver->sender(); /// 针对视频返回的实际上就是 VideoRtpSender
}



PeerConnection::CreateSende

rtc::scoped_refptr>
PeerConnection::CreateSender(
    cricket::MediaType media_type,
    const std::string& id,
    rtc::scoped_refptr track, /// 如果是视频,则 track 实际指向的是 webrtc::VideoTrack
    const std::vector& stream_ids, / webrtc::VideoTrack 也继承自 MediaStreamTrackInterface 
    const std::vector& send_encodings) {
  RTC_DCHECK_RUN_ON(signaling_thread());
  rtc::scoped_refptr> sender;
  if (media_type == cricket::MEDIA_TYPE_AUDIO) {  音频
    RTC_DCHECK(!track ||
               (track->kind() == MediaStreamTrackInterface::kAudioKind));
    sender = RtpSenderProxyWithInternal::Create(
        signaling_thread(),
        AudioRtpSender::Create(worker_thread(), id, stats_.get(), this));
    NoteUsageEvent(UsageEvent::AUDIO_ADDED);
  } else {
    RTC_DCHECK_EQ(media_type, cricket::MEDIA_TYPE_VIDEO); // 视频
    RTC_DCHECK(!track ||
               (track->kind() == MediaStreamTrackInterface::kVideoKind));
    sender = RtpSenderProxyWithInternal::Create( // 
        signaling_thread(), VideoRtpSender::Create(worker_thread(), id, this));  
    NoteUsageEvent(UsageEvent::VIDEO_ADDED);
  }
  bool set_track_succeeded = sender->SetTrack(track);  VideoRtpSender::SetTrack 实际上就是其父类 RtpSenderBase::SetTrack
  RTC_DCHECK(set_track_succeeded);
  sender->internal()->set_stream_ids(stream_ids); //
  sender->internal()->set_init_send_encodings(send_encodings);
  return sender;
}

				rtc::scoped_refptr VideoRtpSender::Create(
					rtc::Thread* worker_thread,
					const std::string& id,
					SetStreamsObserver* set_streams_observer) {
				  return rtc::scoped_refptr(
					  new rtc::RefCountedObject(worker_thread, id, 
																set_streams_observer));
				}

/// 如果是视频,则 track 实际指向的是 webrtc::VideoTrack
bool RtpSenderBase::SetTrack(MediaStreamTrackInterface* track) {
  TRACE_EVENT0("webrtc", "RtpSenderBase::SetTrack");
  if (stopped_) {
    RTC_LOG(LS_ERROR) << "SetTrack can't be called on a stopped RtpSender.";
    return false;
  }
  if (track && track->kind() != track_kind()) {
    RTC_LOG(LS_ERROR) << "SetTrack with " << track->kind()
                      << " called on RtpSender with " << track_kind()
                      << " track.";
    return false;
  }

  // Detach from old track.
  if (track_) {
    DetachTrack();
    track_->UnregisterObserver(this);
    RemoveTrackFromStats();
  }

  // Attach to new track.
  bool prev_can_send_track = can_send_track();
  // Keep a reference to the old track to keep it alive until we call SetSend.
  rtc::scoped_refptr old_track = track_;
  track_ = track; // rtc::scoped_refptr track_;
  if (track_) {
    track_->RegisterObserver(this); / Notifier::RegisterObserver
    AttachTrack();  多态,这里是子类的调用 VideoRtpSender::AttachTrack 
  }

  // Update channel.
  if (can_send_track()) {
    SetSend(); /// 多态,这里是子类的调用 VideoRtpSender::SetSend
    AddTrackToStats();
  } else if (prev_can_send_track) {
    ClearSend();
  }
  attachment_id_ = (track_ ? GenerateUniqueId() : 0);
  return true;
}



void VideoRtpSender::SetSend() {
  RTC_DCHECK(!stopped_);
  RTC_DCHECK(can_send_track());
  if (!media_channel_) {
    RTC_LOG(LS_ERROR) << "SetVideoSend: No video channel exists.";
    return;
  }
  cricket::VideoOptions options; / video_track() 返回的是实际上指向 webrtc::VideoTrack 的指针
  VideoTrackSourceInterface* source = video_track()->GetSource(); / VideoTrack::GetSource 直接定义到了头文件中
  if (source) {
    options.is_screencast = source->is_screencast();
    options.video_noise_reduction = source->needs_denoising();
  }
  switch (cached_track_content_hint_) {
    case VideoTrackInterface::ContentHint::kNone:
      break;
    case VideoTrackInterface::ContentHint::kFluid:
      options.is_screencast = false;
      break;
    case VideoTrackInterface::ContentHint::kDetailed:
    case VideoTrackInterface::ContentHint::kText:
      options.is_screencast = true;
      break;
  }
  bool success = worker_thread_->Invoke(RTC_FROM_HERE, [&] {
    return video_media_channel()->SetVideoSend(ssrc_, &options, video_track()); / video_track() 返回的是实际上指向 webrtc::VideoTrack 的指针
  }); // WebRtcVideoChannel::SetVideoSend
  RTC_DCHECK(success);
}



							bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp)
							===>
							webrtc::VideoSendStream::Config config(this, media_transport()); // 
							
							  WebRtcVideoSendStream* stream = new WebRtcVideoSendStream(  
								  call_, sp, std::move(config), default_send_options_, /// 注意这里的 config // 注意里面的成员 config.send_transport 
								  video_config_.enable_cpu_adaptation, bitrate_config_.max_bitrate_bps, /// 也就是说此处的 
								  send_codec_, send_rtp_extensions_, send_params_);

							  uint32_t ssrc = sp.first_ssrc();
  
							send_streams_[ssrc] = stream; 



										VideoSendStream::Config::Config(Transport* send_transport,  send_transport 实际指向的是 WebRtcVideoChannel 
																		MediaTransportInterface* media_transport)
											: rtp(),
											  encoder_settings(VideoEncoder::Capabilities(rtp.lntf.enabled)),
											  send_transport(send_transport), // send_transport 实际指向的是 WebRtcVideoChannel 
											  media_transport(media_transport) {}



							/// VideoTrack::GetSource 
							class VideoTrack : public MediaStreamTrack,
											   public rtc::VideoSourceBase,
											   public ObserverInterface {
							 public: / video_source_ 实际上指向的是 webrtc::VideoTrackSource 的子类,这里指向的是 webrtc::CapturerTrackSource 
							  VideoTrackSourceInterface* GetSource() const override {
								return video_source_.get();
							  }


							bool WebRtcVideoChannel::SetVideoSend(
								uint32_t ssrc,
								const VideoOptions* options,
								rtc::VideoSourceInterface* source) { /// source 是指向 webrtc::VideoTrack 的指针
							  RTC_DCHECK_RUN_ON(&thread_checker_);
							  TRACE_EVENT0("webrtc", "SetVideoSend");
							  RTC_DCHECK(ssrc != 0);
							  RTC_LOG(LS_INFO) << "SetVideoSend (ssrc= " << ssrc << ", options: "
											   << (options ? options->ToString() : "nullptr")
											   << ", source = " << (source ? "(source)" : "nullptr") << ")";
								// WebRtcVideoChannel::AddSendStream 中向 send_streams_ 中加入数据
							  const auto& kv = send_streams_.find(ssrc); / WebRtcVideoChannel 的成员变量  std::map send_streams_ RTC_GUARDED_BY(thread_checker_);
							  if (kv == send_streams_.end()) {
								// Allow unknown ssrc only if source is null.
								RTC_CHECK(source == nullptr);
								RTC_LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc;
								return false;
							  }

							  return kv->second->SetVideoSend(options, source); // WebRtcVideoSendStream::SetVideoSend
							}



bool WebRtcVideoChannel::WebRtcVideoSendStream::SetVideoSend(
    const VideoOptions* options,
    rtc::VideoSourceInterface* source) { // /// source 是指向 webrtc::VideoTrack 的指针 
  TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetVideoSend");
  RTC_DCHECK_RUN_ON(&thread_checker_);

  if (options) {
    VideoOptions old_options = parameters_.options;
    parameters_.options.SetAll(*options);
    if (parameters_.options.is_screencast.value_or(false) !=
            old_options.is_screencast.value_or(false) &&
        parameters_.codec_settings) {
      // If screen content settings change, we may need to recreate the codec
      // instance so that the correct type is used.

      SetCodec(*parameters_.codec_settings);
      // Mark screenshare parameter as being updated, then test for any other
      // changes that may require codec reconfiguration.
      old_options.is_screencast = options->is_screencast;
    }
    if (parameters_.options != old_options) {
      ReconfigureEncoder();
    }
  }

  if (source_ && stream_) { / webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_); 实际指向的是 webrtc::internal::VideoSendStream
    stream_->SetSource(nullptr, webrtc::DegradationPreference::DISABLED);
  } // 注意:class webrtc::internal::VideoSendStream : public webrtc::VideoSendStream
  // Switch to the new source.
  source_ = source; // rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&thread_checker_);
  if (source && stream_) {
    stream_->SetSource(this, GetDegradationPreference()); /// 多态,实际上是子类的调用 webrtc::internal::VideoSendStream::SetSource 
  }
  return true;
}





						WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream(
							webrtc::Call* call,
							const StreamParams& sp,
							webrtc::VideoSendStream::Config config, ///
							const VideoOptions& options,
							bool enable_cpu_overuse_detection,
							int max_bitrate_bps,
							const absl::optional& codec_settings,
							const absl::optional>& rtp_extensions,
							// TODO(deadbeef): Don't duplicate information between send_params,
							// rtp_extensions, options, etc.
							const VideoSendParameters& send_params)
							: worker_thread_(rtc::Thread::Current()),
							  ssrcs_(sp.ssrcs),
							  ssrc_groups_(sp.ssrc_groups),
							  call_(call),
							  enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
							  source_(nullptr),
							  stream_(nullptr),
							  encoder_sink_(nullptr),
							  parameters_(std::move(config), options, max_bitrate_bps, codec_settings), //
							  
							  

						// WebRtcVideoSendStream 中的 webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_); 成员变量的创建 
						void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() {
						  RTC_DCHECK_RUN_ON(&thread_checker_);
						  if (stream_ != NULL) {
							call_->DestroyVideoSendStream(stream_);
						  }

						  RTC_CHECK(parameters_.codec_settings);
						  RTC_DCHECK_EQ((parameters_.encoder_config.content_type ==
										 webrtc::VideoEncoderConfig::ContentType::kScreen),
										parameters_.options.is_screencast.value_or(false))
							  << "encoder content type inconsistent with screencast option";
						  parameters_.encoder_config.encoder_specific_settings =
							  ConfigureVideoEncoderSettings(parameters_.codec_settings->codec);

						  webrtc::VideoSendStream::Config config = parameters_.config.Copy(); // 
						  if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) {
							RTC_LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX "
												   "payload type the set codec. Ignoring RTX.";
							config.rtp.rtx.ssrcs.clear();
						  }
						  if (parameters_.encoder_config.number_of_streams == 1) {
							// SVC is used instead of simulcast. Remove unnecessary SSRCs.
							if (config.rtp.ssrcs.size() > 1) {
							  config.rtp.ssrcs.resize(1);
							  if (config.rtp.rtx.ssrcs.size() > 1) {
								config.rtp.rtx.ssrcs.resize(1);
							  }
							}
						  }
						  stream_ = call_->CreateVideoSendStream(std::move(config), // 
																 parameters_.encoder_config.Copy());

						  parameters_.encoder_config.encoder_specific_settings = NULL;

						  if (source_) {
							stream_->SetSource(this, GetDegradationPreference());
						  }

						  // Call stream_->Start() if necessary conditions are met.
						  UpdateSendState();
						}








/// webrtc::internal::VideoSendStream::SetSource 
void VideoSendStream::SetSource(
    rtc::VideoSourceInterface* source, / source 是指向 webrtc::VideoTrack 的指针 
    const DegradationPreference& degradation_preference) {
  RTC_DCHECK_RUN_ON(&thread_checker_); // video_stream_encoder_ 是 webrtc::internal::VideoSendStream 的成员变量 
  video_stream_encoder_->SetSource(source, degradation_preference); /// std::unique_ptr video_stream_encoder_ 
}  VideoStreamEncoder::SetSource 


 webrtc::internal::VideoSendStream
namespace internal {
VideoSendStream::VideoSendStream(
    Clock* clock,
    int num_cpu_cores,
    ProcessThread* module_process_thread,
    TaskQueueFactory* task_queue_factory,
    CallStats* call_stats,
    RtpTransportControllerSendInterface* transport,
    BitrateAllocatorInterface* bitrate_allocator,
    SendDelayStats* send_delay_stats,
    RtcEventLog* event_log,
    VideoSendStream::Config config,  
    VideoEncoderConfig encoder_config,
    const std::map& suspended_ssrcs,
    const std::map& suspended_payload_states,
    std::unique_ptr fec_controller)
    : worker_queue_(transport->GetWorkerQueue()),
      stats_proxy_(clock, config, encoder_config.content_type),
      config_(std::move(config)), // 
      content_type_(encoder_config.content_type) {
  RTC_DCHECK(config_.encoder_settings.encoder_factory);
  RTC_DCHECK(config_.encoder_settings.bitrate_allocator_factory);

  video_stream_encoder_ =
      CreateVideoStreamEncoder(clock, task_queue_factory, num_cpu_cores, / 
                               &stats_proxy_, config_.encoder_settings);
							   
							   
  worker_queue_->PostTask(ToQueuedTask(
      [this, clock, call_stats, transport, bitrate_allocator, send_delay_stats,
       event_log, &suspended_ssrcs, &encoder_config, &suspended_payload_states,
       &fec_controller]() {
        send_stream_.reset(new VideoSendStreamImpl( // std::unique_ptr send_stream_; 
            clock, &stats_proxy_, worker_queue_, call_stats, transport,
            bitrate_allocator, send_delay_stats, video_stream_encoder_.get(),
            event_log, &config_, encoder_config.max_bitrate_bps,  注意这里的 &config_ , 类型是 const VideoSendStream::Config* config, 注意里面的成员:config->send_transport
            encoder_config.bitrate_priority, suspended_ssrcs,
            suspended_payload_states, encoder_config.content_type,
            std::move(fec_controller), config_.media_transport));
      },
      [this]() { thread_sync_event_.Set(); }));							   
							   
 VideoStreamEncoder 类的继承关系 							   
class VideoStreamEncoder : public VideoStreamEncoderInterface,
                           private EncodedImageCallback,
                           // Protected only to provide access to tests.
                           protected AdaptationObserverInterface							   

namespace webrtc {

std::unique_ptr CreateVideoStreamEncoder(
    Clock* clock,
    TaskQueueFactory* task_queue_factory,
    uint32_t number_of_cores,
    VideoStreamEncoderObserver* encoder_stats_observer,
    const VideoStreamEncoderSettings& settings) {
  return std::make_unique( //
      clock, number_of_cores, encoder_stats_observer, settings,
      std::make_unique(encoder_stats_observer),
      task_queue_factory);
}

}  // namespace webrtc


							   
void VideoStreamEncoder::SetSource(
    rtc::VideoSourceInterface* source, / source 是指向 webrtc::VideoTrack 的指针 
    const DegradationPreference& degradation_preference) {
  RTC_DCHECK_RUN_ON(&thread_checker_); / 在 VideoStreamEncoder 中的构造函数中初始化了 source_proxy_(new VideoSourceProxy(this)),
  source_proxy_->SetSource(source, degradation_preference); / const std::unique_ptr source_proxy_; 这里就是 VideoSourceProxy::SetSource
  encoder_queue_.PostTask([this, degradation_preference] {
    RTC_DCHECK_RUN_ON(&encoder_queue_);
    if (degradation_preference_ != degradation_preference) {
      // Reset adaptation state, so that we're not tricked into thinking there's
      // an already pending request of the same type.
      last_adaptation_request_.reset();
      if (degradation_preference == DegradationPreference::BALANCED ||
          degradation_preference_ == DegradationPreference::BALANCED) {
        // TODO(asapersson): Consider removing |adapt_counters_| map and use one
        // AdaptCounter for all modes.
        source_proxy_->ResetPixelFpsCount();
        adapt_counters_.clear();
      }
    }
    degradation_preference_ = degradation_preference;

    if (encoder_)
      ConfigureQualityScaler(encoder_->GetEncoderInfo());

    if (!IsFramerateScalingEnabled(degradation_preference) &&
        max_framerate_ != -1) {
      // If frame rate scaling is no longer allowed, remove any potential
      // allowance for longer frame intervals.
      overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
    }
  });
}							   


/ VideoSourceProxy 是 VideoStreamEncoder 的内部类 
class VideoStreamEncoder::VideoSourceProxy {
 public:
  explicit VideoSourceProxy(VideoStreamEncoder* video_stream_encoder)
      : video_stream_encoder_(video_stream_encoder),
        degradation_preference_(DegradationPreference::DISABLED),
        source_(nullptr),
        max_framerate_(std::numeric_limits::max()) {}

  void SetSource(rtc::VideoSourceInterface* source, /// source 是指向 webrtc::VideoTrack 的指针 
                 const DegradationPreference& degradation_preference) {
    // Called on libjingle's worker thread.
    RTC_DCHECK_RUN_ON(&main_checker_);
    rtc::VideoSourceInterface* old_source = nullptr;
    rtc::VideoSinkWants wants;
    {
      rtc::CritScope lock(&crit_);
      degradation_preference_ = degradation_preference;
      old_source = source_;
      source_ = source; / rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&crit_);
      wants = GetActiveSinkWantsInternal();
    }

    if (old_source != source && old_source != nullptr) {
      old_source->RemoveSink(video_stream_encoder_);
    }

    if (!source) {
      return;
    }
	///
    source->AddOrUpdateSink(video_stream_encoder_, wants); / 这里才是最关键的,此处实际上是将 VideoStreamEncoder 的 this 指针设置到了 webrtc::VideoTrack 中
  } / 也就是作为 webrtc::VideoTrack 的 sink ,然后一步一步的传递到内部的 VideoBroadcaster 中。
  
  
所以在后面的 broadcaster_.OnFrame(frame) 时,其中一个就会回调到  VideoStreamEncoder::OnFrame 中来,后面分析“像素(RGB YUV)格式的 VideoFrame 送到编码器”
  

      

PeerConnection::CreateAndAddTransceiver
      
rtc::scoped_refptr>
PeerConnection::CreateAndAddTransceiver(
    rtc::scoped_refptr> sender,
    rtc::scoped_refptr>
        receiver) {
  // Ensure that the new sender does not have an ID that is already in use by
  // another sender.
  // Allow receiver IDs to conflict since those come from remote SDP (which
  // could be invalid, but should not cause a crash).
  RTC_DCHECK(!FindSenderById(sender->id()));
  auto transceiver = RtpTransceiverProxyWithInternal::Create(
      signaling_thread(),
      new RtpTransceiver(sender, receiver, channel_manager()));
  transceivers_.push_back(transceiver);
  transceiver->internal()->SignalNegotiationNeeded.connect(
      this, &PeerConnection::OnNegotiationNeeded);
  return transceiver;
}

3.2 采集到的像素格式的VideoFrame沿着流水线送到编码器,并发送到 对端


  像素(RGB YUV)格式的 VideoFrame 送到编码器

 WINDOWS 平台摄像机捕获的码流 //
void CaptureSinkFilter::ProcessCapturedFrame(
    unsigned char* buffer,
    size_t length,
    const VideoCaptureCapability& frame_info) {
  // Called on the capture thread.
  capture_observer_->IncomingFrame(buffer, length, frame_info); / WINDOWS 
}

 LINUX 平台摄像机捕获的码流 //
bool VideoCaptureModuleV4L2::CaptureProcess() {
  int retVal = 0;
  fd_set rSet;
  struct timeval timeout;

  FD_ZERO(&rSet);
  FD_SET(_deviceFd, &rSet);
  timeout.tv_sec = 1;
  timeout.tv_usec = 0;

  // _deviceFd written only in StartCapture, when this thread isn't running.
  retVal = select(_deviceFd + 1, &rSet, NULL, NULL, &timeout);
  if (retVal < 0 && errno != EINTR)  // continue if interrupted
  {
    // select failed
    return false;
  } else if (retVal == 0) {
    // select timed out
    return true;
  } else if (!FD_ISSET(_deviceFd, &rSet)) {
    // not event on camera handle
    return true;
  }

  {
    MutexLock lock(&capture_lock_);

    if (quit_) {
      return false;
    }

    if (_captureStarted) {
      struct v4l2_buffer buf;
      memset(&buf, 0, sizeof(struct v4l2_buffer));
      buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
      buf.memory = V4L2_MEMORY_MMAP;
      // dequeue a buffer - repeat until dequeued properly!
      while (ioctl(_deviceFd, VIDIOC_DQBUF, &buf) < 0) {
        if (errno != EINTR) {
          RTC_LOG(LS_INFO) << "could not sync on a buffer on device "
                           << strerror(errno);
          return true;
        }
      }
      VideoCaptureCapability frameInfo;
      frameInfo.width = _currentWidth;
      frameInfo.height = _currentHeight;
      frameInfo.videoType = _captureVideoType;

      // convert to to I420 if needed
      IncomingFrame((unsigned char*)_pool[buf.index].start, buf.bytesused,
                    frameInfo); / LINUX
      // enqueue the buffer again
      if (ioctl(_deviceFd, VIDIOC_QBUF, &buf) == -1) {
        RTC_LOG(LS_INFO) << "Failed to enqueue capture buffer";
      }
    }
  }
  usleep(0);
  return true;
}





int32_t VideoCaptureImpl::IncomingFrame(uint8_t* videoFrame,
                                        size_t videoFrameLength,
                                        const VideoCaptureCapability& frameInfo,
                                        int64_t captureTime /*=0*/) {
  MutexLock lock(&api_lock_);

  const int32_t width = frameInfo.width;
  const int32_t height = frameInfo.height;

  TRACE_EVENT1("webrtc", "VC::IncomingFrame", "capture_time", captureTime);

  // Not encoded, convert to I420.
  if (frameInfo.videoType != VideoType::kMJPEG &&
      CalcBufferSize(frameInfo.videoType, width, abs(height)) !=
          videoFrameLength) {
    RTC_LOG(LS_ERROR) << "Wrong incoming frame length.";
    return -1;
  }

  int stride_y = width;
  int stride_uv = (width + 1) / 2;
  int target_width = width;
  int target_height = abs(height);

  // SetApplyRotation doesn't take any lock. Make a local copy here.
  bool apply_rotation = apply_rotation_;

  if (apply_rotation) {
    // Rotating resolution when for 90/270 degree rotations.
    if (_rotateFrame == kVideoRotation_90 ||
        _rotateFrame == kVideoRotation_270) {
      target_width = abs(height);
      target_height = width;
    }
  }

  // Setting absolute height (in case it was negative).
  // In Windows, the image starts bottom left, instead of top left.
  // Setting a negative source height, inverts the image (within LibYuv).

  // TODO(nisse): Use a pool?
  rtc::scoped_refptr buffer = I420Buffer::Create(
      target_width, target_height, stride_y, stride_uv, stride_uv);

  libyuv::RotationMode rotation_mode = libyuv::kRotate0;
  if (apply_rotation) {
    switch (_rotateFrame) {
      case kVideoRotation_0:
        rotation_mode = libyuv::kRotate0;
        break;
      case kVideoRotation_90:
        rotation_mode = libyuv::kRotate90;
        break;
      case kVideoRotation_180:
        rotation_mode = libyuv::kRotate180;
        break;
      case kVideoRotation_270:
        rotation_mode = libyuv::kRotate270;
        break;
    }
  }

  const int conversionResult = libyuv::ConvertToI420(
      videoFrame, videoFrameLength, buffer.get()->MutableDataY(),
      buffer.get()->StrideY(), buffer.get()->MutableDataU(),
      buffer.get()->StrideU(), buffer.get()->MutableDataV(),
      buffer.get()->StrideV(), 0, 0,  // No Cropping
      width, height, target_width, target_height, rotation_mode,
      ConvertVideoType(frameInfo.videoType));
  if (conversionResult < 0) {
    RTC_LOG(LS_ERROR) << "Failed to convert capture frame from type "
                      << static_cast(frameInfo.videoType) << "to I420.";
    return -1;
  }

  VideoFrame captureFrame =
      VideoFrame::Builder()
          .set_video_frame_buffer(buffer)
          .set_timestamp_rtp(0)
          .set_timestamp_ms(rtc::TimeMillis())
          .set_rotation(!apply_rotation ? _rotateFrame : kVideoRotation_0)
          .build();
  captureFrame.set_ntp_time_ms(captureTime);

  DeliverCapturedFrame(captureFrame); /

  return 0;
}



int32_t VideoCaptureImpl::DeliverCapturedFrame(VideoFrame& captureFrame) {
  UpdateFrameCount();  // frame count used for local frame rate callback.

  if (_dataCallBack) {
    _dataCallBack->OnFrame(captureFrame); /// VcmCapturer::OnFrame
  }

  return 0;
}

void VcmCapturer::OnFrame(const VideoFrame& frame) {
  TestVideoCapturer::OnFrame(frame);
}


void TestVideoCapturer::OnFrame(const VideoFrame& original_frame) {
  int cropped_width = 0;
  int cropped_height = 0;
  int out_width = 0;
  int out_height = 0;

  VideoFrame frame = MaybePreprocess(original_frame);

  if (!video_adapter_.AdaptFrameResolution(
          frame.width(), frame.height(), frame.timestamp_us() * 1000,
          &cropped_width, &cropped_height, &out_width, &out_height)) {
    // Drop frame in order to respect frame rate constraint.
    return;
  }

  if (out_height != frame.height() || out_width != frame.width()) {
    // Video adapter has requested a down-scale. Allocate a new buffer and
    // return scaled version.
    rtc::scoped_refptr scaled_buffer =
        I420Buffer::Create(out_width, out_height);
    scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
    broadcaster_.OnFrame(VideoFrame::Builder()
                             .set_video_frame_buffer(scaled_buffer)
                             .set_rotation(kVideoRotation_0)
                             .set_timestamp_us(frame.timestamp_us())
                             .set_id(frame.id())
                             .build());
  } else {
    // No adaptations needed, just return the frame as is.
    broadcaster_.OnFrame(frame); /
  }
}


void VideoBroadcaster::OnFrame(const webrtc::VideoFrame& frame) {
  rtc::CritScope cs(&sinks_and_wants_lock_);
  bool current_frame_was_discarded = false;
  for (auto& sink_pair : sink_pairs()) { // for 循环遍历挂在 VideoBroadcaster 这个 source 下面的所有 sink ,并将视频流数据进行分发 
    if (sink_pair.wants.rotation_applied &&
        frame.rotation() != webrtc::kVideoRotation_0) {
      // Calls to OnFrame are not synchronized with changes to the sink wants.
      // When rotation_applied is set to true, one or a few frames may get here
      // with rotation still pending. Protect sinks that don't expect any
      // pending rotation.
      RTC_LOG(LS_VERBOSE) << "Discarding frame with unexpected rotation.";
      sink_pair.sink->OnDiscardedFrame();
      current_frame_was_discarded = true;
      continue;
    }
    if (sink_pair.wants.black_frames) {
      webrtc::VideoFrame black_frame =
          webrtc::VideoFrame::Builder()
              .set_video_frame_buffer(
                  GetBlackFrameBuffer(frame.width(), frame.height()))
              .set_rotation(frame.rotation())
              .set_timestamp_us(frame.timestamp_us())
              .set_id(frame.id())
              .build();
      sink_pair.sink->OnFrame(black_frame);
    } else if (!previous_frame_sent_to_all_sinks_) {
      // Since last frame was not sent to some sinks, full update is needed.
      webrtc::VideoFrame copy = frame;
      copy.set_update_rect(
          webrtc::VideoFrame::UpdateRect{0, 0, frame.width(), frame.height()});
      sink_pair.sink->OnFrame(copy);
    } else {
      sink_pair.sink->OnFrame(frame);  其中有一个会送到编码器 VideoStreamEncoder::OnFrame 中
    }
  }
  previous_frame_sent_to_all_sinks_ = !current_frame_was_discarded;
}



void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {
  RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
  VideoFrame incoming_frame = video_frame;

  // Local time in webrtc time base.
  int64_t current_time_us = clock_->TimeInMicroseconds();
  int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec;
  // In some cases, e.g., when the frame from decoder is fed to encoder,
  // the timestamp may be set to the future. As the encoding pipeline assumes
  // capture time to be less than present time, we should reset the capture
  // timestamps here. Otherwise there may be issues with RTP send stream.
  if (incoming_frame.timestamp_us() > current_time_us)
    incoming_frame.set_timestamp_us(current_time_us);

  // Capture time may come from clock with an offset and drift from clock_.
  int64_t capture_ntp_time_ms;
  if (video_frame.ntp_time_ms() > 0) {
    capture_ntp_time_ms = video_frame.ntp_time_ms();
  } else if (video_frame.render_time_ms() != 0) {
    capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_;
  } else {
    capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_;
  }
  incoming_frame.set_ntp_time_ms(capture_ntp_time_ms);

  // Convert NTP time, in ms, to RTP timestamp.
  const int kMsToRtpTimestamp = 90;
  incoming_frame.set_timestamp(
      kMsToRtpTimestamp * static_cast(incoming_frame.ntp_time_ms()));

  if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
    // We don't allow the same capture time for two frames, drop this one.
    RTC_LOG(LS_WARNING) << "Same/old NTP timestamp ("
                        << incoming_frame.ntp_time_ms()
                        << " <= " << last_captured_timestamp_
                        << ") for incoming frame. Dropping.";
    encoder_queue_.PostTask([this, incoming_frame]() {
      RTC_DCHECK_RUN_ON(&encoder_queue_);
      accumulated_update_rect_.Union(incoming_frame.update_rect());
    });
    return;
  }

  bool log_stats = false;
  if (current_time_ms - last_frame_log_ms_ > kFrameLogIntervalMs) {
    last_frame_log_ms_ = current_time_ms;
    log_stats = true;
  }

  last_captured_timestamp_ = incoming_frame.ntp_time_ms();

  int64_t post_time_us = rtc::TimeMicros();
  ++posted_frames_waiting_for_encode_;

  encoder_queue_.PostTask(
      [this, incoming_frame, post_time_us, log_stats]() {
        RTC_DCHECK_RUN_ON(&encoder_queue_);
        encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(),
                                                 incoming_frame.height());
        ++captured_frame_count_;
        const int posted_frames_waiting_for_encode =
            posted_frames_waiting_for_encode_.fetch_sub(1);
        RTC_DCHECK_GT(posted_frames_waiting_for_encode, 0);
        if (posted_frames_waiting_for_encode == 1) {
          MaybeEncodeVideoFrame(incoming_frame, post_time_us); /// VideoStreamEncoder::MaybeEncodeVideoFrame
        } else {
          // There is a newer frame in flight. Do not encode this frame.
          RTC_LOG(LS_VERBOSE)
              << "Incoming frame dropped due to that the encoder is blocked.";
          ++dropped_frame_count_;
          encoder_stats_observer_->OnFrameDropped(
              VideoStreamEncoderObserver::DropReason::kEncoderQueue);
          accumulated_update_rect_.Union(incoming_frame.update_rect());
        }
        if (log_stats) {
          RTC_LOG(LS_INFO) << "Number of frames: captured "
                           << captured_frame_count_
                           << ", dropped (due to encoder blocked) "
                           << dropped_frame_count_ << ", interval_ms "
                           << kFrameLogIntervalMs;
          captured_frame_count_ = 0;
          dropped_frame_count_ = 0;
        }
      });
}



void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
                                               int64_t time_when_posted_us) {
  RTC_DCHECK_RUN_ON(&encoder_queue_);

  if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
      video_frame.height() != last_frame_info_->height ||
      video_frame.is_texture() != last_frame_info_->is_texture) {
    pending_encoder_reconfiguration_ = true;
    last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(),
                                      video_frame.is_texture());
    RTC_LOG(LS_INFO) << "Video frame parameters changed: dimensions="
                     << last_frame_info_->width << "x"
                     << last_frame_info_->height
                     << ", texture=" << last_frame_info_->is_texture << ".";
    // Force full frame update, since resolution has changed.
    accumulated_update_rect_ =
        VideoFrame::UpdateRect{0, 0, video_frame.width(), video_frame.height()};
  }

  // We have to create then encoder before the frame drop logic,
  // because the latter depends on encoder_->GetScalingSettings.
  // According to the testcase
  // InitialFrameDropOffWhenEncoderDisabledScaling, the return value
  // from GetScalingSettings should enable or disable the frame drop.

  // Update input frame rate before we start using it. If we update it after
  // any potential frame drop we are going to artificially increase frame sizes.
  // Poll the rate before updating, otherwise we risk the rate being estimated
  // a little too high at the start of the call when then window is small.
  uint32_t framerate_fps = GetInputFramerateFps();
  input_framerate_.Update(1u, clock_->TimeInMilliseconds());

  int64_t now_ms = clock_->TimeInMilliseconds();
  if (pending_encoder_reconfiguration_) {
    ReconfigureEncoder();
    last_parameters_update_ms_.emplace(now_ms);
  } else if (!last_parameters_update_ms_ ||
             now_ms - *last_parameters_update_ms_ >=
                 kParameterUpdateIntervalMs) {
    if (last_encoder_rate_settings_) {
      // Clone rate settings before update, so that SetEncoderRates() will
      // actually detect the change between the input and
      // |last_encoder_rate_setings_|, triggering the call to SetRate() on the
      // encoder.
      EncoderRateSettings new_rate_settings = *last_encoder_rate_settings_;
      new_rate_settings.rate_control.framerate_fps =
          static_cast(framerate_fps);
      SetEncoderRates(
          UpdateBitrateAllocationAndNotifyObserver(new_rate_settings));
    }
    last_parameters_update_ms_.emplace(now_ms);
  }

  // Because pending frame will be dropped in any case, we need to
  // remember its updated region.
  if (pending_frame_) {
    encoder_stats_observer_->OnFrameDropped(
        VideoStreamEncoderObserver::DropReason::kEncoderQueue);
    accumulated_update_rect_.Union(pending_frame_->update_rect());
  }

  if (DropDueToSize(video_frame.size())) {
    RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
    int fps_count = GetConstAdaptCounter().FramerateCount(kQuality);
    int res_count = GetConstAdaptCounter().ResolutionCount(kQuality);
    AdaptDown(kQuality);
    if (degradation_preference_ == DegradationPreference::BALANCED &&
        GetConstAdaptCounter().FramerateCount(kQuality) > fps_count) {
      // Adapt framerate in same step as resolution.
      AdaptDown(kQuality);
    }
    if (GetConstAdaptCounter().ResolutionCount(kQuality) > res_count) {
      encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
    }
    ++initial_framedrop_;
    // Storing references to a native buffer risks blocking frame capture.
    if (video_frame.video_frame_buffer()->type() !=
        VideoFrameBuffer::Type::kNative) {
      pending_frame_ = video_frame;
      pending_frame_post_time_us_ = time_when_posted_us;
    } else {
      // Ensure that any previously stored frame is dropped.
      pending_frame_.reset();
      accumulated_update_rect_.Union(video_frame.update_rect());
    }
    return;
  }
  initial_framedrop_ = kMaxInitialFramedrop;

  if (EncoderPaused()) {
    // Storing references to a native buffer risks blocking frame capture.
    if (video_frame.video_frame_buffer()->type() !=
        VideoFrameBuffer::Type::kNative) {
      if (pending_frame_)
        TraceFrameDropStart();
      pending_frame_ = video_frame;
      pending_frame_post_time_us_ = time_when_posted_us;
    } else {
      // Ensure that any previously stored frame is dropped.
      pending_frame_.reset();
      TraceFrameDropStart();
      accumulated_update_rect_.Union(video_frame.update_rect());
    }
    return;
  }

  pending_frame_.reset();

  frame_dropper_.Leak(framerate_fps);
  // Frame dropping is enabled iff frame dropping is not force-disabled, and
  // rate controller is not trusted.
  const bool frame_dropping_enabled =
      !force_disable_frame_dropper_ &&
      !encoder_info_.has_trusted_rate_controller;
  frame_dropper_.Enable(frame_dropping_enabled);
  if (frame_dropping_enabled && frame_dropper_.DropFrame()) {
    RTC_LOG(LS_VERBOSE)
        << "Drop Frame: "
        << "target bitrate "
        << (last_encoder_rate_settings_
                ? last_encoder_rate_settings_->encoder_target.bps()
                : 0)
        << ", input frame rate " << framerate_fps;
    OnDroppedFrame(
        EncodedImageCallback::DropReason::kDroppedByMediaOptimizations);
    accumulated_update_rect_.Union(video_frame.update_rect());
    return;
  }

  EncodeVideoFrame(video_frame, time_when_posted_us); 
}

void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
                                          int64_t time_when_posted_us) {
  RTC_DCHECK_RUN_ON(&encoder_queue_);

  // If the encoder fail we can't continue to encode frames. When this happens
  // the WebrtcVideoSender is notified and the whole VideoSendStream is
  // recreated.
  if (encoder_failed_)
    return;

  TraceFrameDropEnd();

  // Encoder metadata needs to be updated before encode complete callback.
  VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
  if (info.implementation_name != encoder_info_.implementation_name) {
    encoder_stats_observer_->OnEncoderImplementationChanged(
        info.implementation_name);
    if (bitrate_adjuster_) {
      // Encoder implementation changed, reset overshoot detector states.
      bitrate_adjuster_->Reset();
    }
  }

  if (bitrate_adjuster_) {
    for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
      if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) {
        bitrate_adjuster_->OnEncoderInfo(info);
        break;
      }
    }
  }
  encoder_info_ = info;
  last_encode_info_ms_ = clock_->TimeInMilliseconds();

  VideoFrame out_frame(video_frame);  

  const VideoFrameBuffer::Type buffer_type =
      out_frame.video_frame_buffer()->type();
  const bool is_buffer_type_supported =
      buffer_type == VideoFrameBuffer::Type::kI420 ||
      (buffer_type == VideoFrameBuffer::Type::kNative &&
       info.supports_native_handle);

  if (!is_buffer_type_supported) {
    // This module only supports software encoding.
    rtc::scoped_refptr converted_buffer(
        out_frame.video_frame_buffer()->ToI420());

    if (!converted_buffer) {
      RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame.";
      return;
    }

    VideoFrame::UpdateRect update_rect = out_frame.update_rect();
    if (!update_rect.IsEmpty() &&
        out_frame.video_frame_buffer()->GetI420() == nullptr) {
      // UpdatedRect is reset to full update if it's not empty, and buffer was
      // converted, therefore we can't guarantee that pixels outside of
      // UpdateRect didn't change comparing to the previous frame.
      update_rect =
          VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()};
    }

    out_frame.set_video_frame_buffer(converted_buffer);
    out_frame.set_update_rect(update_rect);
  }

  // Crop frame if needed.
  if ((crop_width_ > 0 || crop_height_ > 0) &&
      out_frame.video_frame_buffer()->type() !=
          VideoFrameBuffer::Type::kNative) {
    // If the frame can't be converted to I420, drop it.
    auto i420_buffer = video_frame.video_frame_buffer()->ToI420();
    if (!i420_buffer) {
      RTC_LOG(LS_ERROR) << "Frame conversion for crop failed, dropping frame.";
      return;
    }
    int cropped_width = video_frame.width() - crop_width_;
    int cropped_height = video_frame.height() - crop_height_;
    rtc::scoped_refptr cropped_buffer =
        I420Buffer::Create(cropped_width, cropped_height);
    // TODO(ilnik): Remove scaling if cropping is too big, as it should never
    // happen after SinkWants signaled correctly from ReconfigureEncoder.
    VideoFrame::UpdateRect update_rect = video_frame.update_rect();
    if (crop_width_ < 4 && crop_height_ < 4) {
      cropped_buffer->CropAndScaleFrom(*i420_buffer, crop_width_ / 2,
                                       crop_height_ / 2, cropped_width,
                                       cropped_height);
      update_rect.offset_x -= crop_width_ / 2;
      update_rect.offset_y -= crop_height_ / 2;
      update_rect.Intersect(
          VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height});

    } else {
      cropped_buffer->ScaleFrom(*i420_buffer);
      if (!update_rect.IsEmpty()) {
        // Since we can't reason about pixels after scaling, we invalidate whole
        // picture, if anything changed.
        update_rect =
            VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height};
      }
    }
    out_frame.set_video_frame_buffer(cropped_buffer);
    out_frame.set_update_rect(update_rect);
    out_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
    // Since accumulated_update_rect_ is constructed before cropping,
    // we can't trust it. If any changes were pending, we invalidate whole
    // frame here.
    if (!accumulated_update_rect_.IsEmpty()) {
      accumulated_update_rect_ =
          VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()};
    }
  }

  if (!accumulated_update_rect_.IsEmpty()) {
    accumulated_update_rect_.Union(out_frame.update_rect());
    accumulated_update_rect_.Intersect(
        VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()});
    out_frame.set_update_rect(accumulated_update_rect_);
    accumulated_update_rect_.MakeEmptyUpdate();
  }

  TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
                          "Encode");

  overuse_detector_->FrameCaptured(out_frame, time_when_posted_us);

  RTC_DCHECK_LE(send_codec_.width, out_frame.width());
  RTC_DCHECK_LE(send_codec_.height, out_frame.height());
  // Native frames should be scaled by the client.
  // For internal encoders we scale everything in one place here.
  RTC_DCHECK((out_frame.video_frame_buffer()->type() ==
              VideoFrameBuffer::Type::kNative) ||
             (send_codec_.width == out_frame.width() &&
              send_codec_.height == out_frame.height()));

  TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp",
               out_frame.timestamp());

  frame_encode_metadata_writer_.OnEncodeStarted(out_frame);

  const int32_t encode_status = encoder_->Encode(out_frame, &next_frame_types_); // 具体的编码操作,比如 H264EncoderImpl::Encode 、LibvpxVp8Encoder::Encode 
  was_encode_called_since_last_initialization_ = true;

  if (encode_status < 0) {
    if (encode_status == WEBRTC_VIDEO_CODEC_ENCODER_FAILURE) {
      RTC_LOG(LS_ERROR) << "Encoder failed, failing encoder format: "
                        << encoder_config_.video_format.ToString();
      if (settings_.encoder_switch_request_callback) {
        encoder_failed_ = true;
        settings_.encoder_switch_request_callback->RequestEncoderFallback();
      } else {
        RTC_LOG(LS_ERROR)
            << "Encoder failed but no encoder fallback callback is registered";
      }
    } else {
      RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: "
                        << encode_status;
    }

    return;
  }

  for (auto& it : next_frame_types_) {
    it = VideoFrameType::kVideoFrameDelta;
  }
}



int32_t H264EncoderImpl::Encode(
    const VideoFrame& input_frame,
    const std::vector* frame_types) {
  if (encoders_.empty()) {
    ReportError();
    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  }
  if (!encoded_image_callback_) {
    RTC_LOG(LS_WARNING)
        << "InitEncode() has been called, but a callback function "
        << "has not been set with RegisterEncodeCompleteCallback()";
    ReportError();
    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  }

  rtc::scoped_refptr frame_buffer =
      input_frame.video_frame_buffer()->ToI420();

  bool send_key_frame = false;
  for (size_t i = 0; i < configurations_.size(); ++i) {
    if (configurations_[i].key_frame_request && configurations_[i].sending) {
      send_key_frame = true;
      break;
    }
  }

  if (!send_key_frame && frame_types) {
    for (size_t i = 0; i < configurations_.size(); ++i) {
      const size_t simulcast_idx =
          static_cast(configurations_[i].simulcast_idx);
      if (configurations_[i].sending && simulcast_idx < frame_types->size() &&
          (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) {
        send_key_frame = true;
        break;
      }
    }
  }

  RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
  RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());

  // Encode image for each layer.
  for (size_t i = 0; i < encoders_.size(); ++i) {
    // EncodeFrame input.
    pictures_[i] = {0};
    pictures_[i].iPicWidth = configurations_[i].width;
    pictures_[i].iPicHeight = configurations_[i].height;
    pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420;
    pictures_[i].uiTimeStamp = input_frame.ntp_time_ms();
    // Downscale images on second and ongoing layers.
    if (i == 0) {
      pictures_[i].iStride[0] = frame_buffer->StrideY();
      pictures_[i].iStride[1] = frame_buffer->StrideU();
      pictures_[i].iStride[2] = frame_buffer->StrideV();
      pictures_[i].pData[0] = const_cast(frame_buffer->DataY());
      pictures_[i].pData[1] = const_cast(frame_buffer->DataU());
      pictures_[i].pData[2] = const_cast(frame_buffer->DataV());
    } else {
      pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY();
      pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU();
      pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV();
      pictures_[i].pData[0] =
          const_cast(downscaled_buffers_[i - 1]->DataY());
      pictures_[i].pData[1] =
          const_cast(downscaled_buffers_[i - 1]->DataU());
      pictures_[i].pData[2] =
          const_cast(downscaled_buffers_[i - 1]->DataV());
      // Scale the image down a number of times by downsampling factor.
      libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0],
                        pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1],
                        pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2],
                        configurations_[i - 1].width,
                        configurations_[i - 1].height, pictures_[i].pData[0],
                        pictures_[i].iStride[0], pictures_[i].pData[1],
                        pictures_[i].iStride[1], pictures_[i].pData[2],
                        pictures_[i].iStride[2], configurations_[i].width,
                        configurations_[i].height, libyuv::kFilterBilinear);
    }

    if (!configurations_[i].sending) {
      continue;
    }
    if (frame_types != nullptr) {
      // Skip frame?
      if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) {
        continue;
      }
    }
    if (send_key_frame) {
      // API doc says ForceIntraFrame(false) does nothing, but calling this
      // function forces a key frame regardless of the |bIDR| argument's value.
      // (If every frame is a key frame we get lag/delays.)
      encoders_[i]->ForceIntraFrame(true);
      configurations_[i].key_frame_request = false;
    }
    // EncodeFrame output.
    SFrameBSInfo info;
    memset(&info, 0, sizeof(SFrameBSInfo));

    // Encode!
    int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
    if (enc_ret != 0) {
      RTC_LOG(LS_ERROR)
          << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret
          << ".";
      ReportError();
      return WEBRTC_VIDEO_CODEC_ERROR;
    }

    encoded_images_[i]._encodedWidth = configurations_[i].width;
    encoded_images_[i]._encodedHeight = configurations_[i].height;
    encoded_images_[i].SetTimestamp(input_frame.timestamp());
    encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
    encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);

    // Split encoded image up into fragments. This also updates
    // |encoded_image_|.
	// 编码后,编码数据保存在info中,RtpFragmentize将编码数据拷贝到 encoded_images_[i]中,并将其中的nalu信息统计在frag_header内
    RTPFragmentationHeader frag_header;
    RtpFragmentize(&encoded_images_[i], *frame_buffer, &info, &frag_header);

    // Encoder can skip frames to save bandwidth in which case
    // |encoded_images_[i]._length| == 0.
    if (encoded_images_[i].size() > 0) {
      // Parse QP.
      h264_bitstream_parser_.ParseBitstream(encoded_images_[i].data(),
                                            encoded_images_[i].size());
      h264_bitstream_parser_.GetLastSliceQp(&encoded_images_[i].qp_);

      // Deliver encoded image.
      CodecSpecificInfo codec_specific;
      codec_specific.codecType = kVideoCodecH264;
      codec_specific.codecSpecific.H264.packetization_mode =
          packetization_mode_;
      codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
      codec_specific.codecSpecific.H264.idr_frame =
          info.eFrameType == videoFrameTypeIDR;
      codec_specific.codecSpecific.H264.base_layer_sync = false;
      if (configurations_[i].num_temporal_layers > 1) {
        const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
        codec_specific.codecSpecific.H264.temporal_idx = tid;
        codec_specific.codecSpecific.H264.base_layer_sync =
            tid > 0 && tid < tl0sync_limit_[i];
        if (codec_specific.codecSpecific.H264.base_layer_sync) {
          tl0sync_limit_[i] = tid;
        }
        if (tid == 0) {
          tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
        }
      }
	  // 编码成功后,将数据回调出去,接收者即为 VideoStreamEncoder
      encoded_image_callback_->OnEncodedImage(encoded_images_[i],
                                              &codec_specific, &frag_header);
    }
  }
  return WEBRTC_VIDEO_CODEC_OK;
}


						/ encoded_image_callback_ 的设置 
						void VideoStreamEncoder::ReconfigureEncoder()
						===>
							  encoder_initialized_ = true;
							  encoder_->RegisterEncodeCompleteCallback(this); /// 
							  frame_encode_metadata_writer_.OnEncoderInit(send_codec_,
																		  HasInternalSource());
																		  
																		  
						int32_t H264EncoderImpl::RegisterEncodeCompleteCallback(
							EncodedImageCallback* callback) {
						  encoded_image_callback_ = callback;
						  return WEBRTC_VIDEO_CODEC_OK;
						}





EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
    const EncodedImage& encoded_image,
    const CodecSpecificInfo* codec_specific_info,
    const RTPFragmentationHeader* fragmentation) {
  TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
                       "timestamp", encoded_image.Timestamp());
  const size_t spatial_idx = encoded_image.SpatialIndex().value_or(0);
  EncodedImage image_copy(encoded_image);

  frame_encode_metadata_writer_.FillTimingInfo(spatial_idx, &image_copy);

  std::unique_ptr fragmentation_copy =
      frame_encode_metadata_writer_.UpdateBitstream(codec_specific_info,
                                                    fragmentation, &image_copy);

  // Piggyback ALR experiment group id and simulcast id into the content type.
  const uint8_t experiment_id =
      experiment_groups_[videocontenttypehelpers::IsScreenshare(
          image_copy.content_type_)];

  // TODO(ilnik): This will force content type extension to be present even
  // for realtime video. At the expense of miniscule overhead we will get
  // sliced receive statistics.
  RTC_CHECK(videocontenttypehelpers::SetExperimentId(&image_copy.content_type_,
                                                     experiment_id));
  // We count simulcast streams from 1 on the wire. That's why we set simulcast
  // id in content type to +1 of that is actual simulcast index. This is because
  // value 0 on the wire is reserved for 'no simulcast stream specified'.
  RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
      &image_copy.content_type_, static_cast(spatial_idx + 1)));

  // Encoded is called on whatever thread the real encoder implementation run
  // on. In the case of hardware encoders, there might be several encoders
  // running in parallel on different threads.
  encoder_stats_observer_->OnSendEncodedImage(image_copy, codec_specific_info);

  // The simulcast id is signaled in the SpatialIndex. This makes it impossible
  // to do simulcast for codecs that actually support spatial layers since we
  // can't distinguish between an actual spatial layer and a simulcast stream.
  // TODO(bugs.webrtc.org/10520): Signal the simulcast id explicitly.
  int simulcast_id = 0;
  if (codec_specific_info &&
      (codec_specific_info->codecType == kVideoCodecVP8 ||
       codec_specific_info->codecType == kVideoCodecH264 ||
       codec_specific_info->codecType == kVideoCodecGeneric)) {
    simulcast_id = encoded_image.SpatialIndex().value_or(0);
  }

  std::unique_ptr codec_info_copy;
  {
    rtc::CritScope cs(&encoded_image_lock_);

    if (codec_specific_info && codec_specific_info->generic_frame_info) {
      codec_info_copy =
          std::make_unique(*codec_specific_info);
      GenericFrameInfo& generic_info = *codec_info_copy->generic_frame_info;
      generic_info.frame_id = next_frame_id_++;

      if (encoder_buffer_state_.size() <= static_cast(simulcast_id)) {
        RTC_LOG(LS_ERROR) << "At most " << encoder_buffer_state_.size()
                          << " simulcast streams supported.";
      } else {
        std::array& state =
            encoder_buffer_state_[simulcast_id];
        for (const CodecBufferUsage& buffer : generic_info.encoder_buffers) {
          if (state.size() <= static_cast(buffer.id)) {
            RTC_LOG(LS_ERROR)
                << "At most " << state.size() << " encoder buffers supported.";
            break;
          }

          if (buffer.referenced) {
            int64_t diff = generic_info.frame_id - state[buffer.id];
            if (diff <= 0) {
              RTC_LOG(LS_ERROR) << "Invalid frame diff: " << diff << ".";
            } else if (absl::c_find(generic_info.frame_diffs, diff) ==
                       generic_info.frame_diffs.end()) {
              generic_info.frame_diffs.push_back(diff);
            }
          }

          if (buffer.updated)
            state[buffer.id] = generic_info.frame_id;
        }
      }
    }
  }
	// EncoderSink* sink_; 设置见下方
  EncodedImageCallback::Result result = sink_->OnEncodedImage(  VideoSendStreamImpl::OnEncodedImage 
      image_copy, codec_info_copy ? codec_info_copy.get() : codec_specific_info,
      fragmentation_copy ? fragmentation_copy.get() : fragmentation);

  // We are only interested in propagating the meta-data about the image, not
  // encoded data itself, to the post encode function. Since we cannot be sure
  // the pointer will still be valid when run on the task queue, set it to null.
  image_copy.set_buffer(nullptr, 0);

  int temporal_index = 0;
  if (codec_specific_info) {
    if (codec_specific_info->codecType == kVideoCodecVP9) {
      temporal_index = codec_specific_info->codecSpecific.VP9.temporal_idx;
    } else if (codec_specific_info->codecType == kVideoCodecVP8) {
      temporal_index = codec_specific_info->codecSpecific.VP8.temporalIdx;
    }
  }
  if (temporal_index == kNoTemporalIdx) {
    temporal_index = 0;
  }

  RunPostEncode(image_copy, rtc::TimeMicros(), temporal_index);

  if (result.error == Result::OK) {
    // In case of an internal encoder running on a separate thread, the
    // decision to drop a frame might be a frame late and signaled via
    // atomic flag. This is because we can't easily wait for the worker thread
    // without risking deadlocks, eg during shutdown when the worker thread
    // might be waiting for the internal encoder threads to stop.
    if (pending_frame_drops_.load() > 0) {
      int pending_drops = pending_frame_drops_.fetch_sub(1);
      RTC_DCHECK_GT(pending_drops, 0);
      result.drop_next_frame = true;
    }
  }

  return result;
}

						// VideoSendStreamImpl 是 webrtc::internal::VideoSendStream 的成员变量 send_stream_ ,并在 
						// VideoSendStream 的构造函数中被初始化,上面的代码中已经讲解了
						VideoSendStreamImpl::VideoSendStreamImpl
						====>
						// video_stream_encoder_ 就是在 webrtc::internal::VideoSendStream 中创建的,然后
						// 被传递到了 VideoSendStreamImpl 的构造函数中
						video_stream_encoder_->SetSink(this, rotation_applied)


						void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
						  source_proxy_->SetWantsRotationApplied(rotation_applied);
						  encoder_queue_.PostTask([this, sink] {
							RTC_DCHECK_RUN_ON(&encoder_queue_);
							sink_ = sink;
						  });
						}



EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage(
    const EncodedImage& encoded_image,
    const CodecSpecificInfo* codec_specific_info,
    const RTPFragmentationHeader* fragmentation) {
  // Encoded is called on whatever thread the real encoder implementation run
  // on. In the case of hardware encoders, there might be several encoders
  // running in parallel on different threads.

  // Indicate that there still is activity going on.
  activity_ = true;

  auto enable_padding_task = [this]() {
    if (disable_padding_) {
      RTC_DCHECK_RUN_ON(worker_queue_);
      disable_padding_ = false;
      // To ensure that padding bitrate is propagated to the bitrate allocator.
      SignalEncoderActive();
    }
  };
  if (!worker_queue_->IsCurrent()) {
    worker_queue_->PostTask(enable_padding_task);
  } else {
    enable_padding_task();
  }

  EncodedImageCallback::Result result(EncodedImageCallback::Result::OK);
  if (media_transport_) {
    int64_t frame_id;
    {
      // TODO(nisse): Responsibility for allocation of frame ids should move to
      // VideoStreamEncoder.
      rtc::CritScope cs(&media_transport_id_lock_);
      frame_id = media_transport_frame_id_++;
    }
    // TODO(nisse): Responsibility for reference meta data should be moved
    // upstream, ideally close to the encoders, but probably VideoStreamEncoder
    // will need to do some translation to produce reference info using frame
    // ids.
    std::vector referenced_frame_ids;
    if (encoded_image._frameType != VideoFrameType::kVideoFrameKey) {
      RTC_DCHECK_GT(frame_id, 0);
      referenced_frame_ids.push_back(frame_id - 1);
    }
    media_transport_->SendVideoFrame(
        config_->rtp.ssrcs[0], webrtc::MediaTransportEncodedVideoFrame(
                                   frame_id, referenced_frame_ids,
                                   config_->rtp.payload_type, encoded_image));
  } else {
    result = rtp_video_sender_->OnEncodedImage(  真正的发送,RtpVideoSender::OnEncodedImage 
        encoded_image, codec_specific_info, fragmentation); /// rtp_video_sender_ 在 VideoSendStreamImpl 中的构造函数中被创建
  }
  // Check if there's a throttled VideoBitrateAllocation that we should try
  // sending.
  rtc::WeakPtr send_stream = weak_ptr_;
  auto update_task = [send_stream]() {
    if (send_stream) {
      RTC_DCHECK_RUN_ON(send_stream->worker_queue_);
      auto& context = send_stream->video_bitrate_allocation_context_;
      if (context && context->throttled_allocation) {
        send_stream->OnBitrateAllocationUpdated(*context->throttled_allocation);
      }
    }
  };
  if (!worker_queue_->IsCurrent()) {
    worker_queue_->PostTask(update_task);
  } else {
    update_task();
  }

  return result;
}


						/// VideoSendStreamImpl 的成员变量 rtp_video_sender_ 的构造
						VideoSendStreamImpl::VideoSendStreamImpl
						====>
							  rtp_video_sender_(transport_->CreateRtpVideoSender( /// RtpTransportControllerSend::CreateRtpVideoSender
								  suspended_ssrcs,
								  suspended_payload_states,
								  config_->rtp,
								  config_->rtcp_report_interval_ms,
								  config_->send_transport, / 来自 VideoSendStreamImpl 的入参 const VideoSendStream::Config* config
								  CreateObservers(call_stats,
												  &encoder_feedback_,
												  stats_proxy_,
												  send_delay_stats),
								  event_log,
								  std::move(fec_controller),
								  CreateFrameEncryptionConfig(config_)))
								  

						RtpVideoSenderInterface* RtpTransportControllerSend::CreateRtpVideoSender(
							std::map suspended_ssrcs,
							const std::map& states,
							const RtpConfig& rtp_config,
							int rtcp_report_interval_ms,
							Transport* send_transport, //
							const RtpSenderObservers& observers,
							RtcEventLog* event_log,
							std::unique_ptr fec_controller,
							const RtpSenderFrameEncryptionConfig& frame_encryption_config) {
						  video_rtp_senders_.push_back(std::make_unique( /
							  clock_, suspended_ssrcs, states, rtp_config, rtcp_report_interval_ms,
							  send_transport, observers, / 注意这里的 send_transport 
							  // TODO(holmer): Remove this circular dependency by injecting
							  // the parts of RtpTransportControllerSendInterface that are really used.
							  this, event_log, &retransmission_rate_limiter_, std::move(fec_controller),
							  frame_encryption_config.frame_encryptor,
							  frame_encryption_config.crypto_options));
						  return video_rtp_senders_.back().get();
						}




EncodedImageCallback::Result RtpVideoSender::OnEncodedImage(
    const EncodedImage& encoded_image,
    const CodecSpecificInfo* codec_specific_info,
    const RTPFragmentationHeader* fragmentation) {
  fec_controller_->UpdateWithEncodedData(encoded_image.size(),
                                         encoded_image._frameType);
  rtc::CritScope lock(&crit_);
  RTC_DCHECK(!rtp_streams_.empty());
  if (!active_)
    return Result(Result::ERROR_SEND_FAILED);

  shared_frame_id_++;
  size_t stream_index = 0;
  if (codec_specific_info &&
      (codec_specific_info->codecType == kVideoCodecVP8 ||
       codec_specific_info->codecType == kVideoCodecH264 ||
       codec_specific_info->codecType == kVideoCodecGeneric)) {
    // Map spatial index to simulcast.
    stream_index = encoded_image.SpatialIndex().value_or(0);
  }
  RTC_DCHECK_LT(stream_index, rtp_streams_.size());

  uint32_t rtp_timestamp =
      encoded_image.Timestamp() +
      rtp_streams_[stream_index].rtp_rtcp->StartTimestamp();

  // RTCPSender has it's own copy of the timestamp offset, added in
  // RTCPSender::BuildSR, hence we must not add the in the offset for this call.
  // TODO(nisse): Delete RTCPSender:timestamp_offset_, and see if we can confine
  // knowledge of the offset to a single place.
  if (!rtp_streams_[stream_index].rtp_rtcp->OnSendingRtpFrame(
          encoded_image.Timestamp(), encoded_image.capture_time_ms_,
          rtp_config_.payload_type,
          encoded_image._frameType == VideoFrameType::kVideoFrameKey)) {
    // The payload router could be active but this module isn't sending.
    return Result(Result::ERROR_SEND_FAILED);
  }

  absl::optional expected_retransmission_time_ms;
  if (encoded_image.RetransmissionAllowed()) {
    expected_retransmission_time_ms =
        rtp_streams_[stream_index].rtp_rtcp->ExpectedRetransmissionTimeMs();
  }
  /// 
  bool send_result = rtp_streams_[stream_index].sender_video->SendVideo( // RTPSenderVideo::SendVideo 
      rtp_config_.payload_type, codec_type_, rtp_timestamp,
      encoded_image.capture_time_ms_, encoded_image, fragmentation,
      params_[stream_index].GetRtpVideoHeader(
          encoded_image, codec_specific_info, shared_frame_id_),
      expected_retransmission_time_ms);
  if (frame_count_observer_) {
    FrameCounts& counts = frame_counts_[stream_index];
    if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) {
      ++counts.key_frames;
    } else if (encoded_image._frameType == VideoFrameType::kVideoFrameDelta) {
      ++counts.delta_frames;
    } else {
      RTC_DCHECK(encoded_image._frameType == VideoFrameType::kEmptyFrame);
    }
    frame_count_observer_->FrameCountUpdated(counts,
                                             rtp_config_.ssrcs[stream_index]);
  }
  if (!send_result)
    return Result(Result::ERROR_SEND_FAILED);

  return Result(Result::OK, rtp_timestamp);
}


							 RtpVideoSender 的成员变量 const std::vector rtp_streams_; 的创建
							/ 注意 RtpVideoSender 和 RTPSenderVideo 是两个不同的类 
							RtpVideoSender::RtpVideoSender
							====>
								  rtp_streams_(CreateRtpStreamSenders(clock,
																	  rtp_config,
																	  observers,
																	  rtcp_report_interval_ms,
																	  send_transport,  
																	  transport->GetBandwidthObserver(),
																	  transport,
																	  flexfec_sender_.get(),
																	  event_log,
																	  retransmission_limiter,
																	  this,
																	  frame_encryptor,
																	  crypto_options)),
								  // 对上面创建的 rtp_streams_ 中的数据进行遍历,将它们加入到 PacketRouter 中 
								  for (const RtpStreamSender& stream : rtp_streams_) {
									constexpr bool remb_candidate = true;
									transport->packet_router()->AddSendRtpModule(stream.rtp_rtcp.get(), // PacketRouter::AddSendRtpModule
																				 remb_candidate);
								  }


									void PacketRouter::AddSendRtpModule(RtpRtcp* rtp_module, bool remb_candidate) {
									  rtc::CritScope cs(&modules_crit_);
									  RTC_DCHECK(std::find(rtp_send_modules_.begin(), rtp_send_modules_.end(),
														   rtp_module) == rtp_send_modules_.end());
									  // Put modules which can use regular payload packets (over rtx) instead of
									  // padding first as it's less of a waste
									  if (rtp_module->SupportsRtxPayloadPadding()) {
										rtp_send_modules_.push_front(rtp_module);
									  } else {
										rtp_send_modules_.push_back(rtp_module); // 后面 PacketRouter::SendPacket 的遍历会用到
									  }

									  if (remb_candidate) {
										AddRembModuleCandidate(rtp_module, /* media_sender = */ true);
									  }
									}




							std::vector CreateRtpStreamSenders(
								Clock* clock,
								const RtpConfig& rtp_config,
								const RtpSenderObservers& observers,
								int rtcp_report_interval_ms,
								Transport* send_transport, // 
								RtcpBandwidthObserver* bandwidth_callback,
								RtpTransportControllerSendInterface* transport,
								FlexfecSender* flexfec_sender,
								RtcEventLog* event_log,
								RateLimiter* retransmission_rate_limiter,
								OverheadObserver* overhead_observer,
								FrameEncryptorInterface* frame_encryptor,
								const CryptoOptions& crypto_options) {
							  RTC_DCHECK_GT(rtp_config.ssrcs.size(), 0);

							  RtpRtcp::Configuration configuration;
							  configuration.clock = clock;
							  configuration.audio = false;
							  configuration.receiver_only = false;
							  configuration.outgoing_transport = send_transport; // 
							  configuration.intra_frame_callback = observers.intra_frame_callback;
							  configuration.rtcp_loss_notification_observer =
								  observers.rtcp_loss_notification_observer;
							  configuration.bandwidth_callback = bandwidth_callback;
							  configuration.network_state_estimate_observer =
								  transport->network_state_estimate_observer();
							  configuration.transport_feedback_callback =
								  transport->transport_feedback_observer();
							  configuration.rtt_stats = observers.rtcp_rtt_stats;
							  configuration.rtcp_packet_type_counter_observer =
								  observers.rtcp_type_observer;
							  configuration.paced_sender = transport->packet_sender();
							  configuration.send_bitrate_observer = observers.bitrate_observer;
							  configuration.send_side_delay_observer = observers.send_delay_observer;
							  configuration.send_packet_observer = observers.send_packet_observer;
							  configuration.event_log = event_log;
							  configuration.retransmission_rate_limiter = retransmission_rate_limiter;
							  configuration.overhead_observer = overhead_observer;
							  configuration.rtp_stats_callback = observers.rtp_stats;
							  configuration.frame_encryptor = frame_encryptor;
							  configuration.require_frame_encryption =
								  crypto_options.sframe.require_frame_encryption;
							  configuration.extmap_allow_mixed = rtp_config.extmap_allow_mixed;
							  configuration.rtcp_report_interval_ms = rtcp_report_interval_ms;

							  std::vector rtp_streams;
							  const std::vector& flexfec_protected_ssrcs =
								  rtp_config.flexfec.protected_media_ssrcs;
							  RTC_DCHECK(rtp_config.rtx.ssrcs.empty() ||
										 rtp_config.rtx.ssrcs.size() == rtp_config.rtx.ssrcs.size());
							  for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) {
								configuration.local_media_ssrc = rtp_config.ssrcs[i];
								bool enable_flexfec = flexfec_sender != nullptr &&
													  std::find(flexfec_protected_ssrcs.begin(),
																flexfec_protected_ssrcs.end(),
																configuration.local_media_ssrc) !=
														  flexfec_protected_ssrcs.end();
								configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr;
								auto playout_delay_oracle = std::make_unique();

								configuration.ack_observer = playout_delay_oracle.get();
								if (rtp_config.rtx.ssrcs.size() > i) {
								  configuration.rtx_send_ssrc = rtp_config.rtx.ssrcs[i];
								}

								auto rtp_rtcp = RtpRtcp::Create(configuration); // return std::make_unique(configuration) 
								rtp_rtcp->SetSendingStatus(false);
								rtp_rtcp->SetSendingMediaStatus(false);
								rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
								// Set NACK.
								rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize);

								FieldTrialBasedConfig field_trial_config;
								RTPSenderVideo::Config video_config;
								video_config.clock = configuration.clock;
								video_config.rtp_sender = rtp_rtcp->RtpSender();  
								video_config.flexfec_sender = configuration.flexfec_sender;
								video_config.playout_delay_oracle = playout_delay_oracle.get();
								video_config.frame_encryptor = frame_encryptor;
								video_config.require_frame_encryption =
									crypto_options.sframe.require_frame_encryption;
								video_config.need_rtp_packet_infos = rtp_config.lntf.enabled;
								video_config.enable_retransmit_all_layers = false;
								video_config.field_trials = &field_trial_config;
								const bool should_disable_red_and_ulpfec =
									ShouldDisableRedAndUlpfec(enable_flexfec, rtp_config);
								if (rtp_config.ulpfec.red_payload_type != -1 &&
									!should_disable_red_and_ulpfec) {
								  video_config.red_payload_type = rtp_config.ulpfec.red_payload_type;
								}
								if (rtp_config.ulpfec.ulpfec_payload_type != -1 &&
									!should_disable_red_and_ulpfec) {
								  video_config.ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type;
								}
								auto sender_video = std::make_unique(video_config); / 注意 RtpVideoSender 和 RTPSenderVideo 是两个不同的类 
								rtp_streams.emplace_back(std::move(playout_delay_oracle),
														 std::move(rtp_rtcp), std::move(sender_video));
							  }
							  return rtp_streams;
							}






bool RTPSenderVideo::SendVideo(
    int payload_type,
    absl::optional codec_type,
    uint32_t rtp_timestamp,
    int64_t capture_time_ms,
    rtc::ArrayView payload,
    const RTPFragmentationHeader* fragmentation,
    RTPVideoHeader video_header,
    absl::optional expected_retransmission_time_ms) 
====>
// fec red 等
 LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size);


void RTPSenderVideo::LogAndSendToNetwork(
    std::vector> packets,
    size_t unpacketized_payload_size) {
  int64_t now_ms = clock_->TimeInMilliseconds();
#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
  for (const auto& packet : packets) {
    const uint32_t ssrc = packet->Ssrc();
    BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoTotBitrate_kbps", now_ms,
                                    rtp_sender_->ActualSendBitrateKbit(), ssrc);
    BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoFecBitrate_kbps", now_ms,
                                    FecOverheadRate() / 1000, ssrc);
    BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoNackBitrate_kbps", now_ms,
                                    rtp_sender_->NackOverheadRate() / 1000,
                                    ssrc);
  }
#endif

  {
    rtc::CritScope cs(&stats_crit_);
    size_t packetized_payload_size = 0;
    for (const auto& packet : packets) {
      switch (*packet->packet_type()) {
        case RtpPacketToSend::Type::kVideo:
          video_bitrate_.Update(packet->size(), now_ms);
          packetized_payload_size += packet->payload_size();
          break;
        case RtpPacketToSend::Type::kForwardErrorCorrection:
          fec_bitrate_.Update(packet->size(), clock_->TimeInMilliseconds());
          break;
        default:
          continue;
      }
    }
    RTC_DCHECK_GE(packetized_payload_size, unpacketized_payload_size);
    packetization_overhead_bitrate_.Update(
        packetized_payload_size - unpacketized_payload_size,
        clock_->TimeInMilliseconds());
  }

  // TODO(sprang): Replace with bulk send method.
  for (auto& packet : packets) { / RTPSender* const rtp_sender_; 在 RTPSenderVideo 构造函数中通过 Config 参数传递过来 
    rtp_sender_->SendToNetwork(std::move(packet)); /// RTPSender::SendToNetwork 
  }
}


bool RTPSender::SendToNetwork(std::unique_ptr packet) {
  RTC_DCHECK(packet);
  int64_t now_ms = clock_->TimeInMilliseconds();

  auto packet_type = packet->packet_type();
  RTC_CHECK(packet_type) << "Packet type must be set before sending.";

  if (packet->capture_time_ms() <= 0) {
    packet->set_capture_time_ms(now_ms);
  }

  std::vector> packets;
  packets.emplace_back(std::move(packet));
  paced_sender_->EnqueuePackets(std::move(packets));  PacedSender::EnqueuePacket 入队,然后再在单独的线程中进行发送

  return true;
}



// 单独的线程中进行数据的发送

void PacedSender::Process() {
  rtc::CritScope cs(&critsect_);
  pacing_controller_.ProcessPackets(); / 
}


void PacingController::ProcessPackets() 
===>
  while (!paused_) {
    if (small_first_probe_packet_ && first_packet_in_probe) {
      // If first packet in probe, insert a small padding packet so we have a
      // more reliable start window for the rate estimation.
      auto padding = packet_sender_->GeneratePadding(DataSize::bytes(1));
      // If no RTP modules sending media are registered, we may not get a
      // padding packet back.
      if (!padding.empty()) {
        // Insert with high priority so larger media packets don't preempt it.
        EnqueuePacketInternal(std::move(padding[0]), kFirstPriority);
        // We should never get more than one padding packets with a requested
        // size of 1 byte.
        RTC_DCHECK_EQ(padding.size(), 1u);
      }
      first_packet_in_probe = false;
    }

    auto* packet = GetPendingPacket(pacing_info); /// 从队列中取包 
    if (packet == nullptr) {
      // No packet available to send, check if we should send padding.
      DataSize padding_to_add = PaddingToAdd(recommended_probe_size, data_sent);
      if (padding_to_add > DataSize::Zero()) {
        std::vector> padding_packets =
            packet_sender_->GeneratePadding(padding_to_add);
        if (padding_packets.empty()) {
          // No padding packets were generated, quite send loop.
          break;
        }
        for (auto& packet : padding_packets) {
          EnqueuePacket(std::move(packet));
        }
        // Continue loop to send the padding that was just added.
        continue;
      }

      // Can't fetch new packet and no padding to send, exit send loop.
      break;
    }

    std::unique_ptr rtp_packet = packet->ReleasePacket();
    RTC_DCHECK(rtp_packet);
    packet_sender_->SendRtpPacket(std::move(rtp_packet), pacing_info); // 真正的发送数据 PacedSender::SendRtpPacket 

    data_sent += packet->size();
    // Send succeeded, remove it from the queue.
    OnPacketSent(packet);
    if (recommended_probe_size && data_sent > *recommended_probe_size)
      break;
  }
  
  

void PacedSender::SendRtpPacket(std::unique_ptr packet,
                                const PacedPacketInfo& cluster_info) {
  critsect_.Leave();
  packet_router_->SendPacket(std::move(packet), cluster_info); / PacketRouter::SendPacket 
  critsect_.Enter();
}
  
  
void PacketRouter::SendPacket(std::unique_ptr packet,
                              const PacedPacketInfo& cluster_info) {
  rtc::CritScope cs(&modules_crit_);
  // With the new pacer code path, transport sequence numbers are only set here,
  // on the pacer thread. Therefore we don't need atomics/synchronization.
  if (packet->IsExtensionReserved()) {
    packet->SetExtension(AllocateSequenceNumber());
  }

  auto it = rtp_module_cache_map_.find(packet->Ssrc());
  if (it != rtp_module_cache_map_.end()) {
    if (TrySendPacket(packet.get(), cluster_info, it->second)) {
      return;
    }
    // Entry is stale, remove it.
    rtp_module_cache_map_.erase(it);
  }

  // Slow path, find the correct send module.
  for (auto* rtp_module : rtp_send_modules_) {
    if (TrySendPacket(packet.get(), cluster_info, rtp_module)) { / PacketRouter::TrySendPacket 
      return;
    }
  }

  RTC_LOG(LS_WARNING) << "Failed to send packet, matching RTP module not found "
                         "or transport error. SSRC = "
                      << packet->Ssrc() << ", sequence number "
                      << packet->SequenceNumber();
}



bool PacketRouter::TrySendPacket(RtpPacketToSend* packet,
                                 const PacedPacketInfo& cluster_info,
                                 RtpRtcp* rtp_module) {
  uint32_t ssrc = packet->Ssrc();
  if (rtp_module->TrySendPacket(packet, cluster_info)) {  ModuleRtpRtcpImpl::TrySendPacket 
    // Sending succeeded, make sure this SSRC mapping for future use.
    rtp_module_cache_map_[ssrc] = rtp_module;

    if (rtp_module->SupportsRtxPayloadPadding()) {
      // This is now the last module to send media, and has the desired
      // properties needed for payload based padding. Cache it for later use.
      last_send_module_ = rtp_module;
    }

    return true;
  }
  return false;
}



bool ModuleRtpRtcpImpl::TrySendPacket(RtpPacketToSend* packet,
                                      const PacedPacketInfo& pacing_info) {
  return rtp_sender_->TrySendPacket(packet, pacing_info); // RTPSender::TrySendPacket 
}


bool RTPSender::TrySendPacket(RtpPacketToSend* packet,
                              const PacedPacketInfo& pacing_info) {
  RTC_DCHECK(packet);

  const uint32_t packet_ssrc = packet->Ssrc();
  const auto packet_type = packet->packet_type();
  RTC_DCHECK(packet_type.has_value());

  PacketOptions options;
  bool is_media = false;
  bool is_rtx = false;
  {
    rtc::CritScope lock(&send_critsect_);
    if (!sending_media_) {
      return false;
    }

    switch (*packet_type) {
      case RtpPacketToSend::Type::kAudio:
      case RtpPacketToSend::Type::kVideo:
        if (packet_ssrc != ssrc_) {
          return false;
        }
        is_media = true;
        break;
      case RtpPacketToSend::Type::kRetransmission:
      case RtpPacketToSend::Type::kPadding:
        // Both padding and retransmission must be on either the media or the
        // RTX stream.
        if (packet_ssrc == rtx_ssrc_) {
          is_rtx = true;
        } else if (packet_ssrc != ssrc_) {
          return false;
        }
        break;
      case RtpPacketToSend::Type::kForwardErrorCorrection:
        // FlexFEC is on separate SSRC, ULPFEC uses media SSRC.
        if (packet_ssrc != ssrc_ && packet_ssrc != flexfec_ssrc_) {
          return false;
        }
        break;
    }

    options.included_in_allocation = force_part_of_allocation_;
  }

  // Bug webrtc:7859. While FEC is invoked from rtp_sender_video, and not after
  // the pacer, these modifications of the header below are happening after the
  // FEC protection packets are calculated. This will corrupt recovered packets
  // at the same place. It's not an issue for extensions, which are present in
  // all the packets (their content just may be incorrect on recovered packets).
  // In case of VideoTimingExtension, since it's present not in every packet,
  // data after rtp header may be corrupted if these packets are protected by
  // the FEC.
  int64_t now_ms = clock_->TimeInMilliseconds();
  int64_t diff_ms = now_ms - packet->capture_time_ms();
  if (packet->IsExtensionReserved()) {
    packet->SetExtension(kTimestampTicksPerMs * diff_ms);
  }
  if (packet->IsExtensionReserved()) {
    packet->SetExtension(
        AbsoluteSendTime::MsTo24Bits(now_ms));
  }

  if (packet->HasExtension()) {
    if (populate_network2_timestamp_) {
      packet->set_network2_time_ms(now_ms);
    } else {
      packet->set_pacer_exit_time_ms(now_ms);
    }
  }

  // Downstream code actually uses this flag to distinguish between media and
  // everything else.
  options.is_retransmit = !is_media;
  if (auto packet_id = packet->GetExtension()) {
    options.packet_id = *packet_id;
    options.included_in_feedback = true;
    options.included_in_allocation = true;
    AddPacketToTransportFeedback(*packet_id, *packet, pacing_info);
  }

  options.application_data.assign(packet->application_data().begin(),
                                  packet->application_data().end());

  if (packet->packet_type() != RtpPacketToSend::Type::kPadding &&
      packet->packet_type() != RtpPacketToSend::Type::kRetransmission) {
    UpdateDelayStatistics(packet->capture_time_ms(), now_ms, packet_ssrc);
    UpdateOnSendPacket(options.packet_id, packet->capture_time_ms(),
                       packet_ssrc);
  }

  const bool send_success = SendPacketToNetwork(*packet, options, pacing_info); 

  // Put packet in retransmission history or update pending status even if
  // actual sending fails.
  if (is_media && packet->allow_retransmission()) {
    packet_history_.PutRtpPacket(std::make_unique(*packet),
                                 now_ms);
  } else if (packet->retransmitted_sequence_number()) {
    packet_history_.MarkPacketAsSent(*packet->retransmitted_sequence_number());
  }

  if (send_success) {
    UpdateRtpStats(*packet, is_rtx,
                   packet_type == RtpPacketToSend::Type::kRetransmission);

    rtc::CritScope lock(&send_critsect_);
    media_has_been_sent_ = true;
  }

  // Return true even if transport failed (will be handled by retransmissions
  // instead in that case), so that PacketRouter does not have to iterate over
  // all other RTP modules and fail to send there too.
  return true;
}




bool RTPSender::SendPacketToNetwork(const RtpPacketToSend& packet,
                                    const PacketOptions& options,
                                    const PacedPacketInfo& pacing_info) {
  int bytes_sent = -1;
  if (transport_) {
    UpdateRtpOverhead(packet);
    bytes_sent = transport_->SendRtp(packet.data(), packet.size(), options) // transport_ 是从 WebRtcVideoChannel::AddSendStream 
                     ? static_cast(packet.size()) // 中的 webrtc::VideoSendStream::Config config(this, media_transport());
                     : -1; // 一步步传递过来的,通过前面 VideoSendStream::Config::Config 的分析可知,此处的 transport_ 指向的就是 WebRtcVideoChannel 
    if (event_log_ && bytes_sent > 0) { / 所以这里调用的就是 WebRtcVideoChannel::SendRtp 
      event_log_->Log(std::make_unique(
          packet, pacing_info.probe_cluster_id));
    }
  }
  // TODO(pwestin): Add a separate bitrate for sent bitrate after pacer.
  if (bytes_sent <= 0) {
    RTC_LOG(LS_WARNING) << "Transport failed to send packet.";
    return false;
  }
  return true;
}


									cricket::VideoChannel* PeerConnection::CreateVideoChannel(
										const std::string& mid) {
									  RtpTransportInternal* rtp_transport = GetRtpTransport(mid); 
									  MediaTransportConfig media_transport_config =
										  transport_controller_->GetMediaTransportConfig(mid); /// JsepTransportController::GetMediaTransportConfig 

									  cricket::VideoChannel* video_channel = channel_manager()->CreateVideoChannel( /// ChannelManager::CreateVideoChannel
										  call_ptr_, configuration_.media_config, rtp_transport,
										  media_transport_config, signaling_thread(), mid, SrtpRequired(),
										  GetCryptoOptions(), &ssrc_generator_, video_options_,
										  video_bitrate_allocator_factory_.get());
									  if (!video_channel) {
										return nullptr;
									  }
									  video_channel->SignalDtlsSrtpSetupFailure.connect(
										  this, &PeerConnection::OnDtlsSrtpSetupFailure);
									  video_channel->SignalSentPacket.connect(this,
																			  &PeerConnection::OnSentPacket_w);
									  video_channel->SetRtpTransport(rtp_transport);

									  return video_channel;
									}



									MediaTransportConfig JsepTransportController::GetMediaTransportConfig(
										const std::string& mid) const {
									  auto jsep_transport = GetJsepTransportForMid(mid);
									  if (!jsep_transport) {
										return MediaTransportConfig();
									  }

									  MediaTransportInterface* media_transport = nullptr;
									  if (config_.use_media_transport_for_media) {
										media_transport = jsep_transport->media_transport();
									  }

									  DatagramTransportInterface* datagram_transport = nullptr;
									  if (config_.use_datagram_transport) {
										datagram_transport = jsep_transport->datagram_transport();
									  }

									  // Media transport and datagram transports can not be used together.
									  RTC_DCHECK(!media_transport || !datagram_transport);

									  if (media_transport) {
										return MediaTransportConfig(media_transport);
									  } else if (datagram_transport) {
										return MediaTransportConfig(
											/*rtp_max_packet_size=*/datagram_transport->GetLargestDatagramSize());
									  } else {
										return MediaTransportConfig();
									  }
									}



									VideoChannel* ChannelManager::CreateVideoChannel(
										webrtc::Call* call,
										const cricket::MediaConfig& media_config,
										webrtc::RtpTransportInternal* rtp_transport,
										const webrtc::MediaTransportConfig& media_transport_config,
										rtc::Thread* signaling_thread,
										const std::string& content_name,
										bool srtp_required,
										const webrtc::CryptoOptions& crypto_options,
										rtc::UniqueRandomIdGenerator* ssrc_generator,
										const VideoOptions& options,
										webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) {
									  if (!worker_thread_->IsCurrent()) {
										return worker_thread_->Invoke(RTC_FROM_HERE, [&] {
										  return CreateVideoChannel(
											  call, media_config, rtp_transport, media_transport_config,
											  signaling_thread, content_name, srtp_required, crypto_options,
											  ssrc_generator, options, video_bitrate_allocator_factory);
										});
									  }

									  RTC_DCHECK_RUN_ON(worker_thread_);
									  RTC_DCHECK(initialized_);
									  RTC_DCHECK(call);
									  if (!media_engine_) {
										return nullptr;
									  }

									  // media_channel 实际指向的是 WebRtcVideoChannel 实例
									  VideoMediaChannel* media_channel = media_engine_->video().CreateMediaChannel( / WebRtcVideoEngine::CreateMediaChannel
										  call, media_config, options, crypto_options,
										  video_bitrate_allocator_factory);
									  if (!media_channel) {
										return nullptr;
									  }

									  auto video_channel = std::make_unique(
										  worker_thread_, network_thread_, signaling_thread,
										  absl::WrapUnique(media_channel), content_name, srtp_required,
										  crypto_options, ssrc_generator);

									  video_channel->Init_w(rtp_transport, media_transport_config); /// 是父类的 BaseChannel::Init_w  

									  VideoChannel* video_channel_ptr = video_channel.get();
									  video_channels_.push_back(std::move(video_channel));
									  return video_channel_ptr;
									}



									VideoMediaChannel* WebRtcVideoEngine::CreateMediaChannel(
										webrtc::Call* call,
										const MediaConfig& config,
										const VideoOptions& options,
										const webrtc::CryptoOptions& crypto_options,
										webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) {
									  RTC_LOG(LS_INFO) << "CreateMediaChannel. Options: " << options.ToString();
									  return new WebRtcVideoChannel(call, config, options, crypto_options,
																	encoder_factory_.get(), decoder_factory_.get(),
																	video_bitrate_allocator_factory);
									}



									void BaseChannel::Init_w(
										webrtc::RtpTransportInternal* rtp_transport,
										const webrtc::MediaTransportConfig& media_transport_config) {
									  RTC_DCHECK_RUN_ON(worker_thread_);
									  media_transport_config_ = media_transport_config;

									  network_thread_->Invoke(
										  RTC_FROM_HERE, [this, rtp_transport] { SetRtpTransport(rtp_transport); });

									  // Both RTP and RTCP channels should be set, we can call SetInterface on
									  // the media channel and it can set network options.
									  media_channel_->SetInterface(this, media_transport_config); // this 指向的是 VideoChannel

									  RTC_LOG(LS_INFO) << "BaseChannel::Init_w, media_transport_config="
													   << media_transport_config.DebugString();
									  if (media_transport_config_.media_transport) {
										media_transport_config_.media_transport->AddNetworkChangeCallback(this);
									  }
									}


									void MediaChannel::SetInterface(
										NetworkInterface* iface, // iface 指向的是 VideoChannel
										const webrtc::MediaTransportConfig& media_transport_config) {
									  rtc::CritScope cs(&network_interface_crit_);
									  network_interface_ = iface; // iface 指向的是 VideoChannel
									  media_transport_config_ = media_transport_config; /// 
									  UpdateDscp();
									}
									
									
									/ 
									class MediaChannel : public sigslot::has_slots<>
									class VideoMediaChannel : public MediaChannel, public Delayable
									class WebRtcVideoChannel : public VideoMediaChannel,
															   public webrtc::Transport,
															   public webrtc::EncoderSwitchRequestCallback
															   
															   
									bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp)
									===>
									 webrtc::VideoSendStream::Config config(this, media_transport()); / media_transport() 是 WebRtcVideoChannel 的父类 MediaChannel 中定义的

									WebRtcVideoChannel::WebRtcVideoChannel(
										webrtc::Call* call,
										const MediaConfig& config,
										const VideoOptions& options,
										const webrtc::CryptoOptions& crypto_options,
										webrtc::VideoEncoderFactory* encoder_factory,
										webrtc::VideoDecoderFactory* decoder_factory,
										webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory)
										: VideoMediaChannel(config), // 


									 VideoMediaChannel 的构造函数直接在其头文件中定义 
									explicit VideoMediaChannel(const MediaConfig& config)
									  : MediaChannel(config) {} ///
									  

									MediaChannel::MediaChannel(const MediaConfig& config)
										: enable_dscp_(config.enable_dscp) {}


									// 在 MediaChannel 头文件中定义,也就是 MediaChannel::media_transport
									webrtc::MediaTransportInterface* media_transport() {
									return media_transport_config_.media_transport;
									}

									  



	  
	  

bool WebRtcVideoChannel::SendRtp(const uint8_t* data,
                                 size_t len,
                                 const webrtc::PacketOptions& options) {
  rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen);
  rtc::PacketOptions rtc_options;
  rtc_options.packet_id = options.packet_id;
  if (DscpEnabled()) {
    rtc_options.dscp = PreferredDscp();
  }
  rtc_options.info_signaled_after_sent.included_in_feedback =
      options.included_in_feedback;
  rtc_options.info_signaled_after_sent.included_in_allocation =
      options.included_in_allocation;
  return MediaChannel::SendPacket(&packet, rtc_options); /// 
}


// 直接在 MediaChannel 的头文件中定义
bool SendPacket(rtc::CopyOnWriteBuffer* packet,
			  const rtc::PacketOptions& options) {
return DoSendPacket(packet, false, options);  
}



class BaseChannel : public ChannelInterface,
                    public rtc::MessageHandler,
                    public sigslot::has_slots<>,
                    public MediaChannel::NetworkInterface, //
                    public webrtc::RtpPacketSinkInterface,
                    public webrtc::MediaTransportNetworkChangeCallback 
					
class VideoChannel : public BaseChannel


// 直接在 MediaChannel 的头文件中定义
bool DoSendPacket(rtc::CopyOnWriteBuffer* packet,
				bool rtcp,
				const rtc::PacketOptions& options) {
rtc::CritScope cs(&network_interface_crit_);
if (!network_interface_)
  return false;
 由前面 MediaChannel::SetInterface 可知 network_interface_ 指向的是 VideoChannel
return (!rtcp) ? network_interface_->SendPacket(packet, options)
			   : network_interface_->SendRtcp(packet, options); // NetworkInterface* network_interface_ 
}
  

bool BaseChannel::SendPacket(rtc::CopyOnWriteBuffer* packet,
                             const rtc::PacketOptions& options) {
  return SendPacket(false, packet, options);
}


bool BaseChannel::SendPacket(bool rtcp,
                             rtc::CopyOnWriteBuffer* packet,
                             const rtc::PacketOptions& options) {
  // Until all the code is migrated to use RtpPacketType instead of bool.
  RtpPacketType packet_type = rtcp ? RtpPacketType::kRtcp : RtpPacketType::kRtp;
  // SendPacket gets called from MediaEngine, on a pacer or an encoder thread.
  // If the thread is not our network thread, we will post to our network
  // so that the real work happens on our network. This avoids us having to
  // synchronize access to all the pieces of the send path, including
  // SRTP and the inner workings of the transport channels.
  // The only downside is that we can't return a proper failure code if
  // needed. Since UDP is unreliable anyway, this should be a non-issue.
  if (!network_thread_->IsCurrent()) {
    // Avoid a copy by transferring the ownership of the packet data.
    int message_id = rtcp ? MSG_SEND_RTCP_PACKET : MSG_SEND_RTP_PACKET;
    SendPacketMessageData* data = new SendPacketMessageData;
    data->packet = std::move(*packet);
    data->options = options;
    network_thread_->Post(RTC_FROM_HERE, this, message_id, data);
    return true;
  }

  TRACE_EVENT0("webrtc", "BaseChannel::SendPacket");

  // Now that we are on the correct thread, ensure we have a place to send this
  // packet before doing anything. (We might get RTCP packets that we don't
  // intend to send.) If we've negotiated RTCP mux, send RTCP over the RTP
  // transport.
  if (!rtp_transport_ || !rtp_transport_->IsWritable(rtcp)) {
    return false;
  }

  // Protect ourselves against crazy data.
  if (!IsValidRtpPacketSize(packet_type, packet->size())) {
    RTC_LOG(LS_ERROR) << "Dropping outgoing " << content_name_ << " "
                      << RtpPacketTypeToString(packet_type)
                      << " packet: wrong size=" << packet->size();
    return false;
  }

  if (!srtp_active()) {
    if (srtp_required_) {
      // The audio/video engines may attempt to send RTCP packets as soon as the
      // streams are created, so don't treat this as an error for RTCP.
      // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=6809
      if (rtcp) {
        return false;
      }
      // However, there shouldn't be any RTP packets sent before SRTP is set up
      // (and SetSend(true) is called).
      RTC_LOG(LS_ERROR)
          << "Can't send outgoing RTP packet when SRTP is inactive"
          << " and crypto is required";
      RTC_NOTREACHED();
      return false;
    }

    std::string packet_type = rtcp ? "RTCP" : "RTP";
    RTC_LOG(LS_WARNING) << "Sending an " << packet_type
                        << " packet without encryption.";
  }

  // Bon voyage.
  return rtcp ? rtp_transport_->SendRtcpPacket(packet, options, PF_SRTP_BYPASS) 
              : rtp_transport_->SendRtpPacket(packet, options, PF_SRTP_BYPASS);/  BaseChannel::SetRtpTransport 获取的是 webrtc::DtlsSrtpTransport 
} /所以这里是 webrtc::DtlsSrtpTransport::SendRtpPacket, 该函数继承自 SrtpTransport::SendRtpPacket 



				cricket::VideoChannel* PeerConnection::CreateVideoChannel(
					const std::string& mid) {
				  RtpTransportInternal* rtp_transport = GetRtpTransport(mid); /// 获取 rtp_transport,见下面分析,返回的是 webrtc::DtlsSrtpTransport 

				  cricket::VideoChannel* video_channel = channel_manager()->CreateVideoChannel( 
					  call_ptr_, configuration_.media_config, rtp_transport, signaling_thread(), /// 使用 rtp_transport
					  mid, SrtpRequired(), GetCryptoOptions(), &ssrc_generator_, video_options_,
					  video_bitrate_allocator_factory_.get());
				  if (!video_channel) {
					return nullptr;
				  }
				  video_channel->SignalDtlsSrtpSetupFailure.connect(
					  this, &PeerConnection::OnDtlsSrtpSetupFailure);
				  video_channel->SignalSentPacket.connect(this,
														  &PeerConnection::OnSentPacket_w);
				  video_channel->SetRtpTransport(rtp_transport); /// 使用 rtp_transport

				  return video_channel;
				}


				RtpTransportInternal* GetRtpTransport(const std::string& mid)
				  RTC_RUN_ON(signaling_thread()) {
				auto rtp_transport = transport_controller_->GetRtpTransport(mid); / 
				RTC_DCHECK(rtp_transport);
				return rtp_transport;
				}


				RtpTransportInternal* JsepTransportController::GetRtpTransport(
					const std::string& mid) const {
				  auto jsep_transport = GetJsepTransportForMid(mid);
				  if (!jsep_transport) {
					return nullptr;
				  }
				  return jsep_transport->rtp_transport(); /// JsepTransport::rtp_transport
				}


				 JsepTransport::rtp_transport 
				  webrtc::RtpTransportInternal* rtp_transport() const {
					rtc::CritScope scope(&accessor_lock_);
					if (composite_rtp_transport_) {
					  return composite_rtp_transport_.get(); /// std::unique_ptr composite_rtp_transport_
					} else if (datagram_rtp_transport_) {
					  return datagram_rtp_transport_.get(); ///  std::unique_ptr datagram_rtp_transport_ 
					} else {
					  return default_rtp_transport(); // 在例子的调试过程中会运行到这里 
					}
				  }
				  
				  
				  
				  
				  webrtc::RtpTransportInternal* default_rtp_transport() const
					  RTC_EXCLUSIVE_LOCKS_REQUIRED(accessor_lock_) {
					if (dtls_srtp_transport_) {
					  return dtls_srtp_transport_.get(); // 会执行到这里,返回的就是 std::unique_ptr dtls_srtp_transport_
					} else if (sdes_transport_) {
					  return sdes_transport_.get();
					} else if (unencrypted_rtp_transport_) {
					  return unencrypted_rtp_transport_.get();
					} else {
					  return nullptr;
					}
				  }				  
  
  
  

  bool SrtpTransport::SendRtpPacket(rtc::CopyOnWriteBuffer* packet,
                                  const rtc::PacketOptions& options,
                                  int flags) {
  if (!IsSrtpActive()) {
    RTC_LOG(LS_ERROR)
        << "Failed to send the packet because SRTP transport is inactive.";
    return false;
  }
  rtc::PacketOptions updated_options = options;
  TRACE_EVENT0("webrtc", "SRTP Encode");
  bool res;
  uint8_t* data = packet->data();
  int len = rtc::checked_cast(packet->size());
// If ENABLE_EXTERNAL_AUTH flag is on then packet authentication is not done
// inside libsrtp for a RTP packet. A external HMAC module will be writing
// a fake HMAC value. This is ONLY done for a RTP packet.
// Socket layer will update rtp sendtime extension header if present in
// packet with current time before updating the HMAC.
#if !defined(ENABLE_EXTERNAL_AUTH)
  res = ProtectRtp(data, len, static_cast(packet->capacity()), &len);
#else
  if (!IsExternalAuthActive()) {
    res = ProtectRtp(data, len, static_cast(packet->capacity()), &len);
  } else {
    updated_options.packet_time_params.rtp_sendtime_extension_id =
        rtp_abs_sendtime_extn_id_;
    res = ProtectRtp(data, len, static_cast(packet->capacity()), &len,
                     &updated_options.packet_time_params.srtp_packet_index);
    // If protection succeeds, let's get auth params from srtp.
    if (res) {
      uint8_t* auth_key = nullptr;
      int key_len = 0;
      res = GetRtpAuthParams(
          &auth_key, &key_len,
          &updated_options.packet_time_params.srtp_auth_tag_len);
      if (res) {
        updated_options.packet_time_params.srtp_auth_key.resize(key_len);
        updated_options.packet_time_params.srtp_auth_key.assign(
            auth_key, auth_key + key_len);
      }
    }
  }
#endif
  if (!res) {
    int seq_num = -1;
    uint32_t ssrc = 0;
    cricket::GetRtpSeqNum(data, len, &seq_num);
    cricket::GetRtpSsrc(data, len, &ssrc);
    RTC_LOG(LS_ERROR) << "Failed to protect RTP packet: size=" << len
                      << ", seqnum=" << seq_num << ", SSRC=" << ssrc;
    return false;
  }

  // Update the length of the packet now that we've added the auth tag.
  packet->SetSize(len);
  return SendPacket(/*rtcp=*/false, packet, updated_options, flags); /// RtpTransport::SendPacket 
}



bool RtpTransport::SendPacket(bool rtcp,
                              rtc::CopyOnWriteBuffer* packet,
                              const rtc::PacketOptions& options,
                              int flags) {
  rtc::PacketTransportInternal* transport = rtcp && !rtcp_mux_enabled_
                                                ? rtcp_packet_transport_
                                                : rtp_packet_transport_; / 见 JsepTransportController::CreateDtlsSrtpTransport 分析 
  int ret = transport->SendPacket(packet->cdata(), packet->size(), / 经过下面的分析,这里执行的是 cricket::DtlsTransport::SendPacket
                                  options, flags);
  if (ret != static_cast(packet->size())) {
    if (transport->GetError() == ENOTCONN) {
      RTC_LOG(LS_WARNING) << "Got ENOTCONN from transport.";
      SetReadyToSend(rtcp, false);
    }
    return false;
  }
  return true;
}


								class RtpTransportInternal : public sigslot::has_slots<>
								class RtpTransport : public RtpTransportInternal 
								class SrtpTransport : public RtpTransport 
								class DtlsSrtpTransport : public SrtpTransport


								std::unique_ptr
								JsepTransportController::CreateDtlsSrtpTransport(
									const std::string& transport_name,
									cricket::DtlsTransportInternal* rtp_dtls_transport,
									cricket::DtlsTransportInternal* rtcp_dtls_transport) {
								  RTC_DCHECK(network_thread_->IsCurrent());
								  auto dtls_srtp_transport = std::make_unique(  
									  rtcp_dtls_transport == nullptr);
								  if (config_.enable_external_auth) {
									dtls_srtp_transport->EnableExternalAuth();
								  }

								  dtls_srtp_transport->SetDtlsTransports(rtp_dtls_transport,
																		 rtcp_dtls_transport);
								  dtls_srtp_transport->SetActiveResetSrtpParams(
									  config_.active_reset_srtp_params);
								  dtls_srtp_transport->SignalDtlsStateChange.connect(
									  this, &JsepTransportController::UpdateAggregateStates_n);
								  return dtls_srtp_transport;
								}



								DtlsSrtpTransport::DtlsSrtpTransport(bool rtcp_mux_enabled)
									: SrtpTransport(rtcp_mux_enabled) {}
									
								SrtpTransport::SrtpTransport(bool rtcp_mux_enabled)
									: RtpTransport(rtcp_mux_enabled) {}

								explicit RtpTransport(bool rtcp_mux_enabled)
								  : rtcp_mux_enabled_(rtcp_mux_enabled) {}	


								void DtlsSrtpTransport::SetDtlsTransports(
									cricket::DtlsTransportInternal* rtp_dtls_transport,
									cricket::DtlsTransportInternal* rtcp_dtls_transport) {
								  // Transport names should be the same.
								  if (rtp_dtls_transport && rtcp_dtls_transport) {
									RTC_DCHECK(rtp_dtls_transport->transport_name() ==
											   rtcp_dtls_transport->transport_name());
								  }

								  // When using DTLS-SRTP, we must reset the SrtpTransport every time the
								  // DtlsTransport changes and wait until the DTLS handshake is complete to set
								  // the newly negotiated parameters.
								  // If |active_reset_srtp_params_| is true, intentionally reset the SRTP
								  // parameter even though the DtlsTransport may not change.
								  if (IsSrtpActive() && (rtp_dtls_transport != rtp_dtls_transport_ ||
														 active_reset_srtp_params_)) {
									ResetParams();
								  }

								  const std::string transport_name =
									  rtp_dtls_transport ? rtp_dtls_transport->transport_name() : "null";

								  if (rtcp_dtls_transport && rtcp_dtls_transport != rtcp_dtls_transport_) {
									// This would only be possible if using BUNDLE but not rtcp-mux, which isn't
									// allowed according to the BUNDLE spec.
									RTC_CHECK(!(IsSrtpActive()))
										<< "Setting RTCP for DTLS/SRTP after the DTLS is active "
										   "should never happen.";
								  }

								  RTC_LOG(LS_INFO) << "Setting RTCP Transport on " << transport_name
												   << " transport " << rtcp_dtls_transport;
								  SetRtcpDtlsTransport(rtcp_dtls_transport);
								  SetRtcpPacketTransport(rtcp_dtls_transport);

								  RTC_LOG(LS_INFO) << "Setting RTP Transport on " << transport_name
												   << " transport " << rtp_dtls_transport;
								  SetRtpDtlsTransport(rtp_dtls_transport); / DtlsSrtpTransport::SetRtpDtlsTransport
								  SetRtpPacketTransport(rtp_dtls_transport); ///  RtpTransport::SetRtpPacketTransport 就是这里设置了上面 RtpTransport::SendPacket 中使用的 rtp_packet_transport_

								  MaybeSetupDtlsSrtp();
								}


								void DtlsSrtpTransport::SetRtpDtlsTransport(
									cricket::DtlsTransportInternal* rtp_dtls_transport) {
								  SetDtlsTransport(rtp_dtls_transport, &rtp_dtls_transport_);
								}

								void DtlsSrtpTransport::SetDtlsTransport(
									cricket::DtlsTransportInternal* new_dtls_transport,
									cricket::DtlsTransportInternal** old_dtls_transport) {
								  if (*old_dtls_transport == new_dtls_transport) {
									return;
								  }

								  if (*old_dtls_transport) {
									(*old_dtls_transport)->SignalDtlsState.disconnect(this);
								  }

								  *old_dtls_transport = new_dtls_transport;

								  if (new_dtls_transport) {
									new_dtls_transport->SignalDtlsState.connect(
										this, &DtlsSrtpTransport::OnDtlsState);
								  }
								}






								void RtpTransport::SetRtpPacketTransport(
									rtc::PacketTransportInternal* new_packet_transport) {
								  if (new_packet_transport == rtp_packet_transport_) {
									return;
								  }
								  if (rtp_packet_transport_) {
									rtp_packet_transport_->SignalReadyToSend.disconnect(this);
									rtp_packet_transport_->SignalReadPacket.disconnect(this);
									rtp_packet_transport_->SignalNetworkRouteChanged.disconnect(this);
									rtp_packet_transport_->SignalWritableState.disconnect(this);
									rtp_packet_transport_->SignalSentPacket.disconnect(this);
									// Reset the network route of the old transport.
									SignalNetworkRouteChanged(absl::optional());
								  }
								  if (new_packet_transport) {
									new_packet_transport->SignalReadyToSend.connect(
										this, &RtpTransport::OnReadyToSend);
									new_packet_transport->SignalReadPacket.connect(this,
																				   &RtpTransport::OnReadPacket);
									new_packet_transport->SignalNetworkRouteChanged.connect(
										this, &RtpTransport::OnNetworkRouteChanged);
									new_packet_transport->SignalWritableState.connect(
										this, &RtpTransport::OnWritableState);
									new_packet_transport->SignalSentPacket.connect(this,
																				   &RtpTransport::OnSentPacket);
									// Set the network route for the new transport.
									SignalNetworkRouteChanged(new_packet_transport->network_route());
								  }

								  rtp_packet_transport_ = new_packet_transport;  实际指向的是 cricket::DtlsTransport 
								  // Assumes the transport is ready to send if it is writable. If we are wrong,
								  // ready to send will be updated the next time we try to send.
								  SetReadyToSend(false,
												 rtp_packet_transport_ && rtp_packet_transport_->writable());
								}



 cricket::DtlsTransport::SendPacket 

int DtlsTransport::SendPacket(const char* data,
                              size_t size,
                              const rtc::PacketOptions& options,
                              int flags) {
  if (!dtls_active_) {
    // Not doing DTLS.
    return ice_transport_->SendPacket(data, size, options);
  }

  switch (dtls_state()) {
    case DTLS_TRANSPORT_NEW:
      // Can't send data until the connection is active.
      // TODO([email protected]): assert here if dtls_ is NULL?
      return -1;
    case DTLS_TRANSPORT_CONNECTING:
      // Can't send data until the connection is active.
      return -1;
    case DTLS_TRANSPORT_CONNECTED:
      if (flags & PF_SRTP_BYPASS) {
        RTC_DCHECK(!srtp_ciphers_.empty());
        if (!IsRtpPacket(data, size)) {
          return -1;
        }
		 ice_transport_ 实际上指向 cricket::P2PTransportChannel 
        return ice_transport_->SendPacket(data, size, options); // P2PTransportChannel::SendPacket 
      } else {
        return (dtls_->WriteAll(data, size, NULL, NULL) == rtc::SR_SUCCESS)
                   ? static_cast(size)
                   : -1;
      }
    case DTLS_TRANSPORT_FAILED:
    case DTLS_TRANSPORT_CLOSED:
      // Can't send anything when we're closed.
      return -1;
    default:
      RTC_NOTREACHED();
      return -1;
  }
}



// Send data to the other side, using our selected connection.
int P2PTransportChannel::SendPacket(const char* data,
                                    size_t len,
                                    const rtc::PacketOptions& options,
                                    int flags) {
  RTC_DCHECK_RUN_ON(network_thread_);
  if (flags != 0) {
    error_ = EINVAL;
    return -1;
  }
  // If we don't think the connection is working yet, return ENOTCONN
  // instead of sending a packet that will probably be dropped.
  if (!ReadyToSend(selected_connection_)) {
    error_ = ENOTCONN;
    return -1;
  }

  last_sent_packet_id_ = options.packet_id;
  rtc::PacketOptions modified_options(options);
  modified_options.info_signaled_after_sent.packet_type =
      rtc::PacketType::kData;
  int sent = selected_connection_->Send(data, len, modified_options); / 利用选中的连接进行数据传输 //
  if (sent <= 0) {
    RTC_DCHECK(sent < 0);
    error_ = selected_connection_->GetError();
  }
  return sent;
}

你可能感兴趣的:(webrtc,webrtc,音视频)