WebRtc语音整体框架

图一语音整体框架图 如上图所示，音频整个处理框架除了ligjingle负责p2p数据的传输，主要是VOE(Voice Engine)和Channel适配层 图二创建数据通信channel时序图 上图是本地端 的完整过程，VOE由CreateMediaEngine_w开始创建，Channel适配层由SetLocalDescription根据SDP开始创建，下面来分析下这两个过程 VOE创建过程 /*src\talk\app\webrtc\peerconnectionfactory.cc*/ bool PeerConnectionFactory::Initialize() { ...... default_allocator_factory_ = PortAllocatorFactory::Create(worker_thread_); ..... cricket::MediaEngineInterface* media_engine = worker_thread_->Invoke(rtc::Bind( &PeerConnectionFactory::CreateMediaEngine_w, this)); //定义的宏，实际上就是在worker_thread_线程上运行CreateMediaEngine_w ..... channel_manager_.reset( new cricket::ChannelManager(media_engine, worker_thread_)); ...... } cricket::MediaEngineInterface* PeerConnectionFactory::CreateMediaEngine_w() { ASSERT(worker_thread_ == rtc::Thread::Current()); return cricket::WebRtcMediaEngineFactory::Create( default_adm_.get(), video_encoder_factory_.get(), video_decoder_factory_.get()); } MediaEngineInterface* WebRtcMediaEngineFactory::Create( webrtc::AudioDeviceModule* adm, WebRtcVideoEncoderFactory* encoder_factory, WebRtcVideoDecoderFactory* decoder_factory) { return CreateWebRtcMediaEngine(adm, encoder_factory, decoder_factory); } //CreateWebRtcMediaEngine实际上是WebRtcMediaEngine2，而WebRtcMediaEngine2又是继承至CompositeMediaEngine //模板类，实现在webrtcmediaengine.cc namespace cricket { class WebRtcMediaEngine2 : public CompositeMediaEngine { public: WebRtcMediaEngine2(webrtc::AudioDeviceModule* adm, WebRtcVideoEncoderFactory* encoder_factory, WebRtcVideoDecoderFactory* decoder_factory) { voice_.SetAudioDeviceModule(adm); video_.SetExternalDecoderFactory(decoder_factory); video_.SetExternalEncoderFactory(encoder_factory); } }; } // namespace cricket template class CompositeMediaEngine : public MediaEngineInterface { public: virtual ~CompositeMediaEngine() {} virtual bool Init(rtc::Thread* worker_thread) { if (!voice_.Init(worker_thread)) //此处的voice 即为WebRtcVoiceEngine return false; video_.Init(); //video 为WebRtcVideoEngine2 后面再分析 return true; } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 相关类图如下： 图三VOE引擎类图 WebRtcVoiceEngine::WebRtcVoiceEngine() : voe_wrapper_(new VoEWrapper()), //底层Voice Engine代理类，与底层相关的上层都调用此类完成 tracing_(new VoETraceWrapper()), //调试相关类 adm_(NULL), log_filter_(SeverityToFilter(kDefaultLogSeverity)), is_dumping_aec_(false) { Construct(); } 1 2 3 4 5 6 7 8 下面看看构造WebRtcVoiceEngine相关的类和方法： //VoEWrapper实际上是VoiceEngine--> voice_engine_impl.cc的代理 /* webrtcvoe.h */ class VoEWrapper { public: VoEWrapper() : engine_(webrtc::VoiceEngine::Create()), processing_(engine_), base_(engine_), codec_(engine_), dtmf_(engine_), hw_(engine_), neteq_(engine_), network_(engine_), rtp_(engine_), sync_(engine_), volume_(engine_) { } /*webrtcvoiceengine.cc*/ void WebRtcVoiceEngine::Construct() { ...... //注册引擎状态回调函数，将底层错误信息告知WebRtcVoiceEngine if (voe_wrapper_->base()->RegisterVoiceEngineObserver(*this) == -1) { LOG_RTCERR0(RegisterVoiceEngineObserver); } .... // Load our audio codec list. ConstructCodecs(); // 根据kCodecPrefs表，音质从高到低，从底层获取最高音质的codec ..... options_ = GetDefaultEngineOptions(); //设置默认的音频选项，需要回音消除，降噪，自动调节音量，是否需要dump等... } //WebRtcVoiceEngine初始化函数 bool WebRtcVoiceEngine::Init(rtc::Thread* worker_thread) { ...... bool res = InitInternal(); ...... } bool WebRtcVoiceEngine::InitInternal() { ...... // 初始化底层AudioDeviceModule 在WebRtc中参数dbm_此处传入的是NULL. //voe_wrapper_ 是VoiceEngine的代理类在voice_engine_impl.cc 中实现， //而VoiceEngineImpl继承至VoiceEngine，creat时创建的是VoiceEngineImpl //在voe_base_impl.cc中实现 //并将对象返回给VoEWrapper //此处voe_wrapper_->base()实际上是VoiceEngineImpl对象,下面分析VoiceEngineImpl.Init if (voe_wrapper_->base()->Init(adm_) == -1) { //voe_wrapper_->base() ...... } ...... } /*voe_base_impl.cc*/ int VoEBaseImpl::Init(AudioDeviceModule* external_adm, AudioProcessing* audioproc) { ...... if (external_adm == nullptr) { //上面已经提到，demo中传入的是null #if !defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE) return -1; #else // Create the internal ADM implementation. //创建本地的AudioDeviceModuleImpl 对象 //通过AudioRecorder 和AudioTrack实现音频采集与播放 shared_->set_audio_device(AudioDeviceModuleImpl::Create( VoEId(shared_->instance_id(), -1), shared_->audio_device_layer())); if (shared_->audio_device() == nullptr) { shared_->SetLastError(VE_NO_MEMORY, kTraceCritical, "Init() failed to create the ADM"); return -1; } #endif // WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE } else { // Use the already existing external ADM implementation. shared_->set_audio_device(external_adm); LOG_F(LS_INFO) process_thread()) { shared_->process_thread()->RegisterModule(shared_->audio_device()); } bool available = false; // -------------------- // Reinitialize the ADM // 为音频设备设置监听器 if (shared_->audio_device()->RegisterEventObserver(this) != 0) { shared_->SetLastError( VE_AUDIO_DEVICE_MODULE_ERROR, kTraceWarning, "Init() failed to register event observer for the ADM"); } // 为音频设备注册AudioTransport的实现，实现音频数据的传输 if (shared_->audio_device()->RegisterAudioCallback(this) != 0) { shared_->SetLastError( VE_AUDIO_DEVICE_MODULE_ERROR, kTraceWarning, "Init() failed to register audio callback for the ADM"); } // 音频设备的初始化！ if (shared_->audio_device()->Init() != 0) { shared_->SetLastError(VE_AUDIO_DEVICE_MODULE_ERROR, kTraceError, "Init() failed to initialize the ADM"); return -1; } ...... } AudioDeviceModule* AudioDeviceModuleImpl::Create(const int32_t id, const AudioLayer audioLayer){ ...... RefCountImpl* audioDevice = new RefCountImpl(id, audioLayer); // 检查平台是否支持 if (audioDevice->CheckPlatform() == -1) { delete audioDevice; return NULL; } // 根据不同的平台选择不同的实现，Android平台 是通过JNI的方式( audio_record_jni.cc audio_track_jni.cc), //获取java层的org/webrtc/voiceengine/WebRtcAudioRecord.java //和org/webrtc/voiceengine/WebRtcAudioTrack.java 实现音频采集和播放 if (audioDevice->CreatePlatformSpecificObjects() == -1) { delete audioDevice; return NULL; } // 分配共享内存，通过AudioTransportS实现音频数据的传递 if (audioDevice->AttachAudioBuffer() == -1) { delete audioDevice; return NULL; } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 Channel创建过程 在图二时序图中，在SetLocalDescription中会调用CreateChannels创建根据SDP创建会话所需要的Channels.由此开启了音视频数据和用户数据传输通道，下面详细看看音频channel创建的过程，其他的类似： 相关类图如下： /* webrtcsession.cc */ bool WebRtcSession::CreateChannels(const SessionDescription* desc) { // Creating the media channels and transport proxies. //根据SDP创建VoiceChannel const cricket::ContentInfo* voice = cricket::GetFirstAudioContent(desc); if (voice && !voice->rejected && !voice_channel_) { if (!CreateVoiceChannel(voice)) { LOG(LS_ERROR) rejected && !video_channel_) { if (!CreateVideoChannel(video)) { LOG(LS_ERROR) rejected && !data_channel_) { if (!CreateDataChannel(data)) { LOG(LS_ERROR) CreateVoiceChannel( media_controller_.get(), transport_controller(), content->name, true, audio_options_)); if (!voice_channel_) { return false; } ...... return true; } /* webrtc\src\talk\session\media\channelmanager.cc*/ VoiceChannel* ChannelManager::CreateVoiceChannel( webrtc::MediaControllerInterface* media_controller, TransportController* transport_controller, const std::string& content_name, bool rtcp, const AudioOptions& options) { //定义的宏，实际意思是 在worker_thread_中运行ChannelManager::CreateVoiceChannel_w方法！ return worker_thread_->Invoke( Bind(&ChannelManager::CreateVoiceChannel_w, this, media_controller, transport_controller, content_name, rtcp, options)); } VoiceChannel* ChannelManager::CreateVoiceChannel_w( webrtc::MediaControllerInterface* media_controller, TransportController* transport_controller, const std::string& content_name, bool rtcp, const AudioOptions& options) { ...... //此处的media_engine_为在peerconnectionfactory.cc中创建的WebRtcMediaEngine2 //最终调用WebRtcVoiceEngine::CreateChannel方法 VoiceMediaChannel* media_channel = media_engine_->CreateChannel(media_controller->call_w(), options); if (!media_channel) return nullptr; //VoiceChannel继承BaseChannel，从libjingle获取数据或者是通过libjingle将数据发给远程端！ VoiceChannel* voice_channel = new VoiceChannel(worker_thread_, media_engine_.get(), media_channel, transport_controller, content_name, rtcp); if (!voice_channel->Init()) { delete voice_channel; return nullptr; } voice_channels_.push_back(voice_channel); return voice_channel; } VoiceMediaChannel* WebRtcVoiceEngine::CreateChannel(webrtc::Call* call, const AudioOptions& options) { WebRtcVoiceMediaChannel* ch = new WebRtcVoiceMediaChannel(this, options, call); if (!ch->valid()) { delete ch; return nullptr; } return ch; } WebRtcVoiceMediaChannel::WebRtcVoiceMediaChannel(WebRtcVoiceEngine* engine, const AudioOptions& options, webrtc::Call* call) : engine_(engine), voe_channel_(engine->CreateMediaVoiceChannel()),//调用WebRtcVoiceEngine::CreateMediaVoiceChannel()方法 ...... { //将当前WebRtcVoiceMediaChannel注册给WebRtcVoiceEngine管理放入ChannelList中 engine->RegisterChannel(this); ...... //为上面创造的新channel注册WebRtcVoiceMediaChannel.可以认为WebRtcVoiceMediaChannel是桥梁，底层 //channel通过注册的Transport实现数据流的发送和接受！ ConfigureSendChannel(voe_channel()); SetOptions(options); } int WebRtcVoiceEngine::CreateVoiceChannel(VoEWrapper* voice_engine_wrapper) { //VoEWrapper为VoiceEngine的封装，我觉得相当于是VoiceEngine的代理。 //而在VoiceEngine的实现voice_engine_impl.cc可以看出，VoiceEngine实际上是VoiceEngineImpl的封装 //voice_engine_wrapper->base()的到的是VoiceEngineImpl对象 return voice_engine_wrapper->base()->CreateChannel(voe_config_); } /* voe_base_impl.cc */ int VoEBaseImpl::CreateChannel() { ..... //通过ChannelManager创建Channel对象 voe::ChannelOwner channel_owner = shared_->channel_manager().CreateChannel(); return InitializeChannel(&channel_owner); } /* android\webrtc\src\webrtc\voice_engine\channel_manager.cc*/ ChannelOwner ChannelManager::CreateChannel() { return CreateChannelInternal(config_); } ChannelOwner ChannelManager::CreateChannelInternal(const Config& config) { Channel* channel; //新建Channel对象 Channel::CreateChannel(channel, ++last_channel_id_, instance_id_, event_log_.get(), config); ChannelOwner channel_owner(channel); CriticalSectionScoped crit(lock_.get()); //ChannelManager对所有新建channel的管理 channels_.push_back(channel_owner); //返回封装的ChannelOwner return channel_owner; } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 语音发送流程 采集 在安卓系统的WebRtc demo中，语音还是通过系统的AudioRecorder.java 类实现采集的。在VoEBaseImpl::Init阶段介绍过会为AudioDeviceModule注册数据传输回调函数如下： int VoEBaseImpl::Init(AudioDeviceModule* external_adm, AudioProcessing* audioproc) { ...... // Register the AudioTransport implementation if (shared_->audio_device()->RegisterAudioCallback(this) != 0) { shared_->SetLastError( VE_AUDIO_DEVICE_MODULE_ERROR, kTraceWarning, "Init() failed to register audio callback for the ADM"); } ...... } int32_t AudioDeviceModuleImpl::RegisterAudioCallback(AudioTransport* audioCallback) { CriticalSectionScoped lock(&_critSectAudioCb); //最终将VoEBaseImpl的实现，注册到设备的AudioDeviceBuffer中 _audioDeviceBuffer.RegisterAudioCallback(audioCallback); return 0; } 所以总的来说音频数据会如下流程，最终VoEBaseImpl实现的AudioTransport回调获取数据或者播放数据！ nativeDataIsRecorded(org/webrtc/voiceengine/WebRtcAudioRecord.java)---> (audio_record_jni.cc)AudioRecordJni::DataIsRecorded-->OnDataIsRecorded--> AudioDeviceBuffer.DeliverRecordedData---> AudioTransport.RecordedDataIsAvailable---> (voe_base_impl.cc)VoEBaseImpl::RecordedDataIsAvailable 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 处理 /* voe_base_impl.cc */ int32_t VoEBaseImpl::RecordedDataIsAvailable( const void* audioSamples, size_t nSamples, size_t nBytesPerSample, uint8_t nChannels, uint32_t samplesPerSec, uint32_t totalDelayMS, int32_t clockDrift, uint32_t micLevel, bool keyPressed, uint32_t& newMicLevel) { newMicLevel = static_cast(ProcessRecordedDataWithAPM( nullptr, 0, audioSamples, samplesPerSec, nChannels, nSamples, totalDelayMS, clockDrift, micLevel, keyPressed)); return 0; } //从java层获取的数据，直接送入ProcessRecordedDataWithAPM处理！ int VoEBaseImpl::ProcessRecordedDataWithAPM( const int voe_channels[], int number_of_voe_channels, const void* audio_data, uint32_t sample_rate, uint8_t number_of_channels, size_t number_of_frames, uint32_t audio_delay_milliseconds, int32_t clock_drift, uint32_t volume, bool key_pressed) { ...... //调节音量 if (volume != 0) { // Scale from ADM to VoE level range if (shared_->audio_device()->MaxMicrophoneVolume(&max_volume) == 0) { if (max_volume) { voe_mic_level = static_cast( (volume * kMaxVolumeLevel + static_cast(max_volume / 2)) / max_volume); } } // We learned that on certain systems (e.g Linux) the voe_mic_level // can be greater than the maxVolumeLevel therefore // we are going to cap the voe_mic_level to the maxVolumeLevel // and change the maxVolume to volume if it turns out that // the voe_mic_level is indeed greater than the maxVolumeLevel. if (voe_mic_level > kMaxVolumeLevel) { voe_mic_level = kMaxVolumeLevel; max_volume = volume; } } //这里对音频有一系列的处理，比如：录制到文件，重采样，回音消除，AGC调节等。。。 shared_->transmit_mixer()->PrepareDemux( audio_data, number_of_frames, number_of_channels, sample_rate, static_cast(audio_delay_milliseconds), clock_drift, voe_mic_level, key_pressed); // Copy the audio frame to each sending channel and perform // channel-dependent operations (file mixing, mute, etc.), encode and // packetize+transmit the RTP packet. When |number_of_voe_channels| == 0, // do the operations on all the existing VoE channels; otherwise the // operations will be done on specific channels. if (number_of_voe_channels == 0) { shared_->transmit_mixer()->DemuxAndMix(); shared_->transmit_mixer()->EncodeAndSend(); } else { shared_->transmit_mixer()->DemuxAndMix(voe_channels, number_of_voe_channels); shared_->transmit_mixer()->EncodeAndSend(voe_channels, number_of_voe_channels); } ...... } // Return 0 to indicate no change on the volume. return 0; } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 编码 //shared_->transmit_mixer()->EncodeAndSend //实现数据的编码，编码后触发打包发送 void TransmitMixer::EncodeAndSend(const int voe_channels[], int number_of_voe_channels) { for (int i = 0; i < number_of_voe_channels; ++i) { voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]); voe::Channel* channel_ptr = ch.channel(); if (channel_ptr && channel_ptr->Sending())//判断当前的channel是否处于发送的状态 channel_ptr->EncodeAndSend(); } } uint32_t Channel::EncodeAndSend(){ ...... //编码压缩音频数据 if (audio_coding_->Add10MsData((AudioFrame&)_audioFrame) < 0) { WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId,_channelId), "Channel::EncodeAndSend() ACM encoding failed"); return 0xFFFFFFFF; } ...... } int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) { InputData input_data; CriticalSectionScoped lock(acm_crit_sect_.get()); //编码之前的处理 ，根据需求重采样 并将数据封装在InputData中 int r = Add10MsDataInternal(audio_frame, &input_data); //开始编码 return r < 0 ? r : Encode(input_data); } int32_t AudioCodingModuleImpl::Encode(const InputData& input_data){ ...... //从CodecManager获取当前正在使用的编码器 AudioEncoder* audio_encoder = codec_manager_.CurrentEncoder(); ...... //开始编码 encode_buffer_.SetSize(audio_encoder->MaxEncodedBytes()); encoded_info = audio_encoder->Encode( rtp_timestamp, input_data.audio, input_data.length_per_channel, encode_buffer_.size(), encode_buffer_.data()); encode_buffer_.SetSize(encoded_info.encoded_bytes); ...... { CriticalSectionScoped lock(callback_crit_sect_.get()); if (packetization_callback_) { //触发发送，packetization_callback_由Channel继承AudioPacketizationCallback实现。 //Channel在Init()时调用，audio_coding_->RegisterTransportCallback(this)完成注册！ packetization_callback_->SendData( frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp, encode_buffer_.data(), encode_buffer_.size(), my_fragmentation.fragmentationVectorSize > 0 ? &my_fragmentation : nullptr); } if (vad_callback_) { // 静音检测回调 vad_callback_->InFrameType(frame_type); } } } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 打包 音频数据在编码之后会通过Channel实现的AudioPacketizationCallback.SendData触发数据打包发送流程。 SendData实现如下： /* android\webrtc\src\webrtc\voice_engine\channel.cc*/ int32_t Channel::SendData(FrameType frameType, uint8_t payloadType, uint32_t timeStamp, const uint8_t* payloadData, size_t payloadSize, const RTPFragmentationHeader* fragmentation){ ...... //RTP打包和发送 if (_rtpRtcpModule->SendOutgoingData((FrameType&)frameType, payloadType, timeStamp, // Leaving the time when this frame was // received from the capture device as // undefined for voice for now. -1, payloadData, payloadSize, fragmentation) == -1) { _engineStatisticsPtr->SetLastError( VE_RTP_RTCP_MODULE_ERROR, kTraceWarning, "Channel::SendData() failed to send data to RTP/RTCP module"); return -1; } ...... } /* android\webrtc\src\webrtc\modules\rtp_rtcp\source\rtp_rtcp_impl.cc*/ //最终由RTPSender实现RTP打包和发送 int32_t ModuleRtpRtcpImpl::SendOutgoingData( FrameType frame_type, int8_t payload_type, uint32_t time_stamp, int64_t capture_time_ms, const uint8_t* payload_data, size_t payload_size, const RTPFragmentationHeader* fragmentation, const RTPVideoHeader* rtp_video_hdr) { rtcp_sender_.SetLastRtpTime(time_stamp, capture_time_ms); if (rtcp_sender_.TimeToSendRTCPReport(kVideoFrameKey == frame_type)) { rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); } return rtp_sender_.SendOutgoingData( frame_type, payload_type, time_stamp, capture_time_ms, payload_data, payload_size, fragmentation, rtp_video_hdr); } /*android\webrtc\src\webrtc\modules\rtp_rtcp\source\rtp_sender.cc*/ int32_t RTPSender::SendOutgoingData(FrameType frame_type, int8_t payload_type, uint32_t capture_timestamp, int64_t capture_time_ms, const uint8_t* payload_data, size_t payload_size, const RTPFragmentationHeader* fragmentation, const RTPVideoHeader* rtp_hdr) { ...... //确定传输的是音频还是视频 if (CheckPayloadType(payload_type, &video_type) != 0) { LOG(LS_ERROR) SendAudio(frame_type, payload_type, capture_timestamp, payload_data, payload_size, fragmentation); //若为视频 ret_val = video_->SendVideo(video_type, frame_type, payload_type, capture_timestamp, capture_time_ms, payload_data, payload_size, fragmentation, rtp_hdr); } /*android\webrtc\src\webrtc\modules\rtp_rtcp\source\rtp_sender_audio.cc*/ int32_t RTPSenderAudio::SendAudio( const FrameType frameType, const int8_t payloadType, const uint32_t captureTimeStamp, const uint8_t* payloadData, const size_t dataSize, const RTPFragmentationHeader* fragmentation) { ...... //根据协议打包编码后的音频数据，整个流程较复杂这里不做分析，可以参考源代码做深入的了解 ...... //发送 return _rtpSender->SendToNetwork(dataBuffer, payloadSize, rtpHeaderLength, -1, kAllowRetransmission, RtpPacketSender::kHighPriority); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 发送 上面流程可以了解到，RTP打包完成之后由RTPSender完成发送流程，如下： int32_t RTPSender::SendToNetwork(uint8_t* buffer, size_t payload_length, size_t rtp_header_length, int64_t capture_time_ms, StorageType storage, RtpPacketSender::Priority priority){ ...... //进行一些时间上的处理和重发机制处理后直接发送数据 bool sent = SendPacketToNetwork(buffer, length); ..... //更新统计状态 UpdateRtpStats(buffer, length, rtp_header, false, false); ...... } bool RTPSender::SendPacketToNetwork(const uint8_t *packet, size_t size) { int bytes_sent = -1; if (transport_) { bytes_sent = //此处的transport_实际为Channel，Channel继承自Transport /* 在Channel构造函数中 Channel::Channel(int32_t channelId, uint32_t instanceId, RtcEventLog* const event_log, const Config& config){ RtpRtcp::Configuration configuration; configuration.audio = true; configuration.outgoing_transport = this; //设置Transport configuration.audio_messages = this; configuration.receive_statistics = rtp_receive_statistics_.get(); configuration.bandwidth_callback = rtcp_observer_.get(); _rtpRtcpModule.reset(RtpRtcp::CreateRtpRtcp(configuration)); } //在ModuleRtpRtcpImpl构造方法中会将参数传入RTPSender ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) : rtp_sender_(configuration.audio, configuration.clock, configuration.outgoing_transport, configuration.audio_messages, configuration.paced_sender, configuration.transport_sequence_number_allocator, configuration.transport_feedback_callback, configuration.send_bitrate_observer, configuration.send_frame_count_observer, configuration.send_side_delay_observer), rtcp_sender_(configuration.audio, configuration.clock, configuration.receive_statistics, configuration.rtcp_packet_type_counter_observer), rtcp_receiver_(configuration.clock, configuration.receiver_only, configuration.rtcp_packet_type_counter_observer, configuration.bandwidth_callback, configuration.intra_frame_callback, configuration.transport_feedback_callback, this)......) */ transport_->SendRtp(packet, size) ? static_cast(size) : -1; } ...... return true; } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 通过上面的分析发现最终的发送流程在Channel中由SendRtp实现： bool Channel::SendRtp(const uint8_t *data, size_t len){ ...... //此处的 _transportPtr 由int32_t Channel::RegisterExternalTransport(Transport& transport)注册完成 //联系之前分析的创建Channel的流程可以发现，在webrtcvoiceengine.cc中 // WebRtcVoiceMediaChannel构造函数中调用了ConfigureSendChannel(voe_channel()) /* void WebRtcVoiceMediaChannel::ConfigureSendChannel(int channel) { //在VoENetworkImpl中通过ChannelOwner获取Channel注册Transport if (engine()->voe()->network()->RegisterExternalTransport( channel, *this) == -1) { LOG_RTCERR2(RegisterExternalTransport, channel, this); } // Enable RTCP (for quality stats and feedback messages) EnableRtcp(channel); // Reset all recv codecs; they will be enabled via SetRecvCodecs. ResetRecvCodecs(channel); // Set RTP header extension for the new channel. SetChannelSendRtpHeaderExtensions(channel, send_extensions_); } */ if (!_transportPtr->SendRtp(bufferToSendPtr, bufferLength)) { std::string transport_name = _externalTransport ? "external transport" : "WebRtc sockets"; WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId,_channelId), "Channel::SendPacket() RTP transmission using %s failed", transport_name.c_str()); return false; } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 通过上面的分析可以发现，Channel中注册的Transport实际上是WebRtcVoiceMediaChannel /*android\webrtc\src\talk\media\webrtc\webrtcvoiceengine.h*/ class WebRtcVoiceMediaChannel : public VoiceMediaChannel, public webrtc::Transport { ...... // implements Transport interface bool SendRtp(const uint8_t* data, size_t len) override { rtc::Buffer packet(reinterpret_cast(data), len, kMaxRtpPacketLen); return VoiceMediaChannel::SendPacket(&packet); } ...... } /*android\webrtc\src\talk\media\base\mediachannel.h*/ class VoiceMediaChannel : public MediaChannel { ...... // Base method to send packet using NetworkInterface. bool SendPacket(rtc::Buffer* packet) { return DoSendPacket(packet, false); } bool SendRtcp(rtc::Buffer* packet) { return DoSendPacket(packet, true); } // Sets the abstract interface class for sending RTP/RTCP data. virtual void SetInterface(NetworkInterface *iface) { rtc::CritScope cs(&network_interface_crit_); network_interface_ = iface; } private: bool DoSendPacket(rtc::Buffer* packet, bool rtcp) { rtc::CritScope cs(&network_interface_crit_); if (!network_interface_) return false; //network_interface_通过SetInterface设置， //是由android\webrtc\src\talk\session\media\channel.h实现 在BaseChannel::Init()调用SetInterface完成注册 return (!rtcp) ? network_interface_->SendPacket(packet) : network_interface_->SendRtcp(packet); } ...... } /*android\webrtc\src\talk\media\base\channel.h*/ class BaseChannel : public rtc::MessageHandler, public sigslot::has_slots<>, public MediaChannel::NetworkInterface, public ConnectionStatsGetter { } /*android\webrtc\src\talk\media\base\channel.cc*/ bool BaseChannel::Init() { ...... //为BaseChannel设置TransportChannel if (!SetTransport(content_name())) { return false; } // Both RTP and RTCP channels are set, we can call SetInterface on // media channel and it can set network options. media_channel_->SetInterface(this); return true; } bool BaseChannel::SendPacket(rtc::Buffer* packet, rtc::DiffServCodePoint dscp) { return SendPacket(false, packet, dscp); } bool BaseChannel::SendPacket(bool rtcp, rtc::Buffer* packet, rtc::DiffServCodePoint dscp){ ...... // 获取传输数据的TransportChannel，Init()通过调用SetTransport设置 TransportChannel* channel = (!rtcp || rtcp_mux_filter_.IsActive()) ? transport_channel_ : rtcp_transport_channel_; if (!channel || !channel->writable()) { return false; } ...... // int ret = channel->SendPacket(packet->data(), packet->size(), options, (secure() && secure_dtls()) ? PF_SRTP_BYPASS : 0); } bool BaseChannel::SetTransport(const std::string& transport_name) { return worker_thread_->Invoke( Bind(&BaseChannel::SetTransport_w, this, transport_name));//实际上就是在SetTransport_w线程中调用SetTransport_w } bool BaseChannel::SetTransport_w(const std::string& transport_name) { ...... //先通过TransportController创建相应的 //TransportChannel(TransportChannelImpl继承TransportChannel,P2PTransportChannel继承TransportChannelImpl,最终由P2PTransportChannel实现) set_transport_channel(transport_controller_->CreateTransportChannel_w( transport_name, cricket::ICE_CANDIDATE_COMPONENT_RTP)); if (!transport_channel()) { return false; } ...... } void BaseChannel::set_transport_channel(TransportChannel* new_tc) { TransportChannel* old_tc = transport_channel_; if (old_tc) {//先注销old_tc的事件监听 DisconnectFromTransportChannel(old_tc); //销毁掉没用的Channel节约系统资源 transport_controller_->DestroyTransportChannel_w( transport_name_, cricket::ICE_CANDIDATE_COMPONENT_RTP); } transport_channel_ = new_tc; if (new_tc) {//设置监听事件 ConnectToTransportChannel(new_tc); for (const auto& pair : socket_options_) { new_tc->SetOption(pair.first, pair.second); } } //告知响应的MediaChannel，TransportChannel已经设置完毕 SetReadyToSend(false, new_tc && new_tc->writable()); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 P2PTransportChannel的SendPacket设计到libjingle p2p的实现，这里做过多的分析。 从以上分析结合图一，就能较好理解webRTC整个音频框架！ 语音接收播放流程 接收 如图一的黄色箭头所示，网络数据从libjingle传入BaseChannel。 //在VoiceChannel::Init()中调用BaseChannel::Init() //--->BaseChannel::Init() //--->bool BaseChannel::SetTransport(const std::string& transport_name) //--->bool BaseChannel::SetTransport_w(const std::string& transport_name) //--->void BaseChannel::set_transport_channel(TransportChannel* new_tc) //--->void BaseChannel::ConnectToTransportChannel(TransportChannel* tc) 1 2 3 4 5 6 7 /* 在TransportChannel类中，每接受一个数据包都会触发SignalReadPacket信号 通过信号与曹实现类间的通信 */ void BaseChannel::ConnectToTransportChannel(TransportChannel* tc) { ASSERT(worker_thread_ == rtc::Thread::Current()); tc->SignalWritableState.connect(this, &BaseChannel::OnWritableState); //libjingle每收到一个数据包都会触发BaseChannel::OnChannelRead tc->SignalReadPacket.connect(this, &BaseChannel::OnChannelRead); tc->SignalReadyToSend.connect(this, &BaseChannel::OnReadyToSend); } void BaseChannel::OnChannelRead(TransportChannel* channel, const char* data, size_t len, const rtc::PacketTime& packet_time, int flags) { // OnChannelRead gets called from P2PSocket; now pass data to MediaEngine ASSERT(worker_thread_ == rtc::Thread::Current()); // When using RTCP multiplexing we might get RTCP packets on the RTP // transport. We feed RTP traffic into the demuxer to determine if it is RTCP. bool rtcp = PacketIsRtcp(channel, data, len); rtc::Buffer packet(data, len); HandlePacket(rtcp, &packet, packet_time); } void BaseChannel::HandlePacket(bool rtcp, rtc::Buffer* packet, const rtc::PacketTime& packet_time){ ...... if (!rtcp) { //rtp packet media_channel_->OnPacketReceived(packet, packet_time); } else { // rtcp packet 很显然这里的media_channel_是WebRtcVoiceMediaChannel media_channel_->OnRtcpReceived(packet, packet_time); } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 解包 /*android\webrtc\src\talk\media\webrtc\webrtcvoiceengine.cc*/ void WebRtcVoiceMediaChannel::OnPacketReceived( rtc::Buffer* packet, const rtc::PacketTime& packet_time) { RTC_DCHECK(thread_checker_.CalledOnValidThread()); // Forward packet to Call as well. const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp, packet_time.not_before); //通过PacketReceiver::DeliveryStatus Call::DeliverPacket //--->PacketReceiver::DeliveryStatus Call::DeliverRtp //--->若为音频则调用bool AudioReceiveStream::DeliverRtp 估算延时，估算远程端的比特率,并更新相关状体 //若为视频则调用 bool VideoReceiveStream::DeliverRtp call_->Receiver()->DeliverPacket(webrtc::MediaType::AUDIO, reinterpret_cast(packet->data()), packet->size(), webrtc_packet_time); // Pick which channel to send this packet to. If this packet doesn't match // any multiplexed streams, just send it to the default channel. Otherwise, // send it to the specific decoder instance for that stream. int which_channel = GetReceiveChannelNum(ParseSsrc(packet->data(), packet->size(), false)); if (which_channel == -1) { which_channel = voe_channel(); } // Pass it off to the decoder. //开始解包 解码 engine()->voe()->network()->ReceivedRTPPacket( which_channel, packet->data(), packet->size(), webrtc::PacketTime(packet_time.timestamp, packet_time.not_before)); } /*android\webrtc\src\webrtc\audio\audio_receive_stream.cc*/ bool AudioReceiveStream::DeliverRtp(const uint8_t* packet, size_t length, const PacketTime& packet_time) { ...... //解析包头 if (!rtp_header_parser_->Parse(packet, length, &header)) { return false; } ...... //估算延时和比特率 remote_bitrate_estimator_->IncomingPacket(arrival_time_ms, payload_size, header, false); } /*android\webrtc\src\webrtc\voice_engine\voe_network_impl.cc*/ int VoENetworkImpl::ReceivedRTPPacket(int channel, const void* data, size_t length, const PacketTime& packet_time){ ...... //联系前面的解析，这里的channelPtr实际上就是android\webrtc\src\webrtc\voice_engine\Channel.cc中的Channel return channelPtr->ReceivedRTPPacket((const int8_t*)data, length, packet_time); } /*android\webrtc\src\webrtc\voice_engine\Channel.cc*/ int32_t Channel::ReceivedRTPPacket(const int8_t* data, size_t length, const PacketTime& packet_time){ ...... const uint8_t* received_packet = reinterpret_cast(data); RTPHeader header; //解析包头 if (!rtp_header_parser_->Parse(received_packet, length, &header)) { WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVoice, _channelId, "Incoming packet: invalid RTP header"); return -1; } ...... //开始解包和解码操作 return ReceivePacket(received_packet, length, header, in_order) ? 0 : -1; } bool Channel::ReceivePacket(const uint8_t* packet, size_t packet_length, const RTPHeader& header, bool in_order){ ...... const uint8_t* payload = packet + header.headerLength; ...... //将有效数据给解码器解码 return rtp_receiver_->IncomingRtpPacket(header, payload, payload_length, payload_specific, in_order); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 解码 /*android\webrtc\src\webrtc\modules\rtp_rtcp\source\rtp_receiver_impl.cc*/ bool RtpReceiverImpl::IncomingRtpPacket( const RTPHeader& rtp_header, const uint8_t* payload, size_t payload_length, PayloadUnion payload_specific, bool in_order){ // Trigger our callbacks. CheckSSRCChanged(rtp_header); ...... //通过回调将数据送给解码器 int32_t ret_val = rtp_media_receiver_->ParseRtpPacket( &webrtc_rtp_header, payload_specific, is_red, payload, payload_length, clock_->TimeInMilliseconds(), is_first_packet_in_frame); } /*android\webrtc\src\webrtc\modules\rtp_rtcp\source\rtp_receiver_audio.cc*/ int32_t RTPReceiverAudio::ParseRtpPacket(WebRtcRTPHeader* rtp_header, const PayloadUnion& specific_payload, bool is_red, const uint8_t* payload, size_t payload_length, int64_t timestamp_ms, bool is_first_packet) { ...... return ParseAudioCodecSpecific(rtp_header, payload, payload_length, specific_payload.Audio, is_red); } int32_t RTPReceiverAudio::ParseAudioCodecSpecific( WebRtcRTPHeader* rtp_header, const uint8_t* payload_data, size_t payload_length, const AudioPayload& audio_specific, bool is_red) { //处理DTMF相关 bool telephone_event_packet = TelephoneEventPayloadType(rtp_header->header.payloadType); if (telephone_event_packet) { CriticalSectionScoped lock(crit_sect_.get()); // RFC 4733 2.3 // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // | event |E|R| volume | duration | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // if (payload_length % 4 != 0) { return -1; } size_t number_of_events = payload_length / 4; // sanity if (number_of_events >= MAX_NUMBER_OF_PARALLEL_TELEPHONE_EVENTS) { number_of_events = MAX_NUMBER_OF_PARALLEL_TELEPHONE_EVENTS; } for (size_t n = 0; n < number_of_events; ++n) { bool end = (payload_data[(4 * n) + 1] & 0x80) ? true : false; std::set::iterator event = telephone_event_reported_.find(payload_data[4 * n]); if (event != telephone_event_reported_.end()) { // we have already seen this event if (end) { telephone_event_reported_.erase(payload_data[4 * n]); } } else { if (end) { // don't add if it's a end of a tone } else { telephone_event_reported_.insert(payload_data[4 * n]); } } } ...... //向解码器填入数据 // TODO(holmer): Break this out to have RED parsing handled generically. if (is_red && !(payload_data[0] & 0x80)) { // we recive only one frame packed in a RED packet remove the RED wrapper rtp_header->header.payloadType = payload_data[0]; // only one frame in the RED strip the one byte to help NetEq return data_callback_->OnReceivedPayloadData( payload_data + 1, payload_length - 1, rtp_header); } rtp_header->type.Audio.channel = audio_specific.channels; return data_callback_->OnReceivedPayloadData( payload_data, payload_length, rtp_header); //上面的data_callback_为RtpData类型，由Channel实现 } /*android\webrtc\src\webrtc\voice_engine\Channel.cc*/ int32_t Channel::OnReceivedPayloadData(const uint8_t* payloadData, size_t payloadSize, const WebRtcRTPHeader* rtpHeader){ ...... if (audio_coding_->IncomingPacket(payloadData, payloadSize, *rtpHeader) != 0) { _engineStatisticsPtr->SetLastError( VE_AUDIO_CODING_MODULE_ERROR, kTraceWarning, "Channel::OnReceivedPayloadData() unable to push data to the ACM"); return -1; } ...... } /*android\webrtc\src\webrtc\modules\audio_coding\main\acm2\audio_coding_module_impl.cc*/ int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload, const size_t payload_length, const WebRtcRTPHeader& rtp_header) { return receiver_.InsertPacket(rtp_header, incoming_payload, payload_length); } /*\android\webrtc\src\webrtc\modules\audio_coding\main\acm2\acm_receiver.cc*/ int AcmReceiver::InsertPacket(const WebRtcRTPHeader& rtp_header, const uint8_t* incoming_payload, size_t length_payload){ ...... //根据rtp头从DecoderDatabase管理的解码器中，选择合适的解码器 const Decoder* decoder = RtpHeaderToDecoder(*header, incoming_payload); .....//同步相关处理 ...... //android\webrtc\src\webrtc\modules\audio_coding\neteq\NetEqImpl.CC //NetEq技术是GIPS的核心音频处理技术，后背谷歌收购。详细了解可参考NetEq解析 if (neteq_->InsertPacket(rtp_header, incoming_payload, length_payload, receive_timestamp) < 0) { LOG(LERROR) (header->payloadType) header.markerBit = false; packet->header.payloadType = rtp_header.header.payloadType; packet->header.sequenceNumber = rtp_header.header.sequenceNumber; packet->header.timestamp = rtp_header.header.timestamp; packet->header.ssrc = rtp_header.header.ssrc; packet->header.numCSRCs = 0; packet->payload_length = length_bytes; packet->primary = true; packet->waiting_time = 0; packet->payload = new uint8_t[packet->payload_length]; packet->sync_packet = is_sync_packet; if (!packet->payload) { LOG_F(LS_ERROR) payload, payload, packet->payload_length); // Insert packet in a packet list. packet_list.push_back(packet); // Save main payloads header for later. memcpy(&main_header, &packet->header, sizeof(main_header)); } //处理DTMF相关事件，将事件放入DtmfEvent队列中 PacketList::iterator it = packet_list.begin(); while (it != packet_list.end()) { Packet* current_packet = (*it); assert(current_packet); assert(current_packet->payload); if (decoder_database_->IsDtmf(current_packet->header.payloadType)) { assert(!current_packet->sync_packet); // We had a sanity check for this. DtmfEvent event; int ret = DtmfBuffer::ParseEvent( current_packet->header.timestamp, current_packet->payload, current_packet->payload_length, &event); if (ret != DtmfBuffer::kOK) { PacketBuffer::DeleteAllPackets(&packet_list); return kDtmfParsingError; } if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) { PacketBuffer::DeleteAllPackets(&packet_list); return kDtmfInsertError; } // TODO(hlundin): Let the destructor of Packet handle the payload. delete [] current_packet->payload; delete current_packet; it = packet_list.erase(it); } else { ++it; } } ...... // Update bandwidth estimate, if the packet is not sync-packet. if (!packet_list.empty() && !packet_list.front()->sync_packet) { // The list can be empty here if we got nothing but DTMF payloads. AudioDecoder* decoder = decoder_database_->GetDecoder(main_header.payloadType); assert(decoder); // Should always get a valid object, since we have // already checked that the payload types are known. //在最终的decoder中好像都没有实现 decoder->IncomingPacket(packet_list.front()->payload, packet_list.front()->payload_length, packet_list.front()->header.sequenceNumber, packet_list.front()->header.timestamp, receive_timestamp); // 需要解码的数据放入PacketBuffer 列表中 const size_t buffer_length_before_insert = packet_buffer_->NumPacketsInBuffer(); ret = packet_buffer_->InsertPacketList( &packet_list, *decoder_database_, ¤t_rtp_payload_type_, ¤t_cng_rtp_payload_type_); if (ret == PacketBuffer::kFlushed) { // Reset DSP timestamp etc. if packet buffer flushed. new_codec_ = true; update_sample_rate_and_channels = true; } else if (ret != PacketBuffer::kOK) { PacketBuffer::DeleteAllPackets(&packet_list); return kOtherError; } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 通过int NetEqImpl::GetAudio获取pcm数据。 int NetEqImpl::GetAudio(size_t max_length, int16_t* output_audio, size_t* samples_per_channel, int* num_channels, NetEqOutputType* type){ ...... int error = GetAudioInternal(max_length, output_audio, samples_per_channel, num_channels) ...... } int NetEqImpl::GetAudioInternal(size_t max_length, int16_t* output, size_t* samples_per_channel, int* num_channels){ ...... //解码 int decode_return_value = Decode(&packet_list, &operation, &length, &speech_type); ...... } int NetEqImpl::Decode(PacketList* packet_list, Operations* operation, int* decoded_length, AudioDecoder::SpeechType* speech_type){ ...... //获得当前解码器 AudioDecoder* decoder = decoder_database_->GetActiveDecoder(); ...... //开始解码 if (*operation == kCodecInternalCng) { RTC_DCHECK(packet_list->empty()); return_value = DecodeCng(decoder, decoded_length, speech_type); } else { return_value = DecodeLoop(packet_list, *operation, decoder, decoded_length, speech_type); } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 最终通过GetAudio获取的就是pcm数据！ 播放 org/webrtc/voiceengine/WebRtcAudioTrack.java 通过AudioTrackThread播放线程不断从native获取pcm数据，并将pcm数据送入audiotrack中播放。 nativeGetPlayoutData(WebRtcAudioTrack.java)--> void JNICALL AudioTrackJni::GetPlayoutData(audio_track_jni.cc)--> void AudioTrackJni::OnGetPlayoutData(size_t length)((audio_track_jni.cc)) void AudioTrackJni::OnGetPlayoutData(size_t length) { ...... // Pull decoded data (in 16-bit PCM format) from jitter buffer. //获取数据 int samples = audio_device_buffer_->RequestPlayoutData(frames_per_buffer_); if (samples (samples), frames_per_buffer_); // Copy decoded data into common byte buffer to ensure that it can be // written to the Java based audio track. //拷贝到共享内存 samples = audio_device_buffer_->GetPlayoutData(direct_buffer_address_); ...... } int32_t AudioDeviceBuffer::RequestPlayoutData(size_t nSamples){ ...... /**/ if (_ptrCbAudioTransport) { uint32_t res(0); int64_t elapsed_time_ms = -1; int64_t ntp_time_ms = -1; res = _ptrCbAudioTransport->NeedMorePlayData(_playSamples, playBytesPerSample, playChannels, playSampleRate, &_playBuffer[0], nSamplesOut, &elapsed_time_ms, &ntp_time_ms); if (res != 0) { WEBRTC_TRACE(kTraceError, kTraceAudioDevice, _id, "NeedMorePlayData() failed"); } } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 AudioTransport由VoEBaseImpl实现，具体的注册过程可以参考上面的解析！ /*android\webrtc\src\webrtc\voice_engine\voe_base_impl.cc*/ int32_t VoEBaseImpl::NeedMorePlayData(size_t nSamples, size_t nBytesPerSample, uint8_t nChannels, uint32_t samplesPerSec, void* audioSamples, size_t& nSamplesOut, int64_t* elapsed_time_ms, int64_t* ntp_time_ms) { GetPlayoutData(static_cast(samplesPerSec), static_cast(nChannels), nSamples, true, audioSamples, elapsed_time_ms, ntp_time_ms); nSamplesOut = audioFrame_.samples_per_channel_; return 0; } void VoEBaseImpl::GetPlayoutData(int sample_rate, int number_of_channels, size_t number_of_frames, bool feed_data_to_apm, void* audio_data, int64_t* elapsed_time_ms, int64_t* ntp_time_ms){ //获取数据 shared_->output_mixer()->MixActiveChannels(); //混音和重采样处理 // Additional operations on the combined signal shared_->output_mixer()->DoOperationsOnCombinedSignal(feed_data_to_apm); // Retrieve the final output mix (resampled to match the ADM) shared_->output_mixer()->GetMixedAudio(sample_rate, number_of_channels, &audioFrame_); //拷贝pcm数据 memcpy(audio_data, audioFrame_.data_, sizeof(int16_t) * number_of_frames * number_of_channels); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 shared_->output_mixer()->MixActiveChannels() 通过channel从解码器中获取pcm数据 /*android\webrtc\src\webrtc\voice_engine\output_mixer.cc*/ int32_t OutputMixer::MixActiveChannels() { return _mixerModule.Process(); } /*android\webrtc\src\webrtc\modules\audio_conference_mixer\source\audio_conference_mixer_impl.cc*/ int32_t AudioConferenceMixerImpl::Process() { ...... UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap, &remainingParticipantsAllowedToMix); ...... } void AudioConferenceMixerImpl::UpdateToMix( AudioFrameList* mixList, AudioFrameList* rampOutList, std::map* mixParticipantList, size_t* maxAudioFrameCounter){ ...... for (MixerParticipantList::const_iterator participant = _participantList.begin(); participant != _participantList.end(); ++participant) { ...... //从MixerParticipan获取pcm数据，而MixerParticipant由Channel实现 // if((*participant)->GetAudioFrame(_id, audioFrame) != 0) { WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, "failed to GetAudioFrame() from participant"); _audioFramePool->PushMemory(audioFrame); continue; ...... } } ...... } /*android\webrtc\src\webrtc\voice_engine\channel.cc*/ int32_t Channel::GetAudioFrame(int32_t id, AudioFrame* audioFrame){ ...... //从AudioCodingModule获取解码的pcm数据 if (audio_coding_->PlayoutData10Ms(audioFrame->sample_rate_hz_, audioFrame) == -1) { WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId,_channelId), "Channel::GetAudioFrame() PlayoutData10Ms() failed!"); // In all likelihood, the audio in this frame is garbage. We return an // error so that the audio mixer module doesn't add it to the mix. As // a result, it won't be played out and the actions skipped here are // irrelevant. return -1; } ...... } /*android\webrtc\src\webrtc\modules\audio_coding\main\acm2\audio_coding_module_impl.cc*/ int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz, AudioFrame* audio_frame) { // GetAudio always returns 10 ms, at the requested sample rate. if (receiver_.GetAudio(desired_freq_hz, audio_frame) != 0) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, "PlayoutData failed, RecOut Failed"); return -1; } audio_frame->id_ = id_; return 0; } /*android\webrtc\src\webrtc\modules\audio_coding\main\acm2\acm_receiver.cc*/ int AcmReceiver::GetAudio(int desired_freq_hz, AudioFrame* audio_frame){ ...... // 结合之前的解码分析，可知，这里是从缓冲区中获取的压缩音频数据，然后通过decoder解码后送出！ if (neteq_->GetAudio(AudioFrame::kMaxDataSizeSamples, audio_buffer_.get(), &samples_per_channel, &num_channels, &type) != NetEq::kOK) { LOG(LERROR) << "AcmReceiver::GetAudio - NetEq Failed."; return -1; } ...... } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 从上面的分析可以看出，webrtc整个层次结构非常清晰。结合图一，再结合相关代码很容易了解整个框架！ --------------------- 本文来自 Jalon007 的CSDN 博客 ，全文地址请点击：https://blog.csdn.net/u010657219/article/details/54931154?utm_source=copy