WebRTC本地音频回调、选用音频采集设备及自定义输入音频
由于工作需要，开始研究WebRTC源码，现将如何把本地音频回调出来分享一下。
如果要使用Native WebRTC封装SDK，就要把RTC的一些基础能力暴露出来，本地视频、远端音视频都可以在相应的track里添加Sink就能拿到，但本地音频回调就没那么容易了，需要修改RTC的源码才可以。
首先来看一下AudioDeviceModule这个音频设备模块，它把录音和播放封装在一块，其中与录音相关的API有
//获取音频采集设备个数
virtual int16_t RecordingDevices() = 0;
//跟据设备索引获取设备名称和GUID(有的设备名称可能是一样，但GUID唯一)
virtual int32_t RecordingDeviceName(uint16_t index,
char name[kAdmMaxDeviceNameSize],
char guid[kAdmMaxGuidSize]) = 0;
//更换音频采集设备
virtual int32_t SetRecordingDevice(uint16_t index) = 0;
//初始化采集设备
virtual int32_t InitRecording() = 0;
//开始采集音频
virtual int32_t StartRecording() = 0;
//停止采集音频
virtual int32_t StopRecording() = 0;
这个模块会在创建webrtc::PeerConnectionFactoryInterface会用到
rtc::scoped_refptr
CreatePeerConnectionFactory(
rtc::Thread* network_thread,
rtc::Thread* worker_thread,
rtc::Thread* signaling_thread,
rtc::scoped_refptr default_adm,
rtc::scoped_refptr audio_encoder_factory,
rtc::scoped_refptr audio_decoder_factory,
std::unique_ptr video_encoder_factory,
std::unique_ptr video_decoder_factory,
rtc::scoped_refptr audio_mixer,
rtc::scoped_refptr audio_processing);
如果传入参数default_adm为nullptr，WebRTC会用AudioDeviceModuleImpl创建它，来看一下AudioDeviceModuleImpl的主要成员
//当前平台
PlatformType platform_type_ = kPlatformNotSupported;
//用于处理从audio_device_采集出来的音频，交由AudioTransport做一些合法性检查然后传输出去
AudioDeviceBuffer audio_device_buffer_;
//音频设备类，不同的平台有不同的实现 ，windows上用会AudioDeviceWindowsCore实现
std::unique_ptr audio_device_;
audio_device_采集的音频会交由audio_device_buffer_处理，在由AudioTransport传输，来看一下AudioTransport
class AudioTransport {
public:
virtual int32_t RecordedDataIsAvailable(const void* audioSamples,
const size_t nSamples,
const size_t nBytesPerSample,
const size_t nChannels,
const uint32_t samplesPerSec,
const uint32_t totalDelayMS,
const int32_t clockDrift,
const uint32_t currentMicLevel,
const bool keyPressed,
uint32_t& newMicLevel) = 0; // NOLINT
// Implementation has to setup safe values for all specified out parameters.
virtual int32_t NeedMorePlayData(const size_t nSamples,
const size_t nBytesPerSample,
const size_t nChannels,
const uint32_t samplesPerSec,
void* audioSamples,
size_t& nSamplesOut, // NOLINT
int64_t* elapsed_time_ms,
int64_t* ntp_time_ms) = 0; // NOLINT
// Method to pull mixed render audio data from all active VoE channels.
// The data will not be passed as reference for audio processing internally.
virtual void PullRenderData(int bits_per_sample,
int sample_rate,
size_t number_of_channels,
size_t number_of_frames,
void* audio_data,
int64_t* elapsed_time_ms,
int64_t* ntp_time_ms) = 0;
protected:
virtual ~AudioTransport() {}
};
在WebRTC的默认实现中会用AudioTransportImpl实例化AudioTransport 。
以上分析可以看出如果想把设备控制权从WebRTC里拿出来可以自定义类继承至AudioDeviceModuleImpl，如果要把采集到的音频回调出来可以自定类继承至AudioTransportImpl。那么如何让音频回调是从我们自定义的类里创建的呢？再来看看默认的AudioTransportImpl是如何创建的。
通过源码可以看到在AudioState里有AudioTransportImpl对象，它是类对象，如果想让它从子类中创建必须为类对象指针，所以将它改为AudioTransportImpl*类型，将AudioState的构造改为
AudioState::AudioState(const AudioState::Config& config)
: config_(config),audio_transport_(nullptr) {
process_thread_checker_.Detach();
RTC_DCHECK(config_.audio_mixer);
RTC_DCHECK(config_.audio_device_module);
audio_transport_ = audio_device_module()->CreateAudioTransPort(
config_.audio_mixer, config_.audio_processing.get());
needDelete_ = audio_transport_ == nullptr;
if (needDelete_) {
audio_transport_ = new AudioTransportImpl(config_.audio_mixer,
config_.audio_processing.get());
}
RTC_DCHECK(audio_transport_);
}
上面的代码audio_device_module()是得到adm_对象，CreateAudioTransPort就是我们要增加的纯虚函数所以在AudioDeviceModule类中增加
virtual AudioTransportImpl* CreateAudioTransPort(
AudioMixer* mixer,
AudioProcessing* audio_processing) = 0;
这个在AudioDeviceModuleImpl的实现为返回nullptr，这样如果没有子类实现它，结果就和原来的没有区别。
另外如果想要实现自定义输入，必须把WebRTC的采集停止掉，在AudioDeviceModule增加纯虚函数：
virtual int32_t SendRecordedBuffer(const uint8_t* audio_data,
uint32_t data_len,
int bits_per_sample,
int sample_rate,
size_t number_of_channels) = 0;
在AudioDeviceModuleImpl实现：
int32_t AudioDeviceModuleImpl::SendRecordedBuffer(const uint8_t* audio_data,
uint32_t data_len,
int bits_per_sample,
int sample_rate,
size_t number_of_channels) {
//if in external mode,shoudle control at subclass
audio_device_buffer_.SetRecordingChannels(number_of_channels);
audio_device_buffer_.SetRecordingSampleRate(sample_rate);
audio_device_buffer_.SetRecordedBuffer(audio_data, (data_len >> 1) / number_of_channels);
audio_device_buffer_.SetVQEData(0, 0);
audio_device_buffer_.DeliverRecordedData();
return 0;
}
整个的一个继承代码类似这样
//本地音频回调
class RecordedDataObserver {
public:
virtual void onRecodedData(
const void* audioSamples,
const size_t nSamples,
const size_t nBytesPerSample,
const size_t nChannels,
const uint32_t samplesRate) = 0;
virtual ~RecordedDataObserver() {}
};
namespace base {
using namespace webrtc;
class TaskQueueFactory;
class AudioMixer;
class AudioProcessing;
class CustomizedAudioDeviceModule : public webrtc::AudioDeviceModuleImpl{
public:
static rtc::scoped_refptr CreateADM();
CustomizedAudioDeviceModule(AudioLayer audio_layer,
webrtc::TaskQueueFactory* task_queue_factory);
virtual ~CustomizedAudioDeviceModule();
class AudioRecordCallBack : public AudioTransportImpl {
public:
AudioRecordCallBack(webrtc::AudioMixer* mixer, webrtc::AudioProcessing* audio_processing);
virtual ~AudioRecordCallBack();
int32_t RecordedDataIsAvailable(const void* audioSamples,
const size_t nSamples,
const size_t nBytesPerSample,
const size_t nChannels,
const uint32_t samplesPerSec,
const uint32_t totalDelayMS,
const int32_t clockDrift,
const uint32_t currentMicLevel,
const bool keyPressed,
uint32_t& newMicLevel) override;
void RegisterObserver(RecordedDataObserver* dataObserver);
private:
RecordedDataObserver* recorderDataOb;
};
AudioTransportImpl* CreateAudioTransPort(
webrtc::AudioMixer* mixer,
webrtc::AudioProcessing* audio_processing) override;
bool IsExternalAudioInput() const override;
void SetExternalAudioMode(bool bExternal);
void RegisterObserver(RecordedDataObserver* dataObserver);
private:
std::unique_ptr audioRecordCb_;
bool externalAudio_;
RecordedDataObserver* dataObserver_;
};
}
实现文件
rtc::scoped_refptr base::CustomizedAudioDeviceModule::CreateADM()
{
// Create the generic reference counted (platform independent) implementation.
rtc::scoped_refptr audioDevice(
new rtc::RefCountedObject(webrtc::AudioDeviceModule::kPlatformDefaultAudio,
&webrtc::GlobalTaskQueueFactory()));
// Ensure that the current platform is supported.
if (audioDevice->CheckPlatform() == -1) {
return nullptr;
}
// Ensure that the generic audio buffer can communicate with the platform
// specific parts.
if (audioDevice->AttachAudioBuffer() == -1) {
return nullptr;
}
return audioDevice;
}
base::CustomizedAudioDeviceModule::~CustomizedAudioDeviceModule() {
}
webrtc::AudioTransportImpl* base::CustomizedAudioDeviceModule::CreateAudioTransPort(
webrtc::AudioMixer* mixer,
webrtc::AudioProcessing* audio_processing) {
audioRecordCb_ = std::make_unique(mixer, audio_processing);
if (dataObserver_) {
audioRecordCb_->RegisterObserver(dataObserver_);
}
return audioRecordCb_.get();
}
bool base::CustomizedAudioDeviceModule::IsExternalAudioInput() const {
return externalAudio_;
}
void base::CustomizedAudioDeviceModule::SetExternalAudioMode(bool bExternal) {
externalAudio_ = bExternal;
}
void base::CustomizedAudioDeviceModule::RegisterObserver(RecordedDataObserver* dataObserver) {
//要在CreatePeerConnectionFactory后调用
RTC_CHECK(dataObserver);
RTC_CHECK(audioRecordCb_);
dataObserver_ = dataObserver;
audioRecordCb_->RegisterObserver(dataObserver);
}
base::CustomizedAudioDeviceModule::AudioRecordCallBack::AudioRecordCallBack(
webrtc::AudioMixer* mixer,
webrtc::AudioProcessing* audio_processing) :
AudioTransportImpl(mixer,audio_processing),
recorderDataOb(nullptr) {
}
base::CustomizedAudioDeviceModule::AudioRecordCallBack::~AudioRecordCallBack() {
}
int32_t base::CustomizedAudioDeviceModule::AudioRecordCallBack::RecordedDataIsAvailable( const void* audioSamples,
const size_t nSamples,
const size_t nBytesPerSample,
const size_t nChannels,
const uint32_t samplesPerSec,
const uint32_t totalDelayMS,
const int32_t clockDrift,
const uint32_t currentMicLevel,
const bool keyPressed,
uint32_t& newMicLevel) {
if (recorderDataOb) {
recorderDataOb->onRecodedData(audioSamples, nSamples, nBytesPerSample, nChannels,samplesPerSec);
}
return AudioTransportImpl::RecordedDataIsAvailable(
audioSamples,
nSamples,
nBytesPerSample,
nChannels,
samplesPerSec,
totalDelayMS,
clockDrift,
currentMicLevel,
keyPressed,
newMicLevel);
}
void base::CustomizedAudioDeviceModule::AudioRecordCallBack::RegisterObserver(RecordedDataObserver* dataObserver) {
recorderDataOb = dataObserver;
}
在webrtc::CreatePeerConnectionFactory的时候把adm对象传进去，需要注意的是，adm对象要由worker_thread创建，原因是AudioDeviceModuleImpl中的audio_device_buffer_要在worker_thread中运行，所以adm的StartRecording()、StopRecording()，也要在worker_thread中运行。
adm对象销毁也要由worker_thread销毁。
换音频采集设置调用adm对象的SetRecordingDevice(uint16_t index)，需要StopRecording,SetRecordingDevice,InitRecording,StartRecording。
自定义输入音频要增加接口类似这种：
int32_t AudioDeviceModuleImpl::SendRecordedBuffer(const uint8_t* audio_data,
uint32_t data_len,
int bits_per_sample,
int sample_rate,
size_t number_of_channels) {
if (!external_audio_input_)
return -1;
audio_device_buffer_.SetRecordingChannels(number_of_channels);
audio_device_buffer_.SetRecordingSampleRate(sample_rate);
audio_device_buffer_.SetRecordedBuffer(audio_data, (data_len >> 1) / number_of_channels);
audio_device_buffer_.SetVQEData(0, 0);
audio_device_buffer_.DeliverRecordedData();
return 0;
}
本人刚开始学习WebRTC，难免有不当之处，欢迎指正。
如果有更好的实现，欢迎分享。