Andoird使用AudioTrack以及OpenSLES渲染音频
最近音视频开发学习到了第四章，这一章讲的是移动平台的音视频渲染，对于AudioTrack以及OpenSLES相当于又复习了一遍。这一章结合之前的ffmpeg解码来进行mp3文件的播放。主要的难点有linux多线程的同步，生产者消费者模型等等。如果对ffmpeg解码不熟悉的可以看https://blog.csdn.net/a568478312/article/details/80268498，这篇文章的重点在于api的使用以及多线程的控制。
使用AudioTrack播放音频
AudioTrack的API其实用起来比较方便，一般需要结合解码器来使用。首先我们需要知道pcm数据的采样率，比特率，声道数等信息，用来之后初始化AudioTrack。和之前的解码一样，这里同样使用ffmpeg。
private boolean initMetaData(String path) {
int[] metaArray = new int[]{0, 0, 0};
decoder.getMusicMetaByPath(path, metaArray);
this.sampleRateInHz = metaArray[0];
this.bitRate = metaArray[1];
this.channel=metaArray[2];
if (sampleRateInHz <= 0 || bitRate <= 0) {
return false;
}
totalCapacity = (new File(path)).length();
mp3CapacityPerSec = bitRate / BITS_PER_BYTE;
if (mp3CapacityPerSec == 0) {
return false;
}
duration = (int) (totalCapacity / mp3CapacityPerSec);
Log.d("tedu", " bitrate duration: " + duration);
int byteCountPerSec = sampleRateInHz * CHANNEL_PER_FRAME * BITS_PER_CHANNEL / BITS_PER_BYTE;
//比特率不变 采样率是被压缩的
Log.d("tedu", "initMetaData: bitRate " + bitRate + "mp3CapacityPerSec " + mp3CapacityPerSec +
"byteCountPerSec " + byteCountPerSec);
DECODE_BUFFER_SIZE = (int) ((byteCountPerSec / 2) * 0.2);
initPlayState();
seekBaseMillsTime = 0;
audioTrackBaseHeadPosition = 0;
return true;
}接着初始化解码的核心功能，并且计算出来之后每一个解码buffer的大小，接着初始化队列以及开启线程。PacketPool维护一个链表实现的队列，用来保存解码的裸数据。
void AccompanyDecoderController::init(const char* accompanyPath,
float packetBufferTimePercent) {
//初始化两个全局变量
volume = 1.0f;
accompanyMax = 1.0f;
//计算计算出伴奏和原唱的bufferSize
int accompanyByteCountPerSec = accompanySampleRate * CHANNEL_PER_FRAME
* BITS_PER_CHANNEL / BITS_PER_BYTE;
accompanyPacketBufferSize = (int) ((accompanyByteCountPerSec / 2)
* packetBufferTimePercent);
//初始化两个decoder
initAccompanyDecoder(accompanyPath);
//初始化队列以及开启线程
packetPool = PacketPool::GetInstance();
packetPool->initDecoderAccompanyPacketQueue();
initDecoderThread();
}下面是开启解码线程。
void AccompanyDecoderController::initDecoderThread() {
isRunning = true;
pthread_mutex_init(&mLock, NULL);
pthread_cond_init(&mCondition, NULL);
pthread_create(&songDecoderThread, NULL, startDecoderThread, this);
}
解码线程中不断去文件中解码数据，并且将解码数据放入队列中，如果队列的数量超过阈值，释放锁并进行等待。
void* AccompanyDecoderController::startDecoderThread(void* ptr) {
LOGI("enter AccompanyDecoderController::startDecoderThread");
AccompanyDecoderController* decoderController =
(AccompanyDecoderController *) ptr;
int getLockCode = pthread_mutex_lock(&decoderController->mLock);
while (decoderController->isRunning) {
decoderController->decodeSongPacket();
if (decoderController->packetPool->geDecoderAccompanyPacketQueueSize() > QUEUE_SIZE_MAX_THRESHOLD) {
pthread_cond_wait(&decoderController->mCondition,
&decoderController->mLock);
}
}
pthread_mutex_unlock(&decoderController->mLock);
}接着我们初始化AudioTrack,注意这里的最好是使用它提供的方法来进行计算。然后使用我们获取的采样率，声道数之类的将AudioTrack初始化。这里需要注意AudioTrack的两种播放模式，一种是流的模式一种静态模式。
- AudioTrack.MODE_STREAM 在播放的文件过大的时候使用，避免内存问题。
- AudioTrack.MODE_STATIC 一次性写入所有数据，一般播放小文件。
private void initAudioTrack() {
int bufferSize = AudioTrack.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat);
audioTrack = new AudioTrack(this.audioStreamType, sampleRateInHz, channelConfig, audioFormat, bufferSize,
AudioTrack.MODE_STREAM);
}接下来我开启java线程来不断从native的队列里读取数据。
class PlayerThread implements Runnable {
private short[] samples;
@Override
public void run() {
int sample_count = 0;
boolean isPlayTemp = isPlaying = false;
try {
//?0.2S的数据
samples = new short[DECODE_BUFFER_SIZE];
short[] fakeSamples = new short[DECODE_BUFFER_SIZE];
int[] extraSlientSampleSize = new int[1];
// int sampleDurationMs = (int)((float)DECODE_BUFFER_SIZE /
// sampleRateInHz / CHANNEL_PER_FRAME * 1000);
while (!isStop) {
// long start = System.currentTimeMillis();
extraSlientSampleSize[0] = 0;
sample_count = decoder.readSamples(samples, extraSlientSampleSize);
// Log.i(TAG, "end Native Mp3 playing......
// DECODE_BUFFER_SIZE="+DECODE_BUFFER_SIZE+"
// sample_count="+sample_count);
if (sample_count == -2) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace();
}
// Log.i(TAG, "WARN : no play data");
continue;
}
if (sample_count < 0) {
break;
}
while (true) {
synchronized (NativeMp3Player.class) {
isPlayTemp = isPlaying;
}
if (isPlayTemp)
break;
else
Thread.yield();
}
if (null != audioTrack && audioTrack.getState() != AudioTrack.STATE_UNINITIALIZED) {
audioTrack.setStereoVolume(1, 1);
if (extraSlientSampleSize[0] > 0) {
int totalSize = extraSlientSampleSize[0] + sample_count;
short[] playSamples = new short[totalSize];
System.arraycopy(samples, 0, playSamples, extraSlientSampleSize[0], sample_count);
audioTrack.write(playSamples, 0, totalSize);
} else {
audioTrack.write(samples, 0, sample_count);
}
}
}
decoder.destory();
} catch (Error e) {
e.printStackTrace();
}
samples = null;
// Log.i(TAG, "end Native Mp3 player thread...");
}
}读取队列里的解码数据，并且在数据数量小于阈值时，通知解码线程进行解码。
int AccompanyDecoderController::readSamples(short* samples, int size,
int* slientSizeArr) {
int result = -1;
AudioPacket* accompanyPacket = NULL;
packetPool->getDecoderAccompanyPacket(&accompanyPacket, true);
if (NULL != accompanyPacket) {
int samplePacketSize = accompanyPacket->size;
if (samplePacketSize != -1 && samplePacketSize <= size) {
//copy the raw data to samples
memcpy(samples, accompanyPacket->buffer, samplePacketSize * 2);
adjustSamplesVolume(samples, samplePacketSize,
volume / accompanyMax);
delete accompanyPacket;
result = samplePacketSize;
}
} else {
result = -2;
}
if (packetPool->geDecoderAccompanyPacketQueueSize() < QUEUE_SIZE_MIN_THRESHOLD) {
int getLockCode = pthread_mutex_lock(&mLock);
if (result != -1) {
pthread_cond_signal(&mCondition);
}
pthread_mutex_unlock (&mLock);
}
return result;
}最后再简单梳理一下流程：
- 使用ffmepge解码mp3文件的采样率，比特率等数据。
- 生产者线程解码pcm数据并放入队列中
- 消费者线程从队列中获取数据并写入audioTrack中进行播放。
源码整个看下来只要熟悉使用linux下的多线程的pthread_mutex_t mLock，pthread_cond_t mCondition来进行线程间的同步，没有太大难点。
使用OpenSLES来播放音频
OpenSL ES是可以让你使用C或者C++来实现高性能，低延迟音频API。ES代表Embedded Systems，在嵌入式系统中单独设计。使用OpenSL ES可以省去native层到java层的数据拷贝的开销，更加高效，然而API设计的并不是特别简单，总感觉略坑。不过使用下来大部分还是模板代码，稍微熟悉一下还是没什么大问题的。要使用OpenSLES我们需要先在CMakeLists.txt中引入库。
add_library( libmedia
STATIC
opensl_es_context.cpp
sound_service.cpp
)
target_link_libraries(
libmedia
#加入OpenSLES
OpenSLES
)接着引入头文件
#include 然后我们就可以初始化OpenSL ES的环境了。首先了解一下，这套API是使用C语言来实现面向对象的设计，通用的使用流程是。
- 初始化SLObjectItf对象
- Realize初始化的对象
- 通过对象获取相关接口来使用
了解了之后我们来正式看代码。
1、创建播放引擎engineObject 这是一个SLObjectItf的对象。
SLresult createEngine() {
// OpenSL ES for Android is designed to be thread-safe,
// so this option request will be ignored, but it will
// make the source code portable to other platforms.
SLEngineOption engineOption[] = {{(SLuint32) SL_ENGINEOPTION_THREADSAFE, (SLuint32) SL_BOOLEAN_TRUE}};
// Create the OpenSL ES engine object
return slCreateEngine(&engineObject, ARRAY_LEN(engineOption), engineOption, 0,
0, 0
);
}
接着我们Realize初始化的引擎对象
void OpenSLESContext::init() {
LOGI("createEngine");
SLresult result = createEngine();
LOGI("createEngine result is s%", ResultToString(result));
if (SL_RESULT_SUCCESS == result) {
LOGI("Realize the engine object");
// Realize the engine object
result = RealizeObject(engineObject);
if (SL_RESULT_SUCCESS == result) {
LOGI("Get the engine interface");
// Get the engine interface
result = GetEngineInterface();
}
}
}
2、初始化混音器，并Realize
简单说明下，创建的OpenSL的对象都至少实现了一个接口，但是其他的接口是可选的或者不支持的。这时候需要自己去指定接口(mids),接口数量，以及是否绝对需要(mreq)。如果指定了绝对需要的，然而不支持，就会报错，返回SL_RESULT_FEATURE_UNSUPPORTED。
SLresult CreateOutputMix() {
const SLInterfaceID mids[1] = {SL_IID_ENVIRONMENTALREVERB};
const SLboolean mreq[1] = {SL_BOOLEAN_TRUE};
return (*engineEngine)->CreateOutputMix(engineEngine, &outputMixObject, 1, mids, mreq);
} LOGI("Realize output mix object");
result = RealizeObject(outputMixObject);
if (SL_RESULT_SUCCESS != result) {
return result;
}3、创建播放器对象，并Realize
SLresult CreateBufferQueueAudioPlayer() {
SLDataLocator_AndroidBufferQueue dataSourceLocator = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
1
};
//PCM data source format
// SLDataFormat_PCM dataSourceFormat = { SL_DATAFORMAT_PCM, // format type
// 2, // channel count
// accompanySampleRate, // samples per second in millihertz
// 16, // bits per sample
// 16, // container size
// SL_SPEAKER_FRONT_CENTER, // channel mask
// SL_BYTEORDER_LITTLEENDIAN // endianness
// };
int samplesPerSec = opensl_get_sample_rate(accompanySampleRate);
//使用pcm数据格式，还可以使用URI等
SLDataFormat_PCM dataSourceFormat={
SL_DATAFORMAT_PCM,
static_cast(channels),
static_cast(samplesPerSec),
SL_PCMSAMPLEFORMAT_FIXED_16, // bits per sample
SL_PCMSAMPLEFORMAT_FIXED_16, // container size
SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT, // channel mask
SL_BYTEORDER_LITTLEENDIAN
};
// Data source is a simple buffer queue with PCM format
SLDataSource dataSource={&dataSourceLocator,&dataSourceFormat};
// Output mix locator for data sink
SLDataLocator_OutputMix dataSinkLocator={SL_DATALOCATOR_OUTPUTMIX,
outputMixObject
};
// Data sink is an output mix
SLDataSink dataSink = { &dataSinkLocator, // locator
0 // format
};
// Interfaces that are requested
SLInterfaceID interfaceIds[]={SL_IID_BUFFERQUEUE};
// Required interfaces. If the required interfaces
// are not available the request will fail
SLboolean requiredInterfaces[] = { SL_BOOLEAN_TRUE // for SL_IID_BUFFERQUEUE
};
return (*engineEngine)->CreateAudioPlayer(engineEngine,&audioPlayerObject,&dataSource,&dataSink,
ARRAY_LEN(interfaceIds), interfaceIds, requiredInterfaces
);
} 接着同样实现
LOGI("Realize audio player object");
// Realize audio player object
result = RealizeObject(audioPlayerObject);
if (SL_RESULT_SUCCESS != result) {
return result;
}4、获取播放队列接口，并注册回调
SLresult GetAudioPlayerBufferQueueInterface() {
// Get the buffer queue interface
return (*audioPlayerObject)->GetInterface(audioPlayerObject, SL_IID_BUFFERQUEUE, &audioPlayerBufferQueue);
};SLresult SoundService::RegisterPlayerCallback() {
return (*audioPlayerBufferQueue)->RegisterCallback(audioPlayerBufferQueue, PlayerCallback,
this);
}
在回调函数中入队数据，这里解码和之前的AudioTrack是一样的，我们只需要在回调函数拿队列的数据就行了。
static void PlayerCallback(SLAndroidSimpleBufferQueueItf bq,void *context){
SoundService* service= static_cast(context);
service->producePacket();
}
void SoundService::producePacket() {
LOGI("SoundService::producePacket() audio player call back method... ");
// Read data
byte *audioBuffer = buffer + (currentFrame % bufferNums) * packetBufferSize;
int result = -1;
if (NULL != decoderController) {
result = decoderController->readSamples(target, packetBufferSize / 2, NULL);
LOGI("enter SoundService::producePacket() PLAYING_STATE_PLAYING packetBufferSize=%d, result=%d",
packetBufferSize, result);
}
if (0 < result) {
//转成char数组后入队数据
convertByteArrayFromShortArray(target, result, audioBuffer);
(*audioPlayerBufferQueue)->Enqueue(audioPlayerBufferQueue, audioBuffer, result * 2);
} else {
LOGI("before DestroyContext");
// DestroyContext();
LOGI("after DestroyContext");
JNIEnv *env;
//播放完毕后会叫java函数
//获取对应线程的env
if (g_jvm->AttachCurrentThread(&env, NULL) != JNI_OK) {
LOGE("%s: AttachCurrentThread() failed", __FUNCTION__);
}
jclass jcls = env->GetObjectClass(obj);
jmethodID onCompletionCallBack = env->GetMethodID(jcls, "onCompletion", "()V");
LOGI("before env->CallVoidMethod");
env->CallVoidMethod(obj, onCompletionCallBack);
LOGI("after env->CallVoidMethod");
if (g_jvm->DetachCurrentThread() != JNI_OK) {
LOGE("%s: DetachCurrentThread() failed", __FUNCTION__);
}
}
currentFrame = (currentFrame + 1) % bufferNums;
}5、设置播放状态并手动调用一次回调函数
Lresult SoundService::play() {
//一开始需要主动调用回调函数，或者直接首先填充数据
LOGI("enter SoundService::play()...");
// Set the audio player state playing
LOGI("Set the audio player state playing");
if (playingState == PLAYING_STATE_PLAYING)
return 0;
SLresult result = SetAudioPlayerStatePlaying();
if (SL_RESULT_SUCCESS != result) {
return result;
}
LOGI("Set the slient audio player state playing");
// result = SetSlientAudioPlayerStatePlaying();
// if (SL_RESULT_SUCCESS != result) {
// return result;
// }
LOGI(" Enqueue the first buffer to start");
// for (int i = 0; i < bufferNums; i++) {
// int result = -1;
// if (NULL != decoderController) {
// result = decoderController->readSamples(target, packetBufferSize / 2, NULL);
// }
// LOGI("read_count %d", result);
// // If data is read
// if (0 < result) {
// convertByteArrayFromShortArray(target, result, buffer + i * packetBufferSize);
// (*audioPlayerBufferQueue)->Enqueue(audioPlayerBufferQueue,
// buffer + i * packetBufferSize, result * 2);
// }
// }
playingState = PLAYING_STATE_PLAYING;
PlayerCallback(nullptr, this);
LOGI("out SoundService::play()...");
return SL_RESULT_SUCCESS;
}
Android Studio 3.1.2写c++的方法没有返回值编译器不提示报错，然而运行的时候却特么会崩溃，真的是醉了，native使用ndk-stack快速调试，可以看这篇文章:https://blog.csdn.net/a568478312/article/details/78182422
最后附上说明OpenSL ES文档https://www.khronos.org/registry/OpenSL-ES/specs/OpenSL_ES_Specification_1.0.1.pdf
源码:https://download.csdn.net/download/a568478312/10421205