Android8.0 Media系统(二)
上一篇从MediaPlayer的创建，设置数据源，播放准备三个流程来看Media系统，今天我们继续来看initFromDataSource()函数，将创建数据提取器，数据提取器将读取数据源文件的元数据信息，将每一路数据流的比特率进行累加，分离数据流，音频/视频/字幕分离：设置视频源mVideoTrack ；设置音频源mAudioTrack；分离字幕等。废话少说直接上代码。
1. 数据提取
status_t NuPlayer::GenericSource::initFromDataSource() {
sp<IMediaExtractor> extractor;
extractor = MediaExtractor::Create(mDataSource, NULL); //根据数据源创建多媒体提取器
mFileMeta = extractor->getMetaData(); //获取元数据
size_t numtracks = extractor->countTracks();//获取数据源的Track数
for (size_t i = 0; i < numtracks; ++i) {
sp<IMediaSource> track = extractor->getTrack(i); //获取MediaSource
sp<MetaData> meta = extractor->getTrackMetaData(i);//获取Track元数据
......
if (!strncasecmp(mime, "audio/", 6)) {
if (mAudioTrack.mSource == NULL) {
mAudioTrack.mIndex = i;
mAudioTrack.mSource = track;
mAudioTrack.mPackets =
new AnotherPacketSource(mAudioTrack.mSource->getFormat());
if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_VORBIS)) {
mAudioIsVorbis = true;
} else {
mAudioIsVorbis = false;
}
mMimes.add(String8(mime));
}
} else if (!strncasecmp(mime, "video/", 6)) {
if (mVideoTrack.mSource == NULL) {
mVideoTrack.mIndex = i;
mVideoTrack.mSource = track;
mVideoTrack.mPackets =
new AnotherPacketSource(mVideoTrack.mSource->getFormat());
// video always at the beginning
mMimes.insertAt(String8(mime), 0);
}
}
//保存每一Track
mSources.push(track);
//时间同步
int64_t durationUs;
if (meta->findInt64(kKeyDuration, &durationUs)) {
if (durationUs > mDurationUs) {
mDurationUs = durationUs;
}
}
//统计比特率
int32_t bitrate;
if (totalBitrate >= 0 && meta->findInt32(kKeyBitRate, &bitrate)) {
totalBitrate += bitrate;
} else {
totalBitrate = -1;
}
}
......
return OK;
}
frameworks\av\media\libstagefright\MediaExtractor.cpp
// static
sp<IMediaExtractor> MediaExtractor::Create(
const sp<DataSource> &source, const char *mime) {
if (!property_get_bool("media.stagefright.extractremote", true)) { //存在本地提取服务
// local extractor
return CreateFromService(source, mime);
} else {
// remote extractor
sp<IBinder> binder = defaultServiceManager()->getService(String16("media.extractor"));
if (binder != 0) {
//获取多媒体提取器服务
sp<IMediaExtractorService> mediaExService(interface_cast<IMediaExtractorService>(binder));
//根据source返回具体的数据提取器
sp<IMediaExtractor> ex = mediaExService->makeExtractor(source->asIDataSource(), mime);
return ex;
} else {
return NULL;
}
}
return NULL;
}
frameworks\av\services\mediaextractor\MediaExtractorService.cpp
sp<IMediaExtractor> MediaExtractorService::makeExtractor(
const sp<IDataSource> &remoteSource, const char *mime) {
//获取localSource
sp<DataSource> localSource = DataSource::CreateFromIDataSource(remoteSource);
//调用 CreateFromService
sp<IMediaExtractor> ret = MediaExtractor::CreateFromService(localSource, mime);
......
return ret;
}
按MIME类型返回具体类型的多媒体数据提取器
sp<MediaExtractor> MediaExtractor::CreateFromService(
const sp<DataSource> &source, const char *mime) {
// initialize source decryption if needed
source->DrmInitialization(nullptr /* mime */); //初始化数字版权
sp<AMessage> meta;
......
MediaExtractor *ret = NULL;
if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG4)
|| !strcasecmp(mime, "audio/mp4")) {
ret = new MPEG4Extractor(source); //MP4数据提取器
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_MPEG)) {
ret = new MP3Extractor(source, meta); //MP3数据提取器
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AMR_NB)
|| !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AMR_WB)) {
ret = new AMRExtractor(source);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_FLAC)) {
ret = new FLACExtractor(source);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_WAV)) {
ret = new WAVExtractor(source);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_OGG)) {
ret = new OggExtractor(source);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MATROSKA)) {
ret = new MatroskaExtractor(source);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG2TS)) {
ret = new MPEG2TSExtractor(source);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC_ADTS)) {
ret = new AACExtractor(source, meta);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG2PS)) {
ret = new MPEG2PSExtractor(source);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_MIDI)) {
ret = new MidiExtractor(source);
}
return ret;
}
//获取MediaSource,成员函数parseChunk()控制从MPEG4DataSource解析元数据信息
sp<IMediaSource> MPEG4Extractor::getTrack(size_t index) {
status_t err;
if ((err = readMetaData()) != OK) {
return NULL;
}
Track *track = mFirstTrack;
while (index > 0) {
track = track->next;
--index;
}
Trex *trex = NULL;
int32_t trackId;
if (track->meta->findInt32(kKeyTrackID, &trackId)) {
for (size_t i = 0; i < mTrex.size(); i++) {
Trex *t = &mTrex.editItemAt(i);
if (t->track_ID == (uint32_t) trackId) {
trex = t;
break;
}
}
} else {
ALOGE("b/21657957");
return NULL;
}
//新建了一个多媒体源
sp<MPEG4Source> source = new MPEG4Source(this,
track->meta, mDataSource, track->timescale, track->sampleTable,
mSidxEntries, trex, mMoofOffset);
if (source->init() != OK) {
return NULL;
}
return source;
}
2. 数据解码
到目前为止MediaPlayer执行完了setDataSource(path), prepare()两个函数，为多媒体播放准备好了数据，下一步将调用start()函数，对提取出来的多媒体元数据进行解码输出。我们直接跳过Java层面繁琐的调用跳转，直接从NuPlayer的start()函数开始，Amazing !
void NuPlayer::start() {
(new AMessage(kWhatStart, this))->post();
}
经过AHandler消息通讯机制，在如下出继续执行onStart()
case kWhatStart:
{
ALOGV("kWhatStart");
if (mStarted) {
// do not resume yet if the source is still buffering
if (!mPausedForBuffering) {
onResume();
}
} else {
onStart();
}
mPausedByClient = false;
break;
}
void NuPlayer::onStart(int64_t startPositionUs, MediaPlayerSeekMode mode) {
......
bool hasAudio = (mSource->getFormat(true /* audio */) != NULL);
bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); //获取音频元数据
//音频是否需要需要解码
mOffloadAudio =
canOffloadStream(audioMeta, hasVideo, mSource->isStreaming(), streamType)
&& (mPlaybackSettings.mSpeed == 1.f && mPlaybackSettings.mPitch == 1.f);
sp<AMessage> notify = new AMessage(kWhatRendererNotify, this);
++mRendererGeneration;
notify->setInt32("generation", mRendererGeneration);
mRenderer = new Renderer(mAudioSink, notify, flags); //渲染
mRendererLooper = new ALooper;
mRendererLooper->setName("NuPlayerRenderer");
mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
mRendererLooper->registerHandler(mRenderer);
float rate = getFrameRate(); //获取帧率
if (rate > 0) {
mRenderer->setVideoFrameRate(rate); //设置给渲染器
}
if (mVideoDecoder != NULL) {
mVideoDecoder->setRenderer(mRenderer);
}
if (mAudioDecoder != NULL) {
mAudioDecoder->setRenderer(mRenderer);
}
//扫描
postScanSources();
}
void NuPlayer::postScanSources() {
sp<AMessage> msg = new AMessage(kWhatScanSources, this);
msg->setInt32("generation", mScanSourcesGeneration);
msg->post();
mScanSourcesPending = true;
}
AHandler消息循环到此处
case kWhatScanSources:
{
// initialize video before audio because successful initialization of
// video may change deep buffer mode of audio.
if (mSurface != NULL) {
//初始化解码器
if (instantiateDecoder(false, &mVideoDecoder) == -EWOULDBLOCK) {
rescan = true;
}
}
// Don't try to re-open audio sink if there's an existing decoder.
if (mAudioSink != NULL && mAudioDecoder == NULL) {
if (instantiateDecoder(true, &mAudioDecoder) == -EWOULDBLOCK) {
rescan = true;
}
}
......
if (rescan) { //延时重扫
msg->post(100000ll);
mScanSourcesPending = true;
}
break;
}
初始化解码器
status_t NuPlayer::instantiateDecoder(
bool audio, sp<DecoderBase> *decoder, bool checkAudioModeChange)
......
if (audio) { //音频
sp<AMessage> notify = new AMessage(kWhatAudioNotify, this);
++mAudioDecoderGeneration;
notify->setInt32("generation", mAudioDecoderGeneration);
if (checkAudioModeChange) {
determineAudioModeChange(format);
}
if (mOffloadAudio) {
mSource->setOffloadAudio(true /* offload */); //设置离载音频
const bool hasVideo = (mSource->getFormat(false /*audio */) != NULL);
format->setInt32("has-video", hasVideo);
*decoder = new DecoderPassThrough(notify, mSource, mRenderer); //创建解码器
ALOGV("instantiateDecoder audio DecoderPassThrough hasVideo: %d", hasVideo);
} else {
mSource->setOffloadAudio(false /* offload */);
*decoder = new Decoder(notify, mSource, mPID, mUID, mRenderer); //创建解码器
ALOGV("instantiateDecoder audio Decoder");
}
mAudioDecoderError = false;
} else { //视频
sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
++mVideoDecoderGeneration;
notify->setInt32("generation", mVideoDecoderGeneration);
*decoder = new Decoder( //解码器
notify, mSource, mPID, mUID, mRenderer, mSurface, mCCDecoder);
mVideoDecoderError = false;
// enable FRC if high-quality AV sync is requested, even if not
// directly queuing to display, as this will even improve textureview
// playback.
{
if (property_get_bool("persist.sys.media.avsync", false)) {
format->setInt32("auto-frc", 1);
}
}
}
(*decoder)->init(); //解码器初始化
// Modular DRM
if (mIsDrmProtected) { //是否受数字版权保护
format->setPointer("crypto", mCrypto.get());
}
(*decoder)->configure(format); //配置解码器
.....
return OK;
}
frameworks\av\media\libmediaplayerservice\nuplayer\NuPlayerDecoder.cpp
配置NuPlayer解码器，将调用StageFright的MediaCodec进行解码
void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
......
mComponentName = mime;
mComponentName.append(" decoder");
//创建解码器
mCodec = MediaCodec::CreateByType(
mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
int32_t secure = 0;
if (format->findInt32("secure", &secure) && secure != 0) {
if (mCodec != NULL) {
mCodec->getName(&mComponentName); //设置组件
mComponentName.append(".secure");
mCodec->release();
//按组件名构建
mCodec = MediaCodec::CreateByComponentName(
mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
}
}
mIsSecure = secure;
mCodec->getName(&mComponentName);
// Modular DRM
void *pCrypto;
if (!format->findPointer("crypto", &pCrypto)) {
pCrypto = NULL;
}
sp<ICrypto> crypto = (ICrypto*)pCrypto;
// non-encrypted source won't have a crypto
mIsEncrypted = (crypto != NULL);
// configure is called once; still using OR in case the behavior changes.
mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted;
ALOGV("onConfigure mCrypto: %p (%d) mIsSecure: %d",
crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
//配置解码器
err = mCodec->configure(
format, mSurface, crypto, 0 /* flags */);
......
if (!mIsAudio) { //视频
int32_t width, height;
if (mOutputFormat->findInt32("width", &width)
&& mOutputFormat->findInt32("height", &height)) {
mStats->setInt32("width", width);
mStats->setInt32("height", height);
}
}
sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
mCodec->setCallback(reply); //处理mCodec返回的消息，数据的输入输出等
err = mCodec->start(); //启动解码
......
}
frameworks\av\media\libmediaplayerservice\nuplayer\NuPlayerDecoder.cpp
NuPlayerDecoder接收来自MediaCodec的解码消息
case MediaCodec::CB_OUTPUT_AVAILABLE:
{
......
handleAnOutputBuffer(index, offset, size, timeUs, flags);
break;
}
处理来自多媒体解码器的渲染输出
bool NuPlayer::Decoder::handleAnOutputBuffer(......) {
// CHECK_LT(bufferIx, mOutputBuffers.size());
sp<MediaCodecBuffer> buffer;
mCodec->getOutputBuffer(index, &buffer);
......
mOutputBuffers.editItemAt(index) = buffer;
buffer->setRange(offset, size);
buffer->meta()->clear();
buffer->meta()->setInt64("timeUs", timeUs);
bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
// we do not expect CODECCONFIG or SYNCFRAME for decoder
//发送kWhatRenderBuffer消息
sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
reply->setSize("buffer-ix", index);
reply->setInt32("generation", mBufferGeneration);
......
mNumFramesTotal += !mIsAudio;
// wait until 1st frame comes out to signal resume complete
notifyResumeCompleteIfNecessary();
if (mRenderer != NULL) {
//把Buffer送到Renderer
mRenderer->queueBuffer(mIsAudio, buffer, reply);
if (eos && !isDiscontinuityPending()) {
mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
}
}
return true;
}
收到渲染输出消息
case kWhatRenderBuffer:
{
if (!isStaleReply(msg)) {
onRenderBuffer(msg);
}
break;
}
渲染输出
void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
if (!mIsAudio) { //视频
int64_t timeUs;
sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
buffer->meta()->findInt64("timeUs", &timeUs);
//字幕显示
if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
mCCDecoder->display(timeUs);
}
}
if (msg->findInt32("render", &render) && render) {
int64_t timestampNs;
CHECK(msg->findInt64("timestampNs", ×tampNs));
err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); //渲染输出
} else {
mNumOutputFramesDropped += !mIsAudio;
err = mCodec->releaseOutputBuffer(bufferIx);
}
......
}
收到来自解码器的多媒体数据输入请求
case MediaCodec::CB_INPUT_AVAILABLE:
{
int32_t index;
CHECK(msg->findInt32("index", &index));
handleAnInputBuffer(index);
break;
}
处理多媒体数据输入请求
bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
......
onRequestInputBuffers();
return true;
}
抓取多媒体数据
bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
.....
// attempt to queue EOS
status_t err = mCodec->queueInputBuffer(
bufferIx,
0,
0,
0,
MediaCodec::BUFFER_FLAG_EOS);
......
return true;
}
循环的发消息请求数据
void NuPlayer::DecoderBase::onRequestInputBuffers() {
if (mRequestInputBuffersPending) {
return;
}
// doRequestBuffers() return true if we should request more data
if (doRequestBuffers()) {
mRequestInputBuffersPending = true;
//注意这里，会自己循环调用，不停地request
sp<AMessage> msg = new AMessage(kWhatRequestInputBuffers, this);
msg->post(10 * 1000ll);
}
}
请求缓冲数据
bool NuPlayer::Decoder::doRequestBuffers() {
......
while (err == OK && !mDequeuedInputBuffers.empty()) {
size_t bufferIx = *mDequeuedInputBuffers.begin();
sp<AMessage> msg = new AMessage();
msg->setSize("buffer-ix", bufferIx);
err = fetchInputData(msg);
......
}
return err == -EWOULDBLOCK
&& mSource->feedMoreTSData() == OK;
}
从DataSource取数据
status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
sp<ABuffer> accessUnit;
bool dropAccessUnit;
do {
//在这里调用source的方法，建立起了联系
status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
......
｝while(...)
}
回到启动解码函数 err = mCodec->start();
case kWhatStart:
{
sp<AReplyToken> replyID;
CHECK(msg->senderAwaitsResponse(&replyID));
if (mState == FLUSHED) {
setState(STARTED);
if (mHavePendingInputBuffers) {
onInputBufferAvailable();
mHavePendingInputBuffers = false;
}
mCodec->signalResume();
PostReplyWithError(replyID, OK);
break;
} else if (mState != CONFIGURED) {
PostReplyWithError(replyID, INVALID_OPERATION);
break;
}
mReplyID = replyID;
setState(STARTING);
mCodec->initiateStart();
break;
}
frameworks\av\media\libstagefright\ACodec.cpp
void ACodec::initiateStart() {
(new AMessage(kWhatStart, this))->post();
}
调用openmax解码
void ACodec::LoadedState::onStart() {
ALOGV("onStart");
status_t err = mCodec->mOMXNode->sendCommand(OMX_CommandStateSet, OMX_StateIdle);
if (err != OK) {
mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
} else {
mCodec->changeState(mCodec->mLoadedToIdleState);
}
}
关于openmax解码部分暂且不做分析，我们从一个Demo开始分析了MediaPlayer的整个流程，对多媒体的分析暂且就到这里，以后涉及到Surface系统，Camera系统再来结合具体情境分析。下一个主题我们去跟踪Camera子系统。从近两年Google在相机方面的改进来看，Camera子系统势必会复杂一点，还是那句话：show me the code !