Video的thumbnail抽取算法
How to generate video thumbnail：
注意生成thumbnail时，传入getFrameAtTime的时间是－1
frameworks/base/media/java/android/media/ThumbnailUtils.java
158 public static Bitmap createVideoThumbnail(String filePath, int kind) {
159 Bitmap bitmap = null;
160 MediaMetadataRetriever retriever = new MediaMetadataRetriever();
161 try {
162 retriever.setDataSource(filePath);
163 bitmap = retriever.getFrameAtTime(-1);
164 } catch (IllegalArgumentException ex) {
165 // Assume this is a corrupt video file
166 } catch (RuntimeException ex) {
167 // Assume this is a corrupt video file.
168 } finally {
169 try {
170 retriever.release();
171 } catch (RuntimeException ex) {
172 // Ignore failures while cleaning up.
173 }
174 }
frameworks / base / media / java / android / media / MediaMetadataRetriever.java
270 public Bitmap getFrameAtTime(long timeUs) {
271 return getFrameAtTime(timeUs, OPTION_CLOSEST_SYNC);
frameworks/av/media/libmedia/mediametadataretriever.cpp
sp<IMemory> MediaMetadataRetriever::getFrameAtTime(int64_t timeUs, int option)
{
ALOGV("getFrameAtTime: time(%lld us) option(%d)", timeUs, option);
Mutex::Autolock _l(mLock);
if (mRetriever == 0) {
ALOGE("retriever is not initialized");
return NULL;
}
return mRetriever->getFrameAtTime(timeUs, option);
}
frameworks/av/media/libmediaplayerservice/MetadataRetrieverClient.cpp
sp<IMemory> MetadataRetrieverClient::getFrameAtTime(int64_t timeUs, int option)
{
ALOGV("getFrameAtTime: time(%lld us) option(%d)", timeUs, option);
Mutex::Autolock lock(mLock);
mThumbnail.clear();
if (mRetriever == NULL) {
ALOGE("retriever is not initialized");
return NULL;
}
VideoFrame *frame = mRetriever->getFrameAtTime(timeUs, option);
media/libstagefright/StagefrightMetadataRetriever.cpp
VideoFrame *StagefrightMetadataRetriever::getFrameAtTime(
int64_t timeUs, int option) {...
if (mExtractor.get() == NULL) {
ALOGV("no extractor.");
return NULL;
}
sp<MetaData> fileMeta = mExtractor->getMetaData();
...
size_t n = mExtractor->countTracks();
size_t i;
for (i = 0; i < n; ++i) {
sp<MetaData> meta = mExtractor->getTrackMetaData(i); // 这里将赋值给kKeyThumbnailTime
const char *mime;
CHECK(meta->findCString(kKeyMIMEType, &mime));
if (!strncasecmp(mime, "video/", 6)) {
break;
}
}
...
if (fileMeta->findData(kKeyAlbumArt, &type, &data, &dataSize)
&& mAlbumArt == NULL) {
mAlbumArt = new MediaAlbumArt;
mAlbumArt->mSize = dataSize;
mAlbumArt->mData = new uint8_t[dataSize];
memcpy(mAlbumArt->mData, data, dataSize);
}
VideoFrame *frame =
extractVideoFrameWithCodecFlags( ／／抽取thumbnail
&mClient, trackMeta, source, OMXCodec::kSoftwareCodecsOnly,
timeUs, option);
...}
下面函数具体抽取thumbnail：
static VideoFrame *extractVideoFrameWithCodecFlags(
OMXClient *client,
const sp<MetaData> &trackMeta,
const sp<MediaSource> &source,
uint32_t flags,
int64_t frameTimeUs,
int seekMode) {...
int64_t thumbNailTime;
if (frameTimeUs < 0) { ／／如果传入的时间为负数
if (!trackMeta->findInt64(kKeyThumbnailTime, &thumbNailTime)
|| thumbNailTime < 0) { ／／查看kKeyThumbnailTime是否存在
thumbNailTime = 0; ／／如不存在取第0帧
}
options.setSeekTo(thumbNailTime, mode);
} else {
thumbNailTime = -1;
options.setSeekTo(frameTimeUs, mode);
}
MediaBuffer *buffer = NULL;
do {
if (buffer != NULL) {
buffer->release();
buffer = NULL;
}
err = decoder->read(&buffer, &options);
options.clearSeekTo();
} while (err == INFO_FORMAT_CHANGED
|| (buffer != NULL && buffer->range_length() == 0));
之前在MPEG4Extractor中读取TrackMetadata时，已经对取哪一帧作为thumbnail进行了计算，并存入到kKeyThumbnailTime中。具体代码见下面的函数getTrackMetaData
注意仅当存在movie fragments ('moof' box）时，将把1/4duration处的帧作为thumbnail，否则将取前20个sample内具有最大值的sample作为thumbnail
media/libstagefright/MPEG4Extractor.cpp
sp<MetaData> MPEG4Extractor::getTrackMetaData(
size_t index, uint32_t flags) {...
if ((flags & kIncludeExtensiveMetaData)
&& !track->includes_expensive_metadata) {
track->includes_expensive_metadata = true; ／／置位
const char *mime;
CHECK(track->meta->findCString(kKeyMIMEType, &mime));
if (!strncasecmp("video/", mime, 6)) { ／／如果是video
if (mMoofOffset > 0) { // if has moof box ,why ??
int64_t duration;
if (track->meta->findInt64(kKeyDuration, &duration)) {
// nothing fancy, just pick a frame near 1/4th of the duration
track->meta->setInt64(
kKeyThumbnailTime, duration / 4);
}
} else {
uint32_t sampleIndex;
uint32_t sampleTime;
if (track->sampleTable->findThumbnailSample(&sampleIndex) == OK
&& track->sampleTable->getMetaDataForSample(
sampleIndex, NULL /* offset */, NULL /* size */,
&sampleTime) == OK) {
track->meta->setInt64( ／／将选定sample处的时间作为thumbnail对应时间
kKeyThumbnailTime,
((int64_t)sampleTime * 1000000) / track->timescale);
}
}
}
具体sample的挑选算法：
media/libstagefright/SampleTable.cpp
status_t SampleTable::findThumbnailSample(uint32_t *sample_index) {
Mutex::Autolock autoLock(mLock);
uint32_t bestSampleIndex = 0;
size_t maxSampleSize = 0;
static const size_t kMaxNumSyncSamplesToScan = 20; ／／取前20个sample
// Consider the first kMaxNumSyncSamplesToScan sync samples and
// pick the one with the largest (compressed) size as the thumbnail.
size_t numSamplesToScan = mNumSyncSamples;
if (numSamplesToScan > kMaxNumSyncSamplesToScan) {
numSamplesToScan = kMaxNumSyncSamplesToScan;
}
for (size_t i = 0; i < numSamplesToScan; ++i) {
uint32_t x = mSyncSamples[i];
// Now x is a sample index.
size_t sampleSize;
status_t err = getSampleSize_l(x, &sampleSize);
if (err != OK) {
return err;
}
if (i == 0 || sampleSize > maxSampleSize) {
bestSampleIndex = x;
maxSampleSize = sampleSize;
}
} *sample_index = bestSampleIndex; return OK; }