android mpeg2ts 流媒体打包MediaMuxer 和 录制MPEG2TSWriter 以及抽帧MPEG2TSExtractor
目前android上，录相大多是mp4的视频，这在一般情况下，已经够用了。但是在一些特定的场景，比如远程临控录相或者行车记录仪上，用mp4录相，就不太理想了。为什么呢？因为远程录相，或者行车记录仪上都有一个共同的问题，那就是录相有可能中断。比如突然撞车了，或者是远程监控断电了，如果这时录的是Mp4的视频，那么就会导致，因为没有来得及写mp4的文件头信息，从而打不开视频。所以在远程监控录相和行车记录仪上，录相的格式，最好使用mpeg2ts流。
现在android无论是8.0还是9.0、10.0上，都支持录制mpeg2ts流视频，但是却不支持用MediaMuxer的writeSampleData去打包mpeg2ts。这就导致了一个问题，比如有的app上，想将一个mp4的视频，转成mpeg2ts流的视频，就无法在java端完成。且现在android8.1(9.0、10.0上没有试)，录下来的mpeg2ts流，经常会丢帧，最后几帧录不下来。这个Mpeg2Ts功能显得很鸡肋。下面，我们就来讨论一下，怎么去解决这些问题。
先说一下mpeg2ts录相丢帧的问题。mpeg2ts录相的framework层cpp文件是frameworks\av\media\libstagefrightMPEG2TSWriter.cpp这一个。写数据的函数是：MPEG2TSWriter::onMessageReceived(const sp
void MPEG2TSWriter::onMessageReceived(const sp &msg) {
switch (msg->what()) {
case kWhatSourceNotify:
{
int32_t sourceIndex;
CHECK(msg->findInt32("source-index", &sourceIndex));
sp source = mSources.editItemAt(sourceIndex);
int32_t what;
CHECK(msg->findInt32("what", &what));
if (what == SourceInfo::kNotifyReachedEOS
|| what == SourceInfo::kNotifyStartFailed) {
source->setEOSReceived();
sp buffer = source->lastAccessUnit();
source->setLastAccessUnit(NULL);
if (buffer != NULL) {
writeTS();
writeAccessUnit(sourceIndex, buffer);
}
++mNumSourcesDone;
} else if (what == SourceInfo::kNotifyBuffer) {
sp buffer;
CHECK(msg->findBuffer("buffer", &buffer));
CHECK(source->lastAccessUnit() == NULL);
int32_t oob;
if (msg->findInt32("oob", &oob) && oob) {
// This is codec specific data delivered out of band.
// It can be written out immediately.
writeTS();
writeAccessUnit(sourceIndex, buffer);
break;
}
// We don't just write out data as we receive it from
// the various sources. That would essentially write them
// out in random order (as the thread scheduler determines
// how the messages are dispatched).
// Instead we gather an access unit for all tracks and
// write out the one with the smallest timestamp, then
// request more data for the written out track.
// Rinse, repeat.
// If we don't have data on any track we don't write
// anything just yet.
source->setLastAccessUnit(buffer);
ALOGV("lastAccessUnitTimeUs[%d] = %.2f secs",
sourceIndex, source->lastAccessUnitTimeUs() / 1E6);
int64_t minTimeUs = -1;
size_t minIndex = 0;
for (size_t i = 0; i < mSources.size(); ++i) {
const sp &source = mSources.editItemAt(i);
if (source->eosReceived()) {
continue;
}
int64_t timeUs = source->lastAccessUnitTimeUs();
if (timeUs < 0) {
minTimeUs = -1;
break;
} else if (minTimeUs < 0 || timeUs < minTimeUs) {
minTimeUs = timeUs;
minIndex = i;
}
}
if (minTimeUs < 0) {
ALOGV("not all tracks have valid data.");
break;
}
ALOGV("writing access unit at time %.2f secs (index %zu)",
minTimeUs / 1E6, minIndex);
source = mSources.editItemAt(minIndex);
buffer = source->lastAccessUnit();
source->setLastAccessUnit(NULL);
writeTS();
writeAccessUnit(minIndex, buffer);
source->readMore();
}
break;
}
default:
TRESPASS();
}
} 在这个函数里，收到数据，并写入到文件的是“what == SourceInfo::kNotifyBuffer”这个条件下的代码段。注意在这个代码段里的那个for循环，我们丢帧就是在这里丢的。
这个for循环的作用是干什么呢？它的作用是，选取当前录制的视频的几个源中，时间戳最小的那一个源的数据，并将选取的源的数据写入文件。这么做的原因上面的注释写了，大意是，一个视频会有几个源，分属不同的线程。因为在不同的线程，所以调度时间有先后顺序，有时视频数据已经读取到了，但是cpu现在调度的是音频源，视频数据就要等音频源写完数据后，再去写视频源的数据，这样就会导致声音和视频有错位。比如播放的时候，声音说完了，对应的画面过了一秒才播出来。
google的这个解释，似乎说的通，似乎有那么一丝的道理。但是实际上，这段代码逻辑却是有混乱不堪。再举个例子，比如当前收到的是视频帧，视频帧的timeUS，也就是时间戳是112233。然后第一次执行完这个for循环后，minTimeUs会等于112233，i=1。然后因为还有音频，会第二次执行这个for循环。假设这时音频的时间戳是112232，它比视频的时间戳小，那么，minTimeUS就被改成了112232， minIndex=2。好了，执行完上面两次for循环后，会马上执行source = mSources.editItemAt(minIndex);，去取出音频的数据，写入文件，然后再紧接着调用source->readMore();去继续读取音频的内容。
不知道大家有没有注意到，本来这次发送SourceInfo::kNotifyBuffer这个整个的源是视频源，但是到最后，写入的数据却是音频源的。那么视频源的数据到哪里去了呢？没错，居然被直接丢弃掉了，丢弃掉了........
不知道写这段逻辑的人的脑子是怎么长的，总之，这里的逻辑是个很明显的错误。想要解决这个问题也很简单，把这个fro循环去掉，SourceInfo::kNotifyBuffer这个源是谁发过来的，就写谁的数据，不用去管时间戳。因为mpeg2ts流数据，每个pes数据包中，都包含了它的时间戳。具体的可以看下面的代码：
void MPEG2TSWriter::writeAccessUnit(
int32_t sourceIndex, const sp &accessUnit) {
........
int64_t timeUs;
CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
uint32_t PTS = (timeUs * 9ll) / 100ll;
........
} 再者，在视频文件里，无论是哪种格式的，音频轨和视频轨都是分开存放的。接收存储数据时，只用管当前轨道的数据是按先后顺序存放的就可以。 所以，根本不需要多此一举，在收到某个源的数据后，还要和其他源的数据比时间戳。修改后的代码如下：
void MPEG2TSWriter::onMessageReceived(const sp &msg) {
switch (msg->what()) {
case kWhatSourceNotify:
{
int32_t sourceIndex;
CHECK(msg->findInt32("source-index", &sourceIndex));
sp source = mSources.editItemAt(sourceIndex);
int32_t what;
CHECK(msg->findInt32("what", &what));
if (what == SourceInfo::kNotifyReachedEOS
|| what == SourceInfo::kNotifyStartFailed) {
source->setEOSReceived();
sp buffer = source->lastAccessUnit();
source->setLastAccessUnit(NULL);
if (buffer != NULL) {
writeTS();
writeAccessUnit(sourceIndex, buffer);
}
++mNumSourcesDone;
} else if (what == SourceInfo::kNotifyBuffer) {
sp buffer;
CHECK(msg->findBuffer("buffer", &buffer));
CHECK(source->lastAccessUnit() == NULL);
int32_t oob;
if (msg->findInt32("oob", &oob) && oob) {
// This is codec specific data delivered out of band.
// It can be written out immediately.
writeTS();
writeAccessUnit(sourceIndex, buffer);
break;
}
// We don't just write out data as we receive it from
// the various sources. That would essentially write them
// out in random order (as the thread scheduler determines
// how the messages are dispatched).
// Instead we gather an access unit for all tracks and
// write out the one with the smallest timestamp, then
// request more data for the written out track.
// Rinse, repeat.
// If we don't have data on any track we don't write
// anything just yet.
source->setLastAccessUnit(buffer);
#if 0
ALOGV("lastAccessUnitTimeUs[%d] = %.2f secs",
sourceIndex, source->lastAccessUnitTimeUs() / 1E6);
int64_t minTimeUs = -1;
size_t minIndex = 0;
for (size_t i = 0; i < mSources.size(); ++i) {
const sp &source = mSources.editItemAt(i);
if (source->eosReceived()) {
continue;
}
int64_t timeUs = source->lastAccessUnitTimeUs();
if (timeUs < 0) {
minTimeUs = -1;
break;
} else if (minTimeUs < 0 || timeUs < minTimeUs) {
minTimeUs = timeUs;
minIndex = i;
}
}
if (minTimeUs < 0) {
ALOGV("not all tracks have valid data.");
break;
}
ALOGV("writing access unit at time %.2f secs (index %zu)",
minTimeUs / 1E6, minIndex);
source = mSources.editItemAt(minIndex);
#endif
buffer = source->lastAccessUnit();
source->setLastAccessUnit(NULL);
writeTS();
//writeAccessUnit(minIndex, buffer);
writeAccessUnit(sourceIndex, buffer);
source->readMore();
}
break;
}
default:
TRESPASS();
}
} 好了，上面这样修改后，经过反复测试验证，录下来的视频不存在丢帧的问题，丢帧的问题完美的解决了。
现在再来说说，怎么去提供mpeg2ts流的mediamuxer给java层使用。先上一段java上的测试代码：
MediaExtractor extractor;
int trackCount;
MediaMuxer muxer;
HashMap indexMap;
private void cloneMediaUsingMuxer(FileDescriptor srcMedia, String dstMediaPath,
int expectedTrackCount, int degrees, int fmt) throws IOException {
// Set up MediaExtractor to read from the source.
extractor = new MediaExtractor();
extractor.setDataSource(srcMedia, 0, testFileLength);
trackCount = extractor.getTrackCount();
muxer = new MediaMuxer(dstMediaPath, fmt);
indexMap = new HashMap(trackCount);
for (int i = 0; i < trackCount; i++) {
extractor.selectTrack(i);
MediaFormat format = extractor.getTrackFormat(i);
int dstIndex = muxer.addTrack(format);
indexMap.put(i, dstIndex);
}
if (degrees >= 0) {
muxer.setOrientationHint(degrees);
}
muxer.start();
Handler handler = new Handler();
handler.postDelayed(new Runnable() {
@Override
public void run() {
// Copy the samples from MediaExtractor to MediaMuxer.
boolean sawEOS = false;
int bufferSize = MAX_SAMPLE_SIZE;
int frameCount = 0;
int offset = 100;
ByteBuffer dstBuf = ByteBuffer.allocate(bufferSize);
BufferInfo bufferInfo = new BufferInfo();
while (!sawEOS) {
bufferInfo.offset = offset;
bufferInfo.size = extractor.readSampleData(dstBuf, offset);
if (bufferInfo.size < 0) {
sawEOS = true;
bufferInfo.size = 0;
} else {
bufferInfo.presentationTimeUs = extractor.getSampleTime();
bufferInfo.flags = extractor.getSampleFlags();
int trackIndex = extractor.getSampleTrackIndex();
muxer.writeSampleData(indexMap.get(trackIndex), dstBuf,
bufferInfo);
extractor.advance();
frameCount++;
}
}
muxer.stop();
muxer.release();
}
//这里延时10毫秒执行，是因为mpeg2ts的muxer有时启动稍慢。如果writeSampleData的
//的时候，muxer还没启动，就会报错
}, 10);
return;
} 这段代码中，就做了一件事，那就是从给定的文件里，用MediaExtractor去抽出每一帧，然后再用MediaMuxer将抽出的帧，打包成指定格式的视频文件。我们的目的是将一个给定的视频，通过mediaMuxer打包成mpeg2ts流视频。但是从frameworks\base\media\java\android\media\MediaMuxer.java的setUpMediaMuxer里可以看出，目前android不支持转成mpeg2ts流。要想达到我们的目的，首先需要在setUpMediaMuxer这个函数里，将mpeg2ts格式给加上去。
public static final class OutputFormat {
/* Do not change these values without updating their counterparts
* in include/media/stagefright/MediaMuxer.h!
*/
private OutputFormat() {}
/** MPEG4 media file format*/
public static final int MUXER_OUTPUT_MPEG_4 = 0;
/** WEBM media file format*/
public static final int MUXER_OUTPUT_WEBM = 1;
/** 3GPP media file format*/
public static final int MUXER_OUTPUT_3GPP = 2;
public static final int MUXER_OUTPUT_MPEG2TS = 3;
};
private void setUpMediaMuxer(@NonNull FileDescriptor fd, @Format int format) throws IOException {
if (format != OutputFormat.MUXER_OUTPUT_MPEG_4 && format != OutputFormat.MUXER_OUTPUT_WEBM
&& format != OutputFormat.MUXER_OUTPUT_3GPP && format != OutputFormat.MUXER_OUTPUT_MPEG2TS) {
throw new IllegalArgumentException("format: " + format + " is invalid");
}
mNativeObject = nativeSetup(fd, format);
mState = MUXER_STATE_INITIALIZED;
mCloseGuard.open("release");
}
在上面，我们新增了一个格式MUXER_OUTPUT_MPEG2TS 。然后这里就一步步的调到了frameworks\av\media\libstagefright\MediaMuxer.cpp，同样，我们需要在这个文件里，增加我们的格式：
enum OutputFormat {
OUTPUT_FORMAT_MPEG_4 = 0,
OUTPUT_FORMAT_WEBM = 1,
OUTPUT_FORMAT_THREE_GPP = 2,
//add by mpeg2ts
OUTPUT_FORMAT_YUNOVO_MPEG2TS = 3,
OUTPUT_FORMAT_LIST_END // must be last - used to validate format type
};
MediaMuxer::MediaMuxer(int fd, OutputFormat format)
: mFormat(format),
mState(UNINITIALIZED) {
ALOGV("MediaMuxer start, format=%d", format);
if (format == OUTPUT_FORMAT_MPEG_4 || format == OUTPUT_FORMAT_THREE_GPP) {
mWriter = new MPEG4Writer(fd);
} else if (format == OUTPUT_FORMAT_WEBM) {
mWriter = new WebmWriter(fd);
}
//add mpeg2ts
else if (format == OUTPUT_FORMAT_YUNOVO_MPEG2TS){
mWriter = new MPEG2TSWriter(fd);
}//add end
if (mWriter != NULL) {
mFileMeta = new MetaData;
mState = INITIALIZED;
}
}好了，到这里为止，从java到c++层的接口，就算是打通了。现在就可以使用extractor.readSampleData去抽取视频帧数据，然后使用muxer.writeSampleData去写mpeg2ts流文件了。
下面顺便说一下，这个抽帧和写帧的流程。我们在MediaMuxer.cpp里构建好Muxer后，就可以在java层上通过muxer.addTrack(format)，将源文件里的视频track和音频track甚至字幕track添加进来了。
ssize_t MediaMuxer::addTrack(const sp &format) {
Mutex::Autolock autoLock(mMuxerLock);
if (format.get() == NULL) {
ALOGE("addTrack() get a null format");
return -EINVAL;
}
if (mState != INITIALIZED) {
ALOGE("addTrack() must be called after constructor and before start().");
return INVALID_OPERATION;
}
sp trackMeta = new MetaData;
convertMessageToMetaData(format, trackMeta);
sp newTrack = new MediaAdapter(trackMeta);
status_t result = mWriter->addSource(newTrack);
if (result == OK) {
return mTrackList.add(newTrack);
}
return -1;
} 我们注意到，这里的track，是一个MediaAdapter类。请大家记住这个类，因为后面我们在java层调用writeSampleData去写帧数据时，最终都是通过这个类去push buffer的。
status_t MediaMuxer::writeSampleData(const sp &buffer, size_t trackIndex,
int64_t timeUs, uint32_t flags) {
Mutex::Autolock autoLock(mMuxerLock);
ALOGV("MediaMuxer::writeSampleData trackIndex= %zu; timeUs= %" PRIu64, trackIndex, timeUs);
if (buffer.get() == NULL) {
ALOGE("WriteSampleData() get an NULL buffer.");
return -EINVAL;
}
if (mState != STARTED) {
ALOGE("WriteSampleData() is called in invalid state %d", mState);
return INVALID_OPERATION;
}
if (trackIndex >= mTrackList.size()) {
ALOGE("WriteSampleData() get an invalid index %zu", trackIndex);
return -EINVAL;
}
ALOGV("MediaMuxer::writeSampleData buffer offset = %zu, length = %zu", buffer->offset(), buffer->size());
MediaBuffer* mediaBuffer = new MediaBuffer(buffer);
mediaBuffer->add_ref(); // Released in MediaAdapter::signalBufferReturned().
mediaBuffer->set_range(buffer->offset(), buffer->size());
sp sampleMetaData = mediaBuffer->meta_data();
sampleMetaData->setInt64(kKeyTime, timeUs);
// Just set the kKeyDecodingTime as the presentation time for now.
sampleMetaData->setInt64(kKeyDecodingTime, timeUs);
if (flags & MediaCodec::BUFFER_FLAG_SYNCFRAME) {
sampleMetaData->setInt32(kKeyIsSyncFrame, true);
}
sp currentTrack = mTrackList[trackIndex];
// This pushBuffer will wait until the mediaBuffer is consumed.
return currentTrack->pushBuffer(mediaBuffer);
} 每写一帧时，都会在mediaMuxer.cpp里，调用MediaAdapter的接口，去pushBuffer。这个pushBuffer，将数据push到哪里去了，可以跟到frameworks\av\media\libstagefright\MediaAdapter.cpp里来看看：
void MediaAdapter::signalBufferReturned(MediaBuffer *buffer) {
Mutex::Autolock autoLock(mAdapterLock);
CHECK(buffer != NULL);
buffer->setObserver(0);
buffer->release();
ALOGV("buffer returned %p", buffer);
mBufferReturnedCond.signal();
}
status_t MediaAdapter::read(
MediaBuffer **buffer, const ReadOptions * /* options */) {
Mutex::Autolock autoLock(mAdapterLock);
if (!mStarted) {
ALOGV("Read before even started!");
return ERROR_END_OF_STREAM;
}
while (mCurrentMediaBuffer == NULL && mStarted) {
ALOGV("waiting @ read()");
mBufferReadCond.wait(mAdapterLock);
}
if (!mStarted) {
ALOGV("read interrupted after stop");
CHECK(mCurrentMediaBuffer == NULL);
return ERROR_END_OF_STREAM;
}
CHECK(mCurrentMediaBuffer != NULL);
*buffer = mCurrentMediaBuffer;
mCurrentMediaBuffer = NULL;
(*buffer)->setObserver(this);
return OK;
}
status_t MediaAdapter::pushBuffer(MediaBuffer *buffer) {
if (buffer == NULL) {
ALOGE("pushBuffer get an NULL buffer");
return -EINVAL;
}
Mutex::Autolock autoLock(mAdapterLock);
if (!mStarted) {
ALOGE("pushBuffer called before start");
return INVALID_OPERATION;
}
mCurrentMediaBuffer = buffer;
mBufferReadCond.signal();
ALOGV("wait for the buffer returned @ pushBuffer! %p", buffer);
mBufferReturnedCond.wait(mAdapterLock);
return OK;
}从pushBuffer函数里可以看到，每当mCurrentMediaBuffer = buffer;这样赋值后，就会通过mBufferReadCond.signal();发送信号。这个mBufferReadCond的接收者在read函数里。当read收到消息后，就会将值通过read的指针传送到调用read的地方。调用read的地方是frameworks\av\media\libstagefright\MPEG2TSWriter.cpp里的下面的函数：
void MPEG2TSWriter::SourceInfo::onMessageReceived(const sp &msg) {
switch (msg->what()) {
......
case kWhatRead:
{
MediaBuffer *buffer;
status_t err = mSource->read(&buffer);
if (err != OK && err != INFO_FORMAT_CHANGED) {
sp notify = mNotify->dup();
notify->setInt32("what", kNotifyReachedEOS);
notify->setInt32("status", err);
notify->post();
break;
}
if (err == OK) {
if (mStreamType == 0x0f && mAACCodecSpecificData == NULL) {
// The first audio buffer must contain CSD if not received yet.
CHECK_GE(buffer->range_length(), 2u);
mAACCodecSpecificData = new ABuffer(buffer->range_length());
memcpy(mAACCodecSpecificData->data(),
(const uint8_t *)buffer->data()
+ buffer->range_offset(),
buffer->range_length());
readMore();
} else if (buffer->range_length() > 0) {
if (mStreamType == 0x0f) {
appendAACFrames(buffer);
} else {
appendAVCFrame(buffer);
}
} else {
readMore();
}
buffer->release();
buffer = NULL;
}
// Do not read more data until told to.
break;
}
default:
TRESPASS();
}
} 这里在读到数据后，通过判断是音频的还是视频的，丢给不同的函数去处理。比如是视频的话，就会丢给appendAVCFrame去处理。
void MPEG2TSWriter::SourceInfo::appendAVCFrame(MediaBuffer *buffer) {
sp notify = mNotify->dup();
notify->setInt32("what", kNotifyBuffer);
if (mBuffer == NULL || buffer->range_length() > mBuffer->capacity()) {
mBuffer = new ABuffer(buffer->range_length());
}
mBuffer->setRange(0, 0);
memcpy(mBuffer->data(),
(const uint8_t *)buffer->data()
+ buffer->range_offset(),
buffer->range_length());
int64_t timeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
mBuffer->meta()->setInt64("timeUs", timeUs);
int32_t isSync;
if (buffer->meta_data()->findInt32(kKeyIsSyncFrame, &isSync)
&& isSync != 0) {
mBuffer->meta()->setInt32("isSync", true);
}
mBuffer->setRange(0, buffer->range_length());
notify->setBuffer("buffer", mBuffer);
notify->post();
} 从这个函数里我们可以看到，appendAVCFrame函数，只对数据帧设置时间戳和同步标志后，就通过一个通知，丢给了MPEG2TSWriter::onMessageReceived去处理。MPEG2TSWriter::onMessageReceived收到帧后的处理过程，就是最开始咱们讨论的那个地方了。
另外，如果我们是从指定的mpeg2ts流文件里抽帧，然后再通过mpeg2tswriter去打包成一个新的ts流的话，有一个地方需要注意。那就是MPEG2TSWriter::SourceInfo::appendAACFrames(MediaBuffer *buffer)这个函数里的开始的地方，加个判断：
if(mIsMuxer)
{
buffer->set_range(7, buffer->range_length()-7);
}
因为这里加个属性来判断,当是在muxer时,就要加上下面这一行.因为现有的ts视频,每一帧音频已经加上了
7个字节的音频头.如果不将这7个字节的音频头给去掉,会导致每一帧音频上又多加了一个7字节的音频头.
这样的后果会导致大部份的播放器识别不了这个音频,播放不出了声音.
到此为止，我们就将mpeg2ts的流程梳理完成了，并且修正了录相丢帧的bug，封装了mpeg2ts muxer java层接口。