live555学习2
接着《live555学习1》来写，由于1的后面有的地方已经出现乱码就放在这里继续写了。
Boolean MultiFramedRTPSink::continuePlaying() {
// Send the first packet.
// (This will also schedule any future sends.)
buildAndSendPacket(True);
return True;
}
void MultiFramedRTPSink::buildAndSendPacket(Boolean isFirstPacket) {
fIsFirstPacket = isFirstPacket;
设置RTP头,注意，接收端需要根据RTP包的序号fSeqNo来重新排序
// Set up the RTP header:
unsigned rtpHdr = 0x80000000; // RTP version 2; marker ('M') bit not set (by default; it can be set later)
rtpHdr |= (fRTPPayloadType<<16);
rtpHdr |= fSeqNo; // sequence number
fOutBuf->enqueueWord(rtpHdr);
//保留一个4 bytes空间，用于设置time stamp
// Note where the RTP timestamp will go.
// (We can't fill this in until we start packing payload frames.)
fTimestampPosition = fOutBuf->curPacketSize();
fOutBuf->skipBytes(4); // leave a hole for the timestamp
fOutBuf->enqueueWord(SSRC());//跟RTCP相关，作用暂不清楚
// Allow for a special, payload-format-specific header following the
// RTP header:
fSpecialHeaderPosition = fOutBuf->curPacketSize();
fSpecialHeaderSize = specialHeaderSize();
fOutBuf->skipBytes(fSpecialHeaderSize);//预留空间
// Begin packing as many (complete) frames into the packet as we can:
fTotalFrameSpecificHeaderSizes = 0;
fNoFramesLeft = False;
fNumFramesUsedSoFar = 0;
packFrame();
}
void MultiFramedRTPSink::packFrame() {
// Get the next frame.
//
//首先需要检查buffer中是否还存在溢出的数据(frame)
//
// First, see if we have an overflow frame that was too big for the last pkt
if (fOutBuf->haveOverflowData()) {
// Use this frame before reading a new one from the source
unsigned frameSize = fOutBuf->overflowDataSize();
struct timeval presentationTime = fOutBuf->overflowPresentationTime();
unsigned durationInMicroseconds = fOutBuf->overflowDurationInMicroseconds();
//
//使用溢出的数据作为packet的内容，注意，这里并不一定进行memcopy操作，
//因为可能已经把packet 的开始位置重置到overflow data的位置
//
fOutBuf->useOverflowData();
//
//获取了数据，就可以准备发送了，当然若是数据量太小，将需要获取更多的数据
//
afterGettingFrame1(frameSize, 0, presentationTime, durationInMicroseconds);
} else {
// Normal case: we need to read a new frame from the source
if (fSource == NULL) return;
//这里，给予当前帧预留空间的机会，保存一些特殊信息，当然frameSpecificHeaderSize函数默认返回0
fCurFrameSpecificHeaderPosition = fOutBuf->curPacketSize();
fCurFrameSpecificHeaderSize = frameSpecificHeaderSize();
fOutBuf->skipBytes(fCurFrameSpecificHeaderSize);
fTotalFrameSpecificHeaderSizes += fCurFrameSpecificHeaderSize;
//
//从source中获取数据，然后调用回调函数afterGettingFrame。注意，在C++中类成员函数是不能作为回调用函数的。
//我们可以看到afterGettingFrame中直接调用了afterGettingFrame1函数，与上面的第一次情况处理类似了。不过这里为什么要用回调函数回？
//
fSource->getNextFrame(fOutBuf->curPtr(), fOutBuf->totalBytesAvailable(),
afterGettingFrame, this, ourHandleClosure, this);
}
}
fSource其实是MediaSink中的一个成员，看以看到在H264VideoRTPSink中已经将其new 为H264FUAFragmenter，所以这里的getNextFrame为H264FUAFragmenter中的getNextFrame。继承关系为：H264VideoRTPSink->VideoRTPSink->MultiFramedRTPSink->RTPSink->MediaSink->Medium
而getNextFrame只在FramedSource中有实现，他们的继承关系：H264FUAFragmenter->FramedFilter->FramedSource->MediaSource->Medium
void FramedSource::getNextFrame(unsigned char* to, unsigned maxSize,
afterGettingFunc* afterGettingFunc,
void* afterGettingClientData,
onCloseFunc* onCloseFunc,
void* onCloseClientData) {
// Make sure we're not already being read:
if (fIsCurrentlyAwaitingData) {
envir() << "FramedSource[" << this << "]::getNextFrame(): attempting to read more than once at the same time!\n";
envir().internalError();
}
fTo = to;
fMaxSize = maxSize;
fNumTruncatedBytes = 0; // by default; could be changed by doGetNextFrame()
fDurationInMicroseconds = 0; // by default; could be changed by doGetNextFrame()
fAfterGettingFunc = afterGettingFunc;
fAfterGettingClientData = afterGettingClientData;
fOnCloseFunc = onCloseFunc;
fOnCloseClientData = onCloseClientData;
fIsCurrentlyAwaitingData = True;
doGetNextFrame();
}
这里的doGetNextFrame在FramedSource中为纯虚函数。
void H264FUAFragmenter::doGetNextFrame() {
if (fNumValidDataBytes == 1) {
// We have no NAL unit data currently in the buffer. Read a new one:
fInputSource->getNextFrame(&fInputBuffer[1], fInputBufferSize - 1,
afterGettingFrame, this,
FramedSource::handleClosure, this);
} else {
// We have NAL unit data in the buffer. There are three cases to consider:
// 1. There is a new NAL unit in the buffer, and it's small enough to deliver
// to the RTP sink (as is).
// 2. There is a new NAL unit in the buffer, but it's too large to deliver to
// the RTP sink in its entirety. Deliver the first fragment of this data,
// as a FU-A packet, with one extra preceding header byte.
// 3. There is a NAL unit in the buffer, and we've already delivered some
// fragment(s) of this. Deliver the next fragment of this data,
// as a FU-A packet, with two extra preceding header bytes.
if (fMaxSize < fMaxOutputPacketSize) { // shouldn't happen
envir() << "H264FUAFragmenter::doGetNextFrame(): fMaxSize ("
<< fMaxSize << ") is smaller than expected\n";
} else {
fMaxSize = fMaxOutputPacketSize;
}
fLastFragmentCompletedNALUnit = True; // by default
if (fCurDataOffset == 1) { // case 1 or 2
if (fNumValidDataBytes - 1 <= fMaxSize) { // case 1
memmove(fTo, &fInputBuffer[1], fNumValidDataBytes - 1);
fFrameSize = fNumValidDataBytes - 1;
fCurDataOffset = fNumValidDataBytes;
} else { // case 2
// We need to send the NAL unit data as FU-A packets. Deliver the first
// packet now. Note that we add FU indicator and FU header bytes to the front
// of the packet (reusing the existing NAL header byte for the FU header).
fInputBuffer[0] = (fInputBuffer[1] & 0xE0) | 28; // FU indicator
fInputBuffer[1] = 0x80 | (fInputBuffer[1] & 0x1F); // FU header (with S bit)
memmove(fTo, fInputBuffer, fMaxSize);
fFrameSize = fMaxSize;
fCurDataOffset += fMaxSize - 1;
fLastFragmentCompletedNALUnit = False;
}
} else { // case 3
// We are sending this NAL unit data as FU-A packets. We've already sent the
// first packet (fragment). Now, send the next fragment. Note that we add
// FU indicator and FU header bytes to the front. (We reuse these bytes that
// we already sent for the first fragment, but clear the S bit, and add the E
// bit if this is the last fragment.)
fInputBuffer[fCurDataOffset-2] = fInputBuffer[0]; // FU indicator
fInputBuffer[fCurDataOffset-1] = fInputBuffer[1]&~0x80; // FU header (no S bit)
unsigned numBytesToSend = 2 + fNumValidDataBytes - fCurDataOffset;
if (numBytesToSend > fMaxSize) {
// We can't send all of the remaining data this time:
numBytesToSend = fMaxSize;
fLastFragmentCompletedNALUnit = False;
} else {
// This is the last fragment:
fInputBuffer[fCurDataOffset-1] |= 0x40; // set the E bit in the FU header
fNumTruncatedBytes = fSaveNumTruncatedBytes;
}
memmove(fTo, &fInputBuffer[fCurDataOffset-2], numBytesToSend);
fFrameSize = numBytesToSend;
fCurDataOffset += numBytesToSend - 2;
}
if (fCurDataOffset >= fNumValidDataBytes) {
// We're done with this data. Reset the pointers for receiving new data:
fNumValidDataBytes = fCurDataOffset = 1;
}
// Complete delivery to the client:
FramedSource::afterGetting(this);
}
}
这里的fInputSource是不是我们之前在new H264FUAFragmenter的时候传给的H264VideoStreamFramer，我们看看他的构造函数
H264FUAFragmenter::H264FUAFragmenter(UsageEnvironment& env,
FramedSource* inputSource,
unsigned inputBufferMax,
unsigned maxOutputPacketSize)
: FramedFilter(env, inputSource),
fInputBufferSize(inputBufferMax+1), fMaxOutputPacketSize(maxOutputPacketSize),
fNumValidDataBytes(1), fCurDataOffset(1), fSaveNumTruncatedBytes(0),
fLastFragmentCompletedNALUnit(True) {
fInputBuffer = new unsigned char[fInputBufferSize];
}
在FramedFilter中我们看到
FramedFilter::FramedFilter(UsageEnvironment& env,
FramedSource* inputSource)
: FramedSource(env),
fInputSource(inputSource) {
}
这里果然给fInputSource初始化了。而在new H264FUAFragmenter的时候传给的的那个fSource是OndemandMediaSubsession中createNewStreamSource返回的那个H264VideoStreamFramer。这里个继承关系如下：
H264FUAFragmenter->FramedFilter->FramedSource->MediaSource->Medium
第一次执行时会getNextFrame去获取一张。getNextFrame在FramedSource中定义
void FramedSource::getNextFrame(unsigned char* to, unsigned maxSize,
afterGettingFunc* afterGettingFunc,
void* afterGettingClientData,
onCloseFunc* onCloseFunc,
void* onCloseClientData) {
// Make sure we're not already being read:
if (fIsCurrentlyAwaitingData) {
envir() << "FramedSource[" << this << "]::getNextFrame(): attempting to read more than once at the same time!\n";
envir().internalError();
}
fTo = to;
fMaxSize = maxSize;
fNumTruncatedBytes = 0; // by default; could be changed by doGetNextFrame()
fDurationInMicroseconds = 0; // by default; could be changed by doGetNextFrame()
fAfterGettingFunc = afterGettingFunc;
fAfterGettingClientData = afterGettingClientData;
fOnCloseFunc = onCloseFunc;
fOnCloseClientData = onCloseClientData;
fIsCurrentlyAwaitingData = True;
doGetNextFrame();
}
fInputSource->getNextFrame(&fInputBuffer[1], fInputBufferSize - 1,
afterGettingFrame, this,
FramedSource::handleClosure, this);
fAfterGettingFunc为afterGettingFrame
先来看doGetNextFrame为纯虚函数，所以这里先来看H264VideoStreamFramer的createNew
H264VideoStreamFramer
::H264VideoStreamFramer(UsageEnvironment& env, FramedSource* inputSource, Boolean createParser, Boolean includeStartCodeInOutput)
: MPEGVideoStreamFramer(env, inputSource),
fIncludeStartCodeInOutput(includeStartCodeInOutput),
fLastSeenSPS(NULL), fLastSeenSPSSize(0), fLastSeenPPS(NULL), fLastSeenPPSSize(0) {
fParser = createParser
? new H264VideoStreamParser(this, inputSource, includeStartCodeInOutput)
vim : NULL;
fNextPresentationTime = fPresentationTimeBase;
fFrameRate = 25.0; // We assume a frame rate of 25 fps, unless we learn otherwise (from parsing a Sequence Parameter Set NAL unit)
}
这里顺便提一下，这里传给H264VideoStreamParser的inputSource是H264VideoStreamFramer createNew是传入的我们自己实现的那个source。
再来看一下
H264VideoStreamParser
::H264VideoStreamParser(H264VideoStreamFramer* usingSource, FramedSource* inputSource, Boolean includeStartCodeInOutput)
: MPEGVideoStreamParser(usingSource, inputSource),
fOutputStartCodeSize(includeStartCodeInOutput ? 4 : 0), fHaveSeenFirstStartCode(False), fHaveSeenFirstByteOfNALUnit(False),
// Default values for our parser variables (in case they're not set explicitly in headers that we parse:
log2_max_frame_num(5), separate_colour_plane_flag(False), frame_mbs_only_flag(True) {
}
其中usingSource 表示正在使用这个Parser的Source，而inputSource正是我们自定义的source。这里usingsource传的是H264VideoStreamFramer。
doGetNextFrame在H264VideoStreamFramer的父类MPEGVideoStreamFramer中实现
void MPEGVideoStreamFramer::doGetNextFrame() {
fParser->registerReadInterest(fTo, fMaxSize);
continueReadProcessing();
}
void MPEGVideoStreamFramer::continueReadProcessing() {
unsigned acquiredFrameSize = fParser->parse();
if (acquiredFrameSize > 0) {
// We were able to acquire a frame from the input.
// It has already been copied to the reader's space.
fFrameSize = acquiredFrameSize;
fNumTruncatedBytes = fParser->numTruncatedBytes();
// "fPresentationTime" should have already been computed.
// Compute "fDurationInMicroseconds" now:
fDurationInMicroseconds
= (fFrameRate == 0.0 || ((int)fPictureCount) < 0) ? 0
: (unsigned)((fPictureCount*1000000)/fFrameRate);
#ifdef DEBUG
fprintf(stderr, "%d bytes @%u.%06d, fDurationInMicroseconds: %d ((%d*1000000)/%f)\n", acquiredFrameSize, fPresentationTime.tv_sec, fPresentationTime.tv_usec, fDurationInMicroseconds, fPictureCount, fFrameRate);
#endif
fPictureCount = 0;
// Call our own 'after getting' function. Because we're not a 'leaf'
// source, we can call this directly, without risking infinite recursion.
afterGetting(this);
} else {
// We were unable to parse a complete frame from the input, because:
// - we had to read more data from the source stream, or
// - the source stream has ended.
}
}
首先是parse方法，这里的fParser是在H264VideoStreamFramer构造函数中new出来的H264VideoStreamParser的是同一个。来看看parse方法，内容比较多，我们把源码贴出来
unsigned H264VideoStreamParser::parse() {
try {
// The stream must start with a 0x00000001:
if (!fHaveSeenFirstStartCode) {
// Skip over any input bytes that precede the first 0x00000001:
u_int32_t first4Bytes;
while ((first4Bytes = test4Bytes()) != 0x00000001) {
get1Byte(); setParseState(); // ensures that we progress over bad data
}
skipBytes(4); // skip this initial code
setParseState();
fHaveSeenFirstStartCode = True; // from now on
}
if (fOutputStartCodeSize > 0) {
// Include a start code in the output:
save4Bytes(0x00000001);
}
// Then save everything up until the next 0x00000001 (4 bytes) or 0x000001 (3 bytes), or we hit EOF.
// Also make note of the first byte, because it contains the "nal_unit_type":
if (haveSeenEOF()) {
// We hit EOF the last time that we tried to parse this data, so we know that any remaining unparsed data
// forms a complete NAL unit, and that there's no 'start code' at the end:
unsigned remainingDataSize = totNumValidBytes() - curOffset();
while (remainingDataSize > 0) {
saveByte(get1Byte());
--remainingDataSize;
}
if (!fHaveSeenFirstByteOfNALUnit) {
// There's no remaining NAL unit.
(void)get1Byte(); // forces another read, which will cause EOF to get handled for real this time
return 0;
}
#ifdef DEBUG
fprintf(stderr, "This NAL unit (%d bytes) ends with EOF\n", curFrameSize()-fOutputStartCodeSize);
#endif
} else {
u_int32_t next4Bytes = test4Bytes();
if (!fHaveSeenFirstByteOfNALUnit) {
fFirstByteOfNALUnit = next4Bytes>>24;
fHaveSeenFirstByteOfNALUnit = True;
}
while (next4Bytes != 0x00000001 && (next4Bytes&0xFFFFFF00) != 0x00000100) {
// We save at least some of "next4Bytes".
if ((unsigned)(next4Bytes&0xFF) > 1) {
// Common case: 0x00000001 or 0x000001 definitely doesn't begin anywhere in "next4Bytes", so we save all of it:
save4Bytes(next4Bytes);
skipBytes(4);
} else {
// Save the first byte, and continue testing the rest:
saveByte(next4Bytes>>24);
skipBytes(1);
}
setParseState(); // ensures forward progress
next4Bytes = test4Bytes();
}
// Assert: next4Bytes starts with 0x00000001 or 0x000001, and we've saved all previous bytes (forming a complete NAL unit).
// Skip over these remaining bytes, up until the start of the next NAL unit:
if (next4Bytes == 0x00000001) {
skipBytes(4);
} else {
skipBytes(3);
}
}
u_int8_t nal_ref_idc = (fFirstByteOfNALUnit&0x60)>>5;
u_int8_t nal_unit_type = fFirstByteOfNALUnit&0x1F;
fHaveSeenFirstByteOfNALUnit = False; // for the next NAL unit that we parse
#ifdef DEBUG
fprintf(stderr, "Parsed %d-byte NAL-unit (nal_ref_idc: %d, nal_unit_type: %d (\"%s\"))\n",
curFrameSize()-fOutputStartCodeSize, nal_ref_idc, nal_unit_type, nal_unit_type_description[nal_unit_type]);
#endif
switch (nal_unit_type) {
case 6: { // Supplemental enhancement information (SEI)
analyze_sei_data();
// Later, perhaps adjust "fPresentationTime" if we saw a "pic_timing" SEI payload??? #####
break;
}
case 7: { // Sequence parameter set
// First, save a copy of this NAL unit, in case the downstream object wants to see it:
usingSource()->saveCopyOfSPS(fStartOfFrame + fOutputStartCodeSize, fTo - fStartOfFrame - fOutputStartCodeSize);
// Parse this NAL unit to check whether frame rate information is present:
unsigned num_units_in_tick, time_scale, fixed_frame_rate_flag;
analyze_seq_parameter_set_data(num_units_in_tick, time_scale, fixed_frame_rate_flag);
if (time_scale > 0 && num_units_in_tick > 0) {
usingSource()->fFrameRate = time_scale/(2.0*num_units_in_tick);
#ifdef DEBUG
fprintf(stderr, "Set frame rate to %f fps\n", usingSource()->fFrameRate);
if (fixed_frame_rate_flag == 0) {
fprintf(stderr, "\tWARNING: \"fixed_frame_rate_flag\" was not set\n");
}
#endif
} else {
#ifdef DEBUG
fprintf(stderr, "\tThis \"Sequence Parameter Set\" NAL unit contained no frame rate information, so we use a default frame rate of %f fps\n", usingSource()->fFrameRate);
#endif
}
break;
}
case 8: { // Picture parameter set
// Save a copy of this NAL unit, in case the downstream object wants to see it:
usingSource()->saveCopyOfPPS(fStartOfFrame + fOutputStartCodeSize, fTo - fStartOfFrame - fOutputStartCodeSize);
}
}
usingSource()->setPresentationTime();
#ifdef DEBUG
unsigned long secs = (unsigned long)usingSource()->fPresentationTime.tv_sec;
unsigned uSecs = (unsigned)usingSource()->fPresentationTime.tv_usec;
fprintf(stderr, "\tPresentation time: %lu.%06u\n", secs, uSecs);
#endif
// If this NAL unit is a VCL NAL unit, we also scan the start of the next NAL unit, to determine whether this NAL unit
// ends the current 'access unit'. We need this information to figure out when to increment "fPresentationTime".
// (RTP streamers also need to know this in order to figure out whether or not to set the "M" bit.)
Boolean thisNALUnitEndsAccessUnit = False; // until we learn otherwise
if (haveSeenEOF()) {
// There is no next NAL unit, so we assume that this one ends the current 'access unit':
thisNALUnitEndsAccessUnit = True;
} else {
Boolean const isVCL = nal_unit_type <= 5 && nal_unit_type > 0; // Would need to include type 20 for SVC and MVC #####
if (isVCL) {
u_int32_t first4BytesOfNextNALUnit = test4Bytes();
u_int8_t firstByteOfNextNALUnit = first4BytesOfNextNALUnit>>24;
u_int8_t next_nal_ref_idc = (firstByteOfNextNALUnit&0x60)>>5;
u_int8_t next_nal_unit_type = firstByteOfNextNALUnit&0x1F;
if (next_nal_unit_type >= 6) {
// The next NAL unit is not a VCL; therefore, we assume that this NAL unit ends the current 'access unit':
#ifdef DEBUG
fprintf(stderr, "\t(The next NAL unit is not a VCL)\n");
#endif
thisNALUnitEndsAccessUnit = True;
} else {
// The next NAL unit is also a VCL. We need to examine it a little to figure out if it's a different 'access unit'.
// (We use many of the criteria described in section 7.4.1.2.4 of the H.264 specification.)
Boolean IdrPicFlag = nal_unit_type == 5;
Boolean next_IdrPicFlag = next_nal_unit_type == 5;
if (next_IdrPicFlag != IdrPicFlag) {
// IdrPicFlag differs in value
#ifdef DEBUG
fprintf(stderr, "\t(IdrPicFlag differs in value)\n");
#endif
thisNALUnitEndsAccessUnit = True;
} else if (next_nal_ref_idc != nal_ref_idc && next_nal_ref_idc*nal_ref_idc == 0) {
// nal_ref_idc differs in value with one of the nal_ref_idc values being equal to 0
#ifdef DEBUG
fprintf(stderr, "\t(nal_ref_idc differs in value with one of the nal_ref_idc values being equal to 0)\n");
#endif
thisNALUnitEndsAccessUnit = True;
} else if ((nal_unit_type == 1 || nal_unit_type == 2 || nal_unit_type == 5)
&& (next_nal_unit_type == 1 || next_nal_unit_type == 2 || next_nal_unit_type == 5)) {
// Both this and the next NAL units begin with a "slice_header".
// Parse this (for each), to get parameters that we can compare:
// Current NAL unit's "slice_header":
unsigned frame_num, pic_parameter_set_id, idr_pic_id;
Boolean field_pic_flag, bottom_field_flag;
analyze_slice_header(fStartOfFrame + fOutputStartCodeSize, fTo, nal_unit_type,
frame_num, pic_parameter_set_id, idr_pic_id, field_pic_flag, bottom_field_flag);
// Next NAL unit's "slice_header":
#ifdef DEBUG
fprintf(stderr, " Next NAL unit's slice_header:\n");
#endif
u_int8_t next_slice_header[NUM_NEXT_SLICE_HEADER_BYTES_TO_ANALYZE];
testBytes(next_slice_header, sizeof next_slice_header);
unsigned next_frame_num, next_pic_parameter_set_id, next_idr_pic_id;
Boolean next_field_pic_flag, next_bottom_field_flag;
analyze_slice_header(next_slice_header, &next_slice_header[sizeof next_slice_header], next_nal_unit_type,
next_frame_num, next_pic_parameter_set_id, next_idr_pic_id, next_field_pic_flag, next_bottom_field_flag);
if (next_frame_num != frame_num) {
// frame_num differs in value
#ifdef DEBUG
fprintf(stderr, "\t(frame_num differs in value)\n");
#endif
thisNALUnitEndsAccessUnit = True;
} else if (next_pic_parameter_set_id != pic_parameter_set_id) {
// pic_parameter_set_id differs in value
#ifdef DEBUG
fprintf(stderr, "\t(pic_parameter_set_id differs in value)\n");
#endif
thisNALUnitEndsAccessUnit = True;
} else if (next_field_pic_flag != field_pic_flag) {
// field_pic_flag differs in value
#ifdef DEBUG
fprintf(stderr, "\t(field_pic_flag differs in value)\n");
#endif
thisNALUnitEndsAccessUnit = True;
} else if (next_bottom_field_flag != bottom_field_flag) {
// bottom_field_flag differs in value
#ifdef DEBUG
fprintf(stderr, "\t(bottom_field_flag differs in value)\n");
#endif
thisNALUnitEndsAccessUnit = True;
} else if (next_IdrPicFlag == 1 && next_idr_pic_id != idr_pic_id) {
// IdrPicFlag is equal to 1 for both and idr_pic_id differs in value
// Note: We already know that IdrPicFlag is the same for both.
#ifdef DEBUG
fprintf(stderr, "\t(IdrPicFlag is equal to 1 for both and idr_pic_id differs in value)\n");
#endif
thisNALUnitEndsAccessUnit = True;
}
}
}
}
}
if (thisNALUnitEndsAccessUnit) {
#ifdef DEBUG
fprintf(stderr, "*****This NAL unit ends the current access unit*****\n");
#endif
usingSource()->fPictureEndMarker = True;
++usingSource()->fPictureCount;
// Note that the presentation time for the next NAL unit will be different:
struct timeval& nextPT = usingSource()->fNextPresentationTime; // alias
nextPT = usingSource()->fPresentationTime;
double nextFraction = nextPT.tv_usec/1000000.0 + 1/usingSource()->fFrameRate;
unsigned nextSecsIncrement = (long)nextFraction;
nextPT.tv_sec += (long)nextSecsIncrement;
nextPT.tv_usec = (long)((nextFraction - nextSecsIncrement)*1000000);
}
setParseState();
return curFrameSize();
} catch (int /*e*/) {
#ifdef DEBUG
fprintf(stderr, "H264VideoStreamParser::parse() EXCEPTION (This is normal behavior - *not* an error)\n");
#endif
return 0; // the parsing got interrupted
}
}
这个之后就是afterGetting(this)，
void FramedSource::afterGetting(FramedSource* source) {
source->fIsCurrentlyAwaitingData = False;
// indicates that we can be read again
// Note that this needs to be done here, in case the "fAfterFunc"
// called below tries to read another frame (which it usually will)
if (source->fAfterGettingFunc != NULL) {
(*(source->fAfterGettingFunc))(source->fAfterGettingClientData,
source->fFrameSize, source->fNumTruncatedBytes,
source->fPresentationTime,
source->fDurationInMicroseconds);
}
}
其实是执行传进去的指针的fAfterGettingFunc的函数。我们在前面分析了，H264FUAFragmenter的fAfterGettingFunc为afterGettingFrame
void H264FUAFragmenter::afterGettingFrame(void* clientData, unsigned frameSize,
unsigned numTruncatedBytes,
struct timeval presentationTime,
unsigned durationInMicroseconds) {
H264FUAFragmenter* fragmenter = (H264FUAFragmenter*)clientData;
fragmenter->afterGettingFrame1(frameSize, numTruncatedBytes, presentationTime,
durationInMicroseconds);
}
void H264FUAFragmenter::afterGettingFrame1(unsigned frameSize,
unsigned numTruncatedBytes,
struct timeval presentationTime,
unsigned durationInMicroseconds) {
fNumValidDataBytes += frameSize;
fSaveNumTruncatedBytes = numTruncatedBytes;
fPresentationTime = presentationTime;
fDurationInMicroseconds = durationInMicroseconds;
// Deliver data to the client:
doGetNextFrame();
}
看看这里最后执行的又是doGetNextFrame又是前面的分析过的。可以看到这里是一个循环过程。
我们这里再把整个循环过程再捋捋。
当MultiFrameRTSPSink需要数据的时候，会执行H264FUAFragmenter的getNextFrame，顺带把fOutBuf传给fTo。然后执行H264FUAFragmenter的doGetNextFrame。这时会分析nal包，针对MTU对nal打包。
没有nal包时会调用fInputSource的getNextFrame。这里可能会觉得奇怪，怎么又执行getNextFrame，不过这次是H264VideoStreamFramer来调用的getNextFrame。这里的fInputSource就是H264VideoStreamFramer。这里把fInputBuffer传给fTo。
但是getNextFrame中成员函数，他不是虚函数。
最后都会调用doGetNextFrame。
doGetNextFrame在H264VideoStreamFramer父类MPEGVideoStreamFramer中实现。
doGetNextFrame调用continueReadProcessing，然后会先条用H264VideoStreamParser中的parse函数。
parse分析完nal后，会调用afterGeting(this)。其实是调用H264FUAFragmenter的fAfterGettingFunc为afterGettingFrame
afterGettingFrame其实又是调用H264FUAFragmenter的doGetNextFrame。
也就是说rtspSink需要发送包会让H264FUAFragmenter对nal进行rtp打包，而H264FUAFragmenter会跟H264VideoStreamFramer要数据，而H264VideoStreamFramer会让H264VideoStreamParser去获取数据并分析出nal。完事后就又调用H264FUAFragmenter的对nal进行rtp打包。
但是如何获取数据和分析nal包，这里还是不清楚，这个工作主要在parse中。我来看parse中首先执行的第一段代码
if (!fHaveSeenFirstStartCode) {
// Skip over any input bytes that precede the first 0x00000001:
u_int32_t first4Bytes;
while ((first4Bytes = test4Bytes()) != 0x00000001) {
get1Byte(); setParseState(); // ensures that we progress over bad data
}
skipBytes(4); // skip this initial code
setParseState();
fHaveSeenFirstStartCode = True; // from now on
}
这里先执行的是test4Bytes()看他是否等于0x00000001这个是判断nal开始的标志。nal开始的标志是0x000001和0x00000001。其他数据中间有这两个的在中间插入0x03。所以这里应该是判断nal的起点。再来看看test4Bytes
u_int32_t test4Bytes() { // as above, but doesn't advance ptr
ensureValidBytes(4);
unsigned char const* ptr = nextToParse();
return (ptr[0]<<24)|(ptr[1]<<16)|(ptr[2]<<8)|ptr[3];
}
再看看ensureValidBytes
void ensureValidBytes(unsigned numBytesNeeded) {
// common case: inlined:
if (fCurParserIndex + numBytesNeeded <= fTotNumValidBytes) return;
ensureValidBytes1(numBytesNeeded);
}
第一条件判断肯定不会return，fCurParserIndex在StreamParser中fCurParserIndex(0)为0，fTotNumValidBytes(0)也初始化为0.
void StreamParser::ensureValidBytes1(unsigned numBytesNeeded) {
// We need to read some more bytes from the input source.
// First, clarify how much data to ask for:
unsigned maxInputFrameSize = fInputSource->maxFrameSize();//这里的fInputSource是我们自己实现的那个FrameSource子类，在这个子类中我们重新实现了maxFrameSize()方法，我们返回的是一个比较大的数。
if (maxInputFrameSize > numBytesNeeded) numBytesNeeded = maxInputFrameSize;//这里numBytesNeeded是这个比较大的数。
// First, check whether these new bytes would overflow the current
// bank. If so, start using a new bank now.
if (fCurParserIndex + numBytesNeeded > BANK_SIZE) {
// Swap banks, but save any still-needed bytes from the old bank:
unsigned numBytesToSave = fTotNumValidBytes - fSavedParserIndex;
unsigned char const* from = &curBank()[fSavedParserIndex];
fCurBankNum = (fCurBankNum + 1)%2;
fCurBank = fBank[fCurBankNum];
memmove(curBank(), from, numBytesToSave);
fCurParserIndex = fCurParserIndex - fSavedParserIndex;
fSavedParserIndex = 0;
fTotNumValidBytes = numBytesToSave;
}
// ASSERT: fCurParserIndex + numBytesNeeded > fTotNumValidBytes
// && fCurParserIndex + numBytesNeeded <= BANK_SIZE
if (fCurParserIndex + numBytesNeeded > BANK_SIZE) {
// If this happens, it means that we have too much saved parser state.
// To fix this, increase BANK_SIZE as appropriate.
fInputSource->envir() << "StreamParser internal error ("
<< fCurParserIndex << " + "
<< numBytesNeeded << " > "
<< BANK_SIZE << ")\n";
fInputSource->envir().internalError();
}
// Try to read as many new bytes as will fit in the current bank:
unsigned maxNumBytesToRead = BANK_SIZE - fTotNumValidBytes;//BANK_SIZE默认是150000，
fInputSource->getNextFrame(&curBank()[fTotNumValidBytes],
maxNumBytesToRead,
afterGettingBytes, this,
onInputClosure, this);
throw NO_MORE_BUFFERED_INPUT;
}
getNextFrame做了一些参数的传递，为了清楚罗列如下：
fTo = &curBank()[fTotNumValidBytes]
fNumTruncatedBytes = 0; // by default; could be changed by doGetNextFrame()
fDurationInMicroseconds = 0; // by default; could be changed by doGetNextFrame()
fMaxSize = maxNumBytesToRead ==200000;
fAfterGettingFunc = afterGettingBytes
fAfterGettingClientData = StreamParser
fOnCloseFunc = onInputClosure
fOnCloseClientData = StreamParser
fIsCurrentlyAwaitingData=true
然后执行的是我们自己实现的doGetNextFrame，然后就是采集视频，接着执行的是afterGetting(this)。
根据FrameSource中对afterGetting的定义其实执行的是afterGettingBytes(StreamParser *,fFrameSize,fNumTruncatedBytes=0,source->fPresentationTime,fDurationInMicroseconds = 0);其中的参数是对应实际的值。
void StreamParser::afterGettingBytes(void* clientData,
unsigned numBytesRead,
unsigned /*numTruncatedBytes*/,
struct timeval presentationTime,
unsigned /*durationInMicroseconds*/){
StreamParser* parser = (StreamParser*)clientData;
if (parser != NULL) parser->afterGettingBytes1(numBytesRead, presentationTime);
}
void StreamParser::afterGettingBytes1(unsigned numBytesRead, struct timeval presentationTime) {
// Sanity check: Make sure we didn't get too many bytes for our bank:
if (fTotNumValidBytes + numBytesRead > BANK_SIZE) {
fInputSource->envir()
<< "StreamParser::afterGettingBytes() warning: read "
<< numBytesRead << " bytes; expected no more than "
<< BANK_SIZE - fTotNumValidBytes << "\n";
}
fLastSeenPresentationTime = presentationTime;
unsigned char* ptr = &curBank()[fTotNumValidBytes];
fTotNumValidBytes += numBytesRead;
// Continue our original calling source where it left off:
restoreSavedParserState();
// Sigh... this is a crock; things would have been a lot simpler
// here if we were using threads, with synchronous I/O...
fClientContinueFunc(fClientContinueClientData, ptr, numBytesRead, presentationTime);
}
现在的fTotNumValidBytes还是0，ptr执行curBank的 起始地址。
fTotNumValidBytes是加上采集的帧的大小。
void StreamParser::restoreSavedParserState() {
fCurParserIndex = fSavedParserIndex;
fRemainingUnparsedBits = fSavedRemainingUnparsedBits;
}
主要是做一些赋值，目前看不出来作用，接着看
这个fClientContinueFunc其实是StreamParser初始化的时候传入的，
MPEGVideoStreamParser
::MPEGVideoStreamParser(MPEGVideoStreamFramer* usingSource,
FramedSource* inputSource)
: StreamParser(inputSource, FramedSource::handleClosure, usingSource,
&MPEGVideoStreamFramer::continueReadProcessing, usingSource),
fUsingSource(usingSource) {
}
其实就是MPEGVideoStreamFramer::continueReadProcessing，而其中的fClientContinueClientData就是usingSource，就是H264VideoSteamFramer。
void MPEGVideoStreamFramer
::continueReadProcessing(void* clientData,
unsigned char* /*ptr*/, unsigned /*size*/,
struct timeval /*presentationTime*/) {
MPEGVideoStreamFramer* framer = (MPEGVideoStreamFramer*)clientData;
framer->continueReadProcessing();
}
void MPEGVideoStreamFramer::continueReadProcessing() {
unsigned acquiredFrameSize = fParser->parse();
我们看到这里又开始执行parse了，没办法这里是嵌套调用，我们接着往下看。
执行test4Bytes，ensureValidBytes，看看ensureValidBytes中的这句
if (fCurParserIndex + numBytesNeeded <= fTotNumValidBytes) return;
fTotNumValidBytes在上面已经加了一帧的大小，所以这里是return
那么继续执行test4Bytes中的
unsigned char const* ptr = nextToParse();
return (ptr[0]<<24)|(ptr[1]<<16)|(ptr[2]<<8)|ptr[3];
unsigned char* nextToParse() { return &curBank()[fCurParserIndex]; }
ptr就是curBank中的首地址，而curBank中已经有了我们刚才采集的一帧数据。
(ptr[0]<<24)|(ptr[1]<<16)|(ptr[2]<<8)|ptr[3]即使curBank中四个字节。一帧的开始数据一般0x00000001
这样while循环就不用进入，接着看skipBytes(4);
void skipBytes(unsigned numBytes) {
ensureValidBytes(numBytes);
fCurParserIndex += numBytes;
}
fCurParserIndex就是有0变为4
void setParseState() {
fSavedTo = fTo;
fSavedNumTruncatedBytes = fNumTruncatedBytes;
saveParserState();
}
void StreamParser::saveParserState() {
fSavedParserIndex = fCurParserIndex;
fSavedRemainingUnparsedBits = fRemainingUnparsedBits;
}
接着看，
if (fOutputStartCodeSize > 0) {
// Include a start code in the output:
save4Bytes(0x00000001);
}
fOutputStartCodeSize是在H264VideoStreamParser初始化的时候赋值的，
fOutputStartCodeSize(includeStartCodeInOutput ? 4 : 0),
是在H264VideoStreamFreamer初始化时候传入的，为false
因此if里面的也不会执行，接着看
if (haveSeenEOF()) {
......
}else{
.....
}
Boolean haveSeenEOF() const { return fHaveSeenEOF; }
fHaveSeenEOF(False)在streamParser初始化的时候初始化为false。
所以执行的是else中的部分，接着看
u_int32_t next4Bytes = test4Bytes();
if (!fHaveSeenFirstByteOfNALUnit) {
fFirstByteOfNALUnit = next4Bytes>>24;
fHaveSeenFirstByteOfNALUnit = True;
}
这里的test4Bytes的下一个4个字节的内容赋值给next4Bytes。fHaveSeenFirstByteOfNALUnit(False),在H264VideoStreamParser初始化的时候就初始化了。
我们再看看这个get1Byte()
u_int8_t get1Byte() { // byte-aligned
ensureValidBytes(1);
fRemainingUnparsedBits = 0;
return curBank()[fCurParserIndex++];
}
再来看看ensurceValidBytes，
void ensureValidBytes(unsigned numBytesNeeded) {
// common case: inlined:
if (fCurParserIndex + numBytesNeeded <= fTotNumValidBytes) return;
ensureValidBytes1(numBytesNeeded);
}
void StreamParser::ensureValidBytes1(unsigned numBytesNeeded) {
// We need to read some more bytes from the input source.
// First, clarify how much data to ask for:
unsigned maxInputFrameSize = fInputSource->maxFrameSize();
if (maxInputFrameSize > numBytesNeeded) numBytesNeeded = maxInputFrameSize;
// First, check whether these new bytes would overflow the current
// bank. If so, start using a new bank now.
if (fCurParserIndex + numBytesNeeded > BANK_SIZE) {
// Swap banks, but save any still-needed bytes from the old bank:
unsigned numBytesToSave = fTotNumValidBytes - fSavedParserIndex;
unsigned char const* from = &curBank()[fSavedParserIndex];
fCurBankNum = (fCurBankNum + 1)%2;
fCurBank = fBank[fCurBankNum];
memmove(curBank(), from, numBytesToSave);
fCurParserIndex = fCurParserIndex - fSavedParserIndex;
fSavedParserIndex = 0;
fTotNumValidBytes = numBytesToSave;
}
// ASSERT: fCurParserIndex + numBytesNeeded > fTotNumValidBytes
// && fCurParserIndex + numBytesNeeded <= BANK_SIZE
if (fCurParserIndex + numBytesNeeded > BANK_SIZE) {
// If this happens, it means that we have too much saved parser state.
// To fix this, increase BANK_SIZE as appropriate.
fInputSource->envir() << "StreamParser internal error ("
<< fCurParserIndex << " + "
<< numBytesNeeded << " > "
<< BANK_SIZE << ")\n";
fInputSource->envir().internalError();
}
// Try to read as many new bytes as will fit in the current bank:
unsigned maxNumBytesToRead = BANK_SIZE - fTotNumValidBytes;
fInputSource->getNextFrame(&curBank()[fTotNumValidBytes],
maxNumBytesToRead,
afterGettingBytes, this,
onInputClosure, this);
throw NO_MORE_BUFFERED_INPUT;
}
我们主要看看这里的fInputSource是什么，这个是StreamParser中的FramedSource基类指针。经过分析，这个fInputSource是我们自己是实现的继承自FrameSource的类。
我们在new H264VideoStreamFramer把这个自己实现的FrameSource传入，通过初始化，StreamParser的fInputSource就是我们自己实现的这个类。
他们的继承关系如下：H264VideoStreamFramer在构造函数中new 出H264VideoStreamParser，H264VideoStreamParser->MPEGVideoStreamParser->StreamParser
getNextFrame就会执行doGetNextFrame，但是这个doGetNextFrame是我们自己实现的，即获取camera中采集的一帧数据。
采集完之后调用afterGettingBytes
void StreamParser::afterGettingBytes(void* clientData,
unsigned numBytesRead,
unsigned /*numTruncatedBytes*/,
struct timeval presentationTime,
unsigned /*durationInMicroseconds*/){
StreamParser* parser = (StreamParser*)clientData;
if (parser != NULL) parser->afterGettingBytes1(numBytesRead, presentationTime);
}
那么根据getNextFrame的定义这里的clientData为streamParser，numBytesRead为soucrce->fFrameSize,在我们的程序中我们是每采集到一帧后就把这帧的大小赋值给fFrameSize。presentationTime为source的fPresentationTime，目前还不知道这个time的原理，不过应该是时间戳。那么传给afterGettingBytes1的就是这个time和获取到的一帧的大小。
void StreamParser::afterGettingBytes1(unsigned numBytesRead, struct timeval presentationTime) {
// Sanity check: Make sure we didn't get too many bytes for our bank:
if (fTotNumValidBytes + numBytesRead > BANK_SIZE) {
fInputSource->envir()
<< "StreamParser::afterGettingBytes() warning: read "
<< numBytesRead << " bytes; expected no more than "
<< BANK_SIZE - fTotNumValidBytes << "\n";
}
fLastSeenPresentationTime = presentationTime;
unsigned char* ptr = &curBank()[fTotNumValidBytes];
fTotNumValidBytes += numBytesRead;
// Continue our original calling source where it left off:
restoreSavedParserState();
// Sigh... this is a crock; things would have been a lot simpler
// here if we were using threads, with synchronous I/O...
fClientContinueFunc(fClientContinueClientData, ptr, numBytesRead, presentationTime);
}
这里的fClientContinueFunc是StreamParser初始化的时候传入的，
MPEGVideoStreamParser
::MPEGVideoStreamParser(MPEGVideoStreamFramer* usingSource,
FramedSource* inputSource)
: StreamParser(inputSource, FramedSource::handleClosure, usingSource,
&MPEGVideoStreamFramer::continueReadProcessing, usingSource),
fUsingSource(usingSource) {
}
可以看到为MPEGVideoStreamFramer::continueReadProcessing
void MPEGVideoStreamFramer
::continueReadProcessing(void* clientData,
unsigned char* /*ptr*/, unsigned /*size*/,
struct timeval /*presentationTime*/) {
MPEGVideoStreamFramer* framer = (MPEGVideoStreamFramer*)clientData;
framer->continueReadProcessing();
}
这里的clientData为usingSource。
在前面看到MPEGVideoStreamFramer::continueReadProcessing一开始执行的为unsigned acquiredFrameSize = fParser->parse();
发现这里是parse()中嵌套parse()啊。第二次执行parse()的时候ensureValidBytes1()的时候，从前面源代码中可以看到会抛出异常。
在StreamParser::ensureValidBytes1中最后是
fInputSource->getNextFrame(&curBank()[fTotNumValidBytes], maxNumBytesToRead, afterGettingBytes, this, onInputClosure, this); throw NO_MORE_BUFFERED_INPUT;
而Parser()中写了try{}catch{}.catch{}中的代码
} catch (int /*e*/) {
#ifdef DEBUG
fprintf(stderr, "H264VideoStreamParser::parse() EXCEPTION (This is normal behavior - *not* an error)\n");
#endif
return 0; // the parsing got interrupted
}
由于抛出异常，返回0.在MPEGVideoStreamFramer::continueReadProcessing中的有这样代码：
void MPEGVideoStreamFramer::continueReadProcessing() {
unsigned acquiredFrameSize = fParser->parse();
if (acquiredFrameSize > 0) {
// We were able to acquire a frame from the input.
// It has already been copied to the reader's space.
fFrameSize = acquiredFrameSize;
fNumTruncatedBytes = fParser->numTruncatedBytes();
// "fPresentationTime" should have already been computed.
// Compute "fDurationInMicroseconds" now:
fDurationInMicroseconds
= (fFrameRate == 0.0 || ((int)fPictureCount) < 0) ? 0
: (unsigned)((fPictureCount*1000000)/fFrameRate);
#ifdef DEBUG
fprintf(stderr, "%d bytes @%u.%06d, fDurationInMicroseconds: %d ((%d*1000000)/%f)\n", acquiredFrameSize, fPresentationTime.tv_sec, fPresentationTime.tv_usec, fDurationInMicroseconds, fPictureCount, fFrameRate);
#endif
fPictureCount = 0;
// Call our own 'after getting' function. Because we're not a 'leaf'
// source, we can call this directly, without risking infinite recursion.
afterGetting(this);
} else {
// We were unable to parse a complete frame from the input, because:
// - we had to read more data from the source stream, or
// - the source stream has ended.
}
}
这里的else中的代码执行，但什么也没做。
但实际上对NALU分析和处里在这次Parser()的调用中已经完成了,不是在它本身完成的,而是在它引起了parser()的嵌套调用中完成.理顺一下这个关系
借鉴前人的一篇博客中的话：
sink要获取数据,执行到MPEGVideoStreamFramer::continueReadProcessing(),MPEGVideoStreamFramer::continueReadProcessing调用parser(),parser()要使用数据时发现没有,于是ensureValidBytes1()被调用来从ByteStreamFileSource获取数据,取得数据后MPEGVideoStreamFramer::afterGettingBytes()被调用,并中转到MPEGVideoStreamFramer::continueReadProcessing(),MPEGVideoStreamFramer::continueReadProcessing()被嵌套调用!,MPEGVideoStreamFramer::continueReadProcessing()中又会调用parser(),此时parser()要使用数据时发现有数据了,所以就进行分析,分析出一个NALU后,返回到MPEGVideoStreamFramer::continueReadProcessing(),MPEGVideoStreamFramer::continueReadProcessing()会调用afterGetting(this)把数据返回给sink.sink处理完数据后返回到MPEGVideoStreamFramer::continueReadProcessing(),MPEGVideoStreamFramer::continueReadProcessing()再返回到ensureValidBytes1(),ensureValidBytes1()抛出异常返回到第一次被调用的parser()的catch{}中,parser()返回到第一次调用的MPEGVideoStreamFramer::continueReadProcessing()中,MPEGVideoStreamFramer::continueReadProcessing()发现parser()没有取得NALU,于是啥也不做,返回到sink中,sink会继续通过source->getNextFrame()->MPEGVideoStreamFramer::continueReadProcessing()...这样再次获取NALU.
这就是整个取得数据并发送的过程。