live555实现ffmpeg解码H264的rtsp流

由于需要实现一个解码H264的rtsp流的web客户端。我首先想到的是live555+ffmpeg。live555用于接收rtsp流，ffmpeg用于解码H264用于显示。看了一下live555发现里面的例子里只有一个openrtsp的例子有点想象，但是那个只是接收rtsp流存在一个文件中。我先尝试写了一个ffmpeg解码H264文件的程序，调试通过。现在只要把live555的例子改一下就可以了，把两个程序联合起来就可以了。这里主要的关键点是找到openrtsp写入文件的地方，只需将这个地方的数据获取到解码显示就可以了。

由于项目忙，也只能抽出时间来记录一下。

main函数在playCommon.cpp。main()的流程比较简单，跟服务端差别不大：建立任务计划对象－－建立环境对象－－处理用户输入的参数(RTSP地址)－－创建RTSPClient实例－－发出第一个RTSP请求（可能是OPTIONS也可能是DESCRIBE）－－进入Loop。

我们主要来看看创建RTPSource在函数createSourceObjects()中，看一下：

 Boolean MediaSubsession::createSourceObjects(int useSpecialRTPoffset) {
   do {
     // First, check "fProtocolName"
     if (strcmp(fProtocolName, "UDP") == 0) {
       // A UDP-packetized stream (*not* a RTP stream)
       fReadSource = BasicUDPSource::createNew(env(), fRTPSocket);
       fRTPSource = NULL; // Note!

       if (strcmp(fCodecName, "MP2T") == 0) { // MPEG-2 Transport Stream
     fReadSource = MPEG2TransportStreamFramer::createNew(env(), fReadSource);
     // this sets "durationInMicroseconds" correctly, based on the PCR values
       }
     } else {
       // Check "fCodecName" against the set of codecs that we support,
       // and create our RTP source accordingly
       // (Later make this code more efficient, as this set grows #####)
       // (Also, add more fmts that can be implemented by SimpleRTPSource#####)
       Boolean createSimpleRTPSource = False; // by default; can be changed below
       Boolean doNormalMBitRule = False; // default behavior if "createSimpleRTPSource" is True
       if (strcmp(fCodecName, "QCELP") == 0) { // QCELP audio
     fReadSource =
       QCELPAudioRTPSource::createNew(env(), fRTPSocket, fRTPSource,
                      fRTPPayloadFormat,
                      fRTPTimestampFrequency);
     // Note that fReadSource will differ from fRTPSource in this case
       } else if (strcmp(fCodecName, "AMR") == 0) { // AMR audio (narrowband)
     fReadSource =
       AMRAudioRTPSource::createNew(env(), fRTPSocket, fRTPSource,
                        fRTPPayloadFormat, 0 /*isWideband*/,
                        fNumChannels, fOctetalign, fInterleaving,
                        fRobustsorting, fCRC);
     // Note that fReadSource will differ from fRTPSource in this case
       } else if (strcmp(fCodecName, "AMR-WB") == 0) { // AMR audio (wideband)
     fReadSource =
       AMRAudioRTPSource::createNew(env(), fRTPSocket, fRTPSource,
                        fRTPPayloadFormat, 1 /*isWideband*/,
                        fNumChannels, fOctetalign, fInterleaving,
                        fRobustsorting, fCRC);
     // Note that fReadSource will differ from fRTPSource in this case
       } else if (strcmp(fCodecName, "MPA") == 0) { // MPEG-1 or 2 audio
     fReadSource = fRTPSource
       = MPEG1or2AudioRTPSource::createNew(env(), fRTPSocket,
                           fRTPPayloadFormat,
                           fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "MPA-ROBUST") == 0) { // robust MP3 audio
     fReadSource = fRTPSource
       = MP3ADURTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,
                        fRTPTimestampFrequency);
     if (fRTPSource == NULL) break;

     if (!fReceiveRawMP3ADUs) {
       // Add a filter that deinterleaves the ADUs after depacketizing them:
       MP3ADUdeinterleaver* deinterleaver
         = MP3ADUdeinterleaver::createNew(env(), fRTPSource);
       if (deinterleaver == NULL) break;

       // Add another filter that converts these ADUs to MP3 frames:
       fReadSource = MP3FromADUSource::createNew(env(), deinterleaver);
     }
       } else if (strcmp(fCodecName, "X-MP3-DRAFT-00") == 0) {
     // a non-standard variant of "MPA-ROBUST" used by RealNetworks
     // (one 'ADU'ized MP3 frame per packet; no headers)
     fRTPSource
       = SimpleRTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,
                        fRTPTimestampFrequency,
                        "audio/MPA-ROBUST" /*hack*/);
     if (fRTPSource == NULL) break;

     // Add a filter that converts these ADUs to MP3 frames:
     fReadSource = MP3FromADUSource::createNew(env(), fRTPSource,
                           False /*no ADU header*/);
       } else if (strcmp(fCodecName, "MP4A-LATM") == 0) { // MPEG-4 LATM audio
     fReadSource = fRTPSource
       = MPEG4LATMAudioRTPSource::createNew(env(), fRTPSocket,
                            fRTPPayloadFormat,
                            fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "VORBIS") == 0) { // Vorbis audio
     fReadSource = fRTPSource
       = VorbisAudioRTPSource::createNew(env(), fRTPSocket,
                         fRTPPayloadFormat,
                         fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "VP8") == 0) { // VP8 video
     fReadSource = fRTPSource
       = VP8VideoRTPSource::createNew(env(), fRTPSocket,
                      fRTPPayloadFormat,
                      fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "AC3") == 0 || strcmp(fCodecName, "EAC3") == 0) { // AC3 audio
     fReadSource = fRTPSource
       = AC3AudioRTPSource::createNew(env(), fRTPSocket,
                      fRTPPayloadFormat,
                      fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "MP4V-ES") == 0) { // MPEG-4 Elementary Stream video
     fReadSource = fRTPSource
       = MPEG4ESVideoRTPSource::createNew(env(), fRTPSocket,
                          fRTPPayloadFormat,
                          fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "MPEG4-GENERIC") == 0) {
     fReadSource = fRTPSource
       = MPEG4GenericRTPSource::createNew(env(), fRTPSocket,
                          fRTPPayloadFormat,
                          fRTPTimestampFrequency,
                          fMediumName, fMode,
                          fSizelength, fIndexlength,
                          fIndexdeltalength);
       } else if (strcmp(fCodecName, "MPV") == 0) { // MPEG-1 or 2 video
     fReadSource = fRTPSource
       = MPEG1or2VideoRTPSource::createNew(env(), fRTPSocket,
                           fRTPPayloadFormat,
                           fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "MP2T") == 0) { // MPEG-2 Transport Stream
     fRTPSource = SimpleRTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,
                         fRTPTimestampFrequency, "video/MP2T",
                         0, False);
     fReadSource = MPEG2TransportStreamFramer::createNew(env(), fRTPSource);
     // this sets "durationInMicroseconds" correctly, based on the PCR values
       } else if (strcmp(fCodecName, "H261") == 0) { // H.261
     fReadSource = fRTPSource
       = H261VideoRTPSource::createNew(env(), fRTPSocket,
                       fRTPPayloadFormat,
                       fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "H263-1998") == 0 ||
          strcmp(fCodecName, "H263-2000") == 0) { // H.263+
     fReadSource = fRTPSource
       = H263plusVideoRTPSource::createNew(env(), fRTPSocket,
                           fRTPPayloadFormat,
                           fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "H264") == 0) {
     fReadSource = fRTPSource
       = H264VideoRTPSource::createNew(env(), fRTPSocket,
                       fRTPPayloadFormat,
                       fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "DV") == 0) {
     fReadSource = fRTPSource
       = DVVideoRTPSource::createNew(env(), fRTPSocket,
                     fRTPPayloadFormat,
                     fRTPTimestampFrequency);
       } else if (strcmp(fCodecName, "JPEG") == 0) { // motion JPEG
     fReadSource = fRTPSource
       = JPEGVideoRTPSource::createNew(env(), fRTPSocket,
                       fRTPPayloadFormat,
                       fRTPTimestampFrequency,
                       videoWidth(),
                       videoHeight());
       } else if (strcmp(fCodecName, "X-QT") == 0
          || strcmp(fCodecName, "X-QUICKTIME") == 0) {
     // Generic QuickTime streams, as defined in
     // <http://developer.apple.com/quicktime/icefloe/dispatch026.html>
     char* mimeType
       = new char[strlen(mediumName()) + strlen(codecName()) + 2] ;
     sprintf(mimeType, "%s/%s", mediumName(), codecName());
     fReadSource = fRTPSource
       = QuickTimeGenericRTPSource::createNew(env(), fRTPSocket,
                          fRTPPayloadFormat,
                          fRTPTimestampFrequency,
                          mimeType);
     delete[] mimeType;
       } else if (  strcmp(fCodecName, "PCMU") == 0 // PCM u-law audio
            || strcmp(fCodecName, "GSM") == 0 // GSM audio
            || strcmp(fCodecName, "DVI4") == 0 // DVI4 (IMA ADPCM) audio
            || strcmp(fCodecName, "PCMA") == 0 // PCM a-law audio
            || strcmp(fCodecName, "MP1S") == 0 // MPEG-1 System Stream
            || strcmp(fCodecName, "MP2P") == 0 // MPEG-2 Program Stream
            || strcmp(fCodecName, "L8") == 0 // 8-bit linear audio
            || strcmp(fCodecName, "L16") == 0 // 16-bit linear audio
            || strcmp(fCodecName, "L20") == 0 // 20-bit linear audio (RFC 3190)
            || strcmp(fCodecName, "L24") == 0 // 24-bit linear audio (RFC 3190)
            || strcmp(fCodecName, "G726-16") == 0 // G.726, 16 kbps
            || strcmp(fCodecName, "G726-24") == 0 // G.726, 24 kbps
            || strcmp(fCodecName, "G726-32") == 0 // G.726, 32 kbps
            || strcmp(fCodecName, "G726-40") == 0 // G.726, 40 kbps
            || strcmp(fCodecName, "SPEEX") == 0 // SPEEX audio
            || strcmp(fCodecName, "T140") == 0 // T.140 text (RFC 4103)
            || strcmp(fCodecName, "DAT12") == 0 // 12-bit nonlinear audio (RFC 3190)
            ) {
     createSimpleRTPSource = True;
     useSpecialRTPoffset = 0;
       } else if (useSpecialRTPoffset >= 0) {
     // We don't know this RTP payload format, but try to receive
     // it using a 'SimpleRTPSource' with the specified header offset:
     createSimpleRTPSource = True;
       } else {
     env().setResultMsg("RTP payload format unknown or not supported");
     break;
       }

       if (createSimpleRTPSource) {
     char* mimeType
       = new char[strlen(mediumName()) + strlen(codecName()) + 2] ;
     sprintf(mimeType, "%s/%s", mediumName(), codecName());
     fReadSource = fRTPSource
       = SimpleRTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,
                        fRTPTimestampFrequency, mimeType,
                        (unsigned)useSpecialRTPoffset,
                        doNormalMBitRule);
     delete[] mimeType;
       }
     }

     return True;
   } while (0);

   return False; // an error occurred
 }

可以看到这里对于h264是

 fReadSource = fRTPSource
       = H264VideoRTPSource::createNew(env(), fRTPSocket,
                       fRTPPayloadFormat,
                       fRTPTimestampFrequency);

在MediaSubsession中把fReadSource和fRTPSource初始化了。

socket建立了，Source也创建了，下一步应该是连接Sink，形成一个流。到此为止还未看到Sink的影子，应该是在下一步SETUP中建立，我们看到在continueAfterDESCRIBE()的最后调用了setupStreams（），那么就来探索一下setupStreams():

 void setupStreams() {
   static MediaSubsessionIterator* setupIter = NULL;
   if (setupIter == NULL) setupIter = new MediaSubsessionIterator(*session);
   while ((subsession = setupIter->next()) != NULL) {
     // We have another subsession left to set up:
     if (subsession->clientPortNum() == 0) continue; // port # was not set

     setupSubsession(subsession, streamUsingTCP, continueAfterSETUP);
     return;
   }

   // We're done setting up subsessions.
   delete setupIter;
   if (!madeProgress) shutdown();

   // Create output files:
   if (createReceivers) {
     if (outputQuickTimeFile) {
       // Create a "QuickTimeFileSink", to write to 'stdout':
       qtOut = QuickTimeFileSink::createNew(*env, *session, "stdout",
                        fileSinkBufferSize,
                        movieWidth, movieHeight,
                        movieFPS,
                        packetLossCompensate,
                        syncStreams,
                        generateHintTracks,
                        generateMP4Format);
       if (qtOut == NULL) {
     *env << "Failed to create QuickTime file sink for stdout: " << env->getResultMsg();
     shutdown();
       }

       qtOut->startPlaying(sessionAfterPlaying, NULL);
     } else if (outputAVIFile) {
       // Create an "AVIFileSink", to write to 'stdout':
       aviOut = AVIFileSink::createNew(*env, *session, "stdout",
                       fileSinkBufferSize,
                       movieWidth, movieHeight,
                       movieFPS,
                       packetLossCompensate);
       if (aviOut == NULL) {
     *env << "Failed to create AVI file sink for stdout: " << env->getResultMsg();
     shutdown();
       }

       aviOut->startPlaying(sessionAfterPlaying, NULL);
     } else {
       // Create and start "FileSink"s for each subsession:
       madeProgress = False;
       MediaSubsessionIterator iter(*session);
       while ((subsession = iter.next()) != NULL) {
     if (subsession->readSource() == NULL) continue; // was not initiated

     // Create an output file for each desired stream:
     char outFileName[1000];
     if (singleMedium == NULL) {
       // Output file name is
       //     "<filename-prefix><medium_name>-<codec_name>-<counter>"
       static unsigned streamCounter = 0;
       snprintf(outFileName, sizeof outFileName, "%s%s-%s-%d",
            fileNamePrefix, subsession->mediumName(),
            subsession->codecName(), ++streamCounter);
     } else {
       sprintf(outFileName, "stdout");
     }
     FileSink* fileSink;
     if (strcmp(subsession->mediumName(), "audio") == 0 &&
         (strcmp(subsession->codecName(), "AMR") == 0 ||
          strcmp(subsession->codecName(), "AMR-WB") == 0)) {
       // For AMR audio streams, we use a special sink that inserts AMR frame hdrs:
       fileSink = AMRAudioFileSink::createNew(*env, outFileName,
                          fileSinkBufferSize, oneFilePerFrame);
     } else if (strcmp(subsession->mediumName(), "video") == 0 &&
         (strcmp(subsession->codecName(), "H264") == 0)) {
       // For H.264 video stream, we use a special sink that insert start_codes:
       fileSink = H264VideoFileSink::createNew(*env, outFileName,
                           subsession->fmtp_spropparametersets(),
                           fileSinkBufferSize, oneFilePerFrame);
     } else {
       // Normal case:
       fileSink = FileSink::createNew(*env, outFileName,
                      fileSinkBufferSize, oneFilePerFrame);
     }
     subsession->sink = fileSink;
     if (subsession->sink == NULL) {
       *env << "Failed to create FileSink for \"" << outFileName
           << "\": " << env->getResultMsg() << "\n";
     } else {
       if (singleMedium == NULL) {
         *env << "Created output file: \"" << outFileName << "\"\n";
       } else {
         *env << "Outputting data from the \"" << subsession->mediumName()
             << "/" << subsession->codecName()
             << "\" subsession to 'stdout'\n";
       }

       if (strcmp(subsession->mediumName(), "video") == 0 &&
           strcmp(subsession->codecName(), "MP4V-ES") == 0 &&
           subsession->fmtp_config() != NULL) {
         // For MPEG-4 video RTP streams, the 'config' information
         // from the SDP description contains useful VOL etc. headers.
         // Insert this data at the front of the output file:
         unsigned configLen;
         unsigned char* configData
           = parseGeneralConfigStr(subsession->fmtp_config(), configLen);
         struct timeval timeNow;
         gettimeofday(&timeNow, NULL);
         fileSink->addData(configData, configLen, timeNow);
         delete[] configData;
       }

       subsession->sink->startPlaying(*(subsession->readSource()),
                      subsessionAfterPlaying,
                      subsession);

       // Also set a handler to be called if a RTCP "BYE" arrives
       // for this subsession:
       if (subsession->rtcpInstance() != NULL) {
         subsession->rtcpInstance()->setByeHandler(subsessionByeHandler, subsession);
       }

       madeProgress = True;
     }
       }
       if (!madeProgress) shutdown();
     }
   }

   // Finally, start playing each subsession, to start the data flow:
   if (duration == 0) {
     if (scale > 0) duration = session->playEndTime() - initialSeekTime; // use SDP end time
     else if (scale < 0) duration = initialSeekTime;
   }
   if (duration < 0) duration = 0.0;

   endTime = initialSeekTime;
   if (scale > 0) {
     if (duration <= 0) endTime = -1.0f;
     else endTime = initialSeekTime + duration;
   } else {
     endTime = initialSeekTime - duration;
     if (endTime < 0) endTime = 0.0f;
   }

   startPlayingSession(session, initialSeekTime, endTime, scale, continueAfterPLAY);
 }

 fileSink = H264VideoFileSink::createNew(*env, outFileName,
                           subsession->fmtp_spropparametersets(),
                           fileSinkBufferSize, oneFilePerFrame);

然后是subsession->sink = fileSink;

然后比较关键的是就是

 subsession->sink->startPlaying(*(subsession->readSource()),
                      subsessionAfterPlaying,
                      subsession);

我们来看看这个startPlaying

 Boolean MediaSink::startPlaying(MediaSource& source,
                 afterPlayingFunc* afterFunc,
                 void* afterClientData) {
   // Make sure we're not already being played:
   if (fSource != NULL) {
     envir().setResultMsg("This sink is already being played");
     return False;
   }

   // Make sure our source is compatible:
   if (!sourceIsCompatibleWithUs(source)) {
     envir().setResultMsg("MediaSink::startPlaying(): source is not compatible!");
     return False;
   }
   fSource = (FramedSource*)&source;

   fAfterFunc = afterFunc;
   fAfterClientData = afterClientData;
   return continuePlaying();
 }

上面中subsession->readSource()返回的是fReadSource就是在 createSourceObjects()中建立的那个source。我们看到这里赋值给了fSource。

continuePlaying()在MediaSink中为纯虚函数，在FileSink中有定义。

 Boolean FileSink::continuePlaying() {
   if (fSource == NULL) return False;

   fSource->getNextFrame(fBuffer, fBufferSize,
             afterGettingFrame, this,
             onSourceClosure, this);

   return True;
 }

其实很简单就是fSource的getNextFrame。这里的fSource就是MediaSink中的fSource。就是H264VideoRTPSource。

所有的getNextFrame都一样就是FrameSource中的getNextFrame。把fBuffer给fTo，fBufferSize就是fMaxSize。

我们来看看这个fBuffer，

 fBuffer = new unsigned char[bufferSize];

在

 fileSink = H264VideoFileSink::createNew(*env, outFileName,
                           subsession->fmtp_spropparametersets(),
                           fileSinkBufferSize, oneFilePerFrame);

中fileSinkBufferSize是100000。

getNextFrame之后执行的是doGetNextFrame(),一般在子类里面实现。H264VideoRTPSource中没有实现，但在他的父类MultiFramedRTPSource里面有实现

 void MultiFramedRTPSource::doGetNextFrame() {
   if (!fAreDoingNetworkReads) {
     // Turn on background read handling of incoming packets:
     fAreDoingNetworkReads = True;
     TaskScheduler::BackgroundHandlerProc* handler
       = (TaskScheduler::BackgroundHandlerProc*)&networkReadHandler;
     fRTPInterface.startNetworkReading(handler);
   }

   fSavedTo = fTo;
   fSavedMaxSize = fMaxSize;
   fFrameSize = 0; // for now
   fNeedDelivery = True;
   doGetNextFrame1();
 }