licode/erizo 转推rtmp时h264帧重排引发崩溃bug排查

崩溃堆栈

#0 0x0000000000886892 in erizo::H264Depacketizer::processPacket (this=0x7fbc9c0037d0)
at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/src/erizo/media/Depacketizer.cpp:179

#1 0x000000000089e21e in erizo::ExternalOutput::maybeWriteVideoPacket (this=0x7fbca4032918, buf=0x7fbcd44c4b74 "\2202\001", len=1377) at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/src/erizo/media/ExternalOutput.cpp:415

#2 0x000000000089dbf3 in erizo::ExternalOutput::writeVideoData (this=0x7fbca4032918, buf=0x7fbcd44c4b74 "\2202\001", len=1377)
at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/src/erizo/media/ExternalOutput.cpp:374

#3 0x00000000008a24cf in erizo::ExternalOutput::sendLoop (this=0x7fbca4032918)
at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/src/erizo/media/ExternalOutput.cpp:920

#4 0x00000000008a8ba5 in boost::_mfi::mf0::operator() (this=0x7fbca41b9f48, p=0x7fbca4032918)
at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/include_linux/boost/bind/mem_fn_template.hpp:49

#5 0x00000000008a8a80 in boost::_bi::list1::operator(), boost::_bi::list0> (
this=0x7fbca41b9f58, f=..., a=...) at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/include_linux/boost/bind/bind.hpp:253

#6 0x00000000008a8923 in boost::_bi::bind_t, boost::_bi::list1 >::operator() (this=0x7fbca41b9f48) at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/include_linux/boost/bind/bind_template.hpp:20

#7 0x00000000008a827a in boost::detail::thread_data, boost::_bi::list1 > >::run (this=0x7fbca41b9d90)
at /home/deploy/momodeploy/shellagent/tmp/live-rtc-server/tmp/live-rtc-rtp-server/include_linux/boost/thread/detail/thread.hpp:117
#8 0x00007fbcf4afd27a in thread_proxy () from /home/server/mediaserver/lib_linux/libboost_thread-mt.so.1.53.0
#9 0x00007fbcf7b92e25 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fbcf400534d in clone () from /lib64/libc.so.6

问题排查

1. 出错的代码段如下：

bool H264Depacketizer::processPacket() {
  switch (last_payload_->nal_type) {
    case single: {
      if (search_state_ == SearchState::lookingForEnd) {
        reset();
      }
      if (last_payload_->dataLength == 0) {
        return false;
      }
      const auto total_size = last_payload_->dataLength + sizeof(RTPPayloadH264::start_sequence);
      if (bufferCheck(total_size)) {
        std::memcpy(getBufferPtr(), RTPPayloadH264::start_sequence, sizeof(RTPPayloadH264::start_sequence));
        setBufferPtr(getBufferPtr() + sizeof(RTPPayloadH264::start_sequence));
        std::memcpy(getBufferPtr(), last_payload_->data, last_payload_->dataLength);
        setBufferPtr(getBufferPtr() + last_payload_->dataLength);
        return true;
      }
      break;
    }
    case fragmented: {
      if (last_payload_->dataLength == 0) {
        return false;
      }
      const auto total_size = last_payload_->dataLength
          + sizeof(RTPPayloadH264::start_sequence)
          + last_payload_->fragment_nal_header_len;
      if (bufferCheck(total_size)) {
        if (last_payload_->start_bit) {
          if (search_state_ == SearchState::lookingForEnd) {
            reset();
          }
          search_state_ = SearchState::lookingForEnd;
          std::memcpy(getBufferPtr(), RTPPayloadH264::start_sequence, sizeof(RTPPayloadH264::start_sequence));
          setBufferPtr(getBufferPtr() + sizeof(RTPPayloadH264::start_sequence));
          std::memcpy(getBufferPtr(), &last_payload_->fragment_nal_header, last_payload_->fragment_nal_header_len);
          setBufferPtr(getBufferPtr() + last_payload_->fragment_nal_header_len);
        }
        std::memcpy(getBufferPtr(), last_payload_->data, last_payload_->dataLength);
        setBufferPtr(getBufferPtr() + last_payload_->dataLength);
        if (last_payload_->end_bit) {
          search_state_ = SearchState::lookingForStart;
          return true;
        }
      }
      break;
    }
    case aggregated: {
      if (last_payload_->unpacked_data_len == 0) {
        return false;
      }
      if (bufferCheck(last_payload_->unpacked_data_len)) {
        std::memcpy(getBufferPtr(), &last_payload_->unpacked_data[0], last_payload_->unpacked_data_len);
        setBufferPtr(getBufferPtr() + last_payload_->unpacked_data_len);
        return true;
      }
    }
  }
  return false;
}

分析堆栈得到在fragmented 即h264帧重组模块中，读取last_payload_的值为空引起的，我们可以分析用法得到调用process前一定会调用fetchPacket  进行送数据。所以 last_payload_ 在该函数一定是有值的，那么只有一种可能，那就是那个reset()操作引起的。

发生reset的场景是因为中间丢了h264的最后一个数据包，但序号是连续的。另一种方式表示就是rtp封装出错了。这里调用reset却不return false，直接引起的last_payload_ 的segment fault 。

同时这里还有另外一个bug，那就是如果丢包了，且恰好是将start_bit 的rtp丢了，那么组出来的帧就是一个错误的帧，可能引起花屏现象。比如在ExternalOutput中转推rtmp，那么推出去的数据将会是花屏。

修改后的代码如下：

bool H264Depacketizer::processPacket() {
  switch (last_payload_->nal_type) {
    case single: {
      if (search_state_ == SearchState::lookingForEnd) {
          ELOG_WARN("appID:%s, channelID:%s, userID:%s, H264Depacketizer processPacket, msg: met Rest in single packet",
                appID.c_str(), channelID.c_str(), userID.c_str());
          reset();
          return false;
      }
      if (last_payload_->dataLength == 0) {
        return false;
      }
      const auto total_size = last_payload_->dataLength + sizeof(RTPPayloadH264::start_sequence);
      if (bufferCheck(total_size)) {
        std::memcpy(getBufferPtr(), RTPPayloadH264::start_sequence, sizeof(RTPPayloadH264::start_sequence));
        setBufferPtr(getBufferPtr() + sizeof(RTPPayloadH264::start_sequence));
        std::memcpy(getBufferPtr(), last_payload_->data, last_payload_->dataLength);
        setBufferPtr(getBufferPtr() + last_payload_->dataLength);
        return true;
      }
      break;
    }
    case fragmented: {
      if (last_payload_->dataLength == 0) {
        return false;
      }
      const auto total_size = last_payload_->dataLength
          + sizeof(RTPPayloadH264::start_sequence)
          + last_payload_->fragment_nal_header_len;
      if (bufferCheck(total_size)) {
        if (last_payload_->start_bit) {
          if (search_state_ == SearchState::lookingForEnd) {
              ELOG_WARN("appID:%s, channelID:%s, userID:%s, H264Depacketizer processPacket, msg: met Rest in fragmented packet",
                appID.c_str(), channelID.c_str(), userID.c_str());
            reset();
            return false;
          }
          search_state_ = SearchState::lookingForEnd;
          // 先将 0x00 00 00 01 拷贝到码流中
          std::memcpy(getBufferPtr(), RTPPayloadH264::start_sequence, sizeof(RTPPayloadH264::start_sequence));
          setBufferPtr(getBufferPtr() + sizeof(RTPPayloadH264::start_sequence));
          // 再拷贝提出rtp封装的nal unit
          std::memcpy(getBufferPtr(), &last_payload_->fragment_nal_header, last_payload_->fragment_nal_header_len);
          setBufferPtr(getBufferPtr() + last_payload_->fragment_nal_header_len);
        }
        if (search_state_ == SearchState::lookingForEnd) {
            std::memcpy(getBufferPtr(), last_payload_->data, last_payload_->dataLength);
            setBufferPtr(getBufferPtr() + last_payload_->dataLength);
            if (last_payload_->end_bit) {
              search_state_ = SearchState::lookingForStart;
              return true;
            }
        } else {
            auto pStr = last_payload_->data;
            ELOG_WARN("appID:%s, channelID:%s, userID:%s, search_state_ not SearchState::lookingForEnd, "
                    "0x%x, 0x%x, 0x%x, 0x%x, 0x%x, type:%d",
                    appID.c_str(), channelID.c_str(), userID.c_str(),
                    pStr[0], pStr[1], pStr[2], pStr[3], pStr[4], (pStr[4]&0x1f));
        }
      }
      break;
    }
    case aggregated: {
      if (last_payload_->unpacked_data_len == 0) {
        return false;
      }
      if (bufferCheck(last_payload_->unpacked_data_len)) {
        std::memcpy(getBufferPtr(), &last_payload_->unpacked_data[0], last_payload_->unpacked_data_len);
        setBufferPtr(getBufferPtr() + last_payload_->unpacked_data_len);
        return true;
      }
    }
  }
  return false;
}