java获取近期视频流关键帧与截图
1、背景
最近在做视频转发的开发时，遇到一个问题，前端订阅播放h264视频流时，有时会出现一段时间黑屏，经过测试发现是没有收到关键帧，只有第一帧是关键帧才能保证后续播放正常。所以后端需要实现一个功能，就是前端在进入播放页面时，后端把最近的一个关键帧发过去。
2、思路（环形缓存区）
后端接收到的视频流是一个个的字节数组，所以在接收时没法直接判断一帧的开始和结束，需要将最近的一段视频流截取出来，然后利用ffmpeg工具进行整体的解析和关键帧提取。
查看ffmpeg工具的代码，可以发现ffmpeg工具入参是inputstream，该工具会不断调用inputstream的read方法进行字节的读取。所以就想通过一个环形缓存区不断的记录最新的一段视频流数据。该环形缓存区再实现inputstream的接口，重写read方法，read读取的开始位置即是环形缓存区头，到环形缓存区的尾时自动结束。
3、依赖包
org.bytedeco
javacpp
1.4.3
org.bytedeco
javacv
1.4.3
org.bytedeco.javacpp-presets
ffmpeg-platform
4.0.2-1.4.3
4、环形缓存区定义
public class CycleBufferInputStream extends InputStream {
/************************************************ 环形缓冲区 ***********************************************/
private ByteBuffer buffer = null;
private int readPos = 0; //将要读取的位置
private int writePos = 0; //将要写入的位置
private boolean isCycle = false; //判断是否已经形成一个环
public CycleBufferInputStream(int capacity) {
this.buffer = ByteBuffer.allocateDirect(capacity);
}
/**
* 将字节数组以覆盖的方式放入环形缓冲区
*/
public void put(byte[] bytes) {
int used = buffer.capacity() - buffer.position();
if (used < bytes.length) {
buffer.put(bytes, 0, used);
buffer.clear();
buffer.put(bytes, used, bytes.length - used);
isCycle = true;
} else if (used == bytes.length) {
buffer.put(bytes, 0, used);
buffer.clear();
isCycle = true;
} else {
buffer.put(bytes, 0, bytes.length);
}
writePos = buffer.position();
}
/**
* 定位读取的初始位置（执行inputstream 读取前，必须要先调用该方法）
*/
public void readPrepare() {
if (buffer.capacity() == writePos || !isCycle) {
readPos = 0;
} else {
readPos = buffer.position() + 1;
}
}
/*************************************************** 输入流传输 ***************************************************/
/**
* Reads the next byte of data from the input stream. The value byte is returned as an int in the range 0 to 255. If no byte is available because the end of the stream has been reached, the value -1 is returned. This method blocks until input data is available, the end of the stream is detected, or an exception is thrown.
* A subclass must provide an implementation of this method.
* Returns: the next byte of data, or -1 if the end of the stream is reached.
* Throws: IOException – if an I/O error occurs.
*/
@Override
public int read() throws IOException {
if (readPos == buffer.capacity()) readPos = 0;
if (readPos == writePos) return -1;
int value = buffer.get(readPos++);
if (value < 0) value = value + 256;
return value;
}
}5、从环形缓存区提取关键帧
/**
* 从环形缓存环获取最近一帧关键帧字节数组
* 这里返回的堆外内存，所以注意要及时进行内存释放
* @param inputStream
*/
public static ByteBuffer dealVideo(InputStream inputStream) {
try {
int j = 0;
FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);
ff.start();
Frame frame = null;
Frame last = null;
while ((frame = ff.grabKeyFrame()) != null && frame.image != null) {
last = frame.clone();
System.out.println("获取一帧" + j++);
}
ff.stop();
if (last != null) {
System.out.println("获取最近的一个关键帧");
ByteBuffer byteBuffer = (ByteBuffer)last.image[0];
return byteBuffer;
}
} catch (Exception e) {
log.error("提取最近一个关键帧异常\n",e);
}
return null;
}6、使用
public static void main(String[] args) throws Exception {
CycleBufferInputStream stream = new CycleBufferInputStream(1024 * 1024 * 10);
FileInputStream fis = new FileInputStream("D:\\tmp-data\\1694511149969.h264");
byte[] bytes = new byte[fis.available()];
fis.read(bytes);
stream.put(bytes);
stream.readPrepare();
dealVideo(stream);
}7、扩展：获取近期视频的截图
其实就是从近期的关键帧中提取出图片，关键代码如下：
public static void dealImage(InputStream inputStream) {
try {
Java2DFrameConverter converter = new Java2DFrameConverter();
FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);
ff.start();
int j = 0;
Frame frame = null;
Frame last = null;
while ((frame = ff.grabImage()) != null) {
last = frame.clone();
System.out.println("获取一张图片" + j++);
}
ff.stop();
if (last != null && last.image != null) {
System.out.println("存储最后一张图片 ");
BufferedImage fecthedImage = converter.getBufferedImage(last);
File screenshotFile = new File("D:\\tmp-data\\", System.currentTimeMillis() + ".jpg");
ImageIO.write(fecthedImage, "jpg", screenshotFile);
}
} catch (Exception e) {
log.error("提取最近一个图片异常\n",e);
}
}8、整体代码
package com.qq.utils;
import lombok.extern.slf4j.Slf4j;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameConverter;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.ByteBuffer;
@Slf4j
public class CycleBufferInputStream extends InputStream {
/************************************************ 环形缓冲区 ***********************************************/
private ByteBuffer buffer = null;
private int readPos = 0; //将要读取的位置
private int writePos = 0; //将要写入的位置
private boolean isCycle = false; //判断是否已经形成一个环
public CycleBufferInputStream(int capacity) {
this.buffer = ByteBuffer.allocateDirect(capacity);
}
/**
* 将字节数组以覆盖的方式放入环形缓冲区
*/
public void put(byte[] bytes) {
int used = buffer.capacity() - buffer.position();
if (used < bytes.length) {
buffer.put(bytes, 0, used);
buffer.clear();
buffer.put(bytes, used, bytes.length - used);
isCycle = true;
} else if (used == bytes.length) {
buffer.put(bytes, 0, used);
buffer.clear();
isCycle = true;
} else {
buffer.put(bytes, 0, bytes.length);
}
writePos = buffer.position();
}
/**
* 定位读取的初始位置（执行inputstream方法前，必须要先调用该方法）
*/
public void readPrepare() {
if (buffer.capacity() == writePos || !isCycle) {
readPos = 0;
} else {
readPos = buffer.position() + 1;
}
}
/*************************************************** 输入流传输 ***************************************************/
@Override
public int read() throws IOException {
if (readPos == buffer.capacity()) readPos = 0;
if (readPos == writePos) return -1;
int value = buffer.get(readPos++);
if (value < 0) value = value + 256;
return value;
}
/**
* 从环形缓存环获取最近一帧关键帧字节数组
* 这里返回的堆外内存，所以注意要及时进行内存释放
* @param inputStream
*/
public static ByteBuffer dealVideo(InputStream inputStream) {
try {
int j = 0;
FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);
ff.start();
Frame frame = null;
Frame last = null;
while ((frame = ff.grabKeyFrame()) != null && frame.image != null) {
last = frame.clone();
System.out.println("获取一帧" + j++);
}
ff.stop();
if (last != null) {
System.out.println("获取最近的一个关键帧");
ByteBuffer byteBuffer = (ByteBuffer)last.image[0];
return byteBuffer;
}
} catch (Exception e) {
log.error("提取最近一个关键帧异常\n",e);
}
return null;
}
public static void dealImage(InputStream inputStream) {
try {
Java2DFrameConverter converter = new Java2DFrameConverter();
FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);
ff.start();
int j = 0;
Frame frame = null;
Frame last = null;
while ((frame = ff.grabImage()) != null) {
last = frame.clone();
System.out.println("获取一张图片" + j++);
}
ff.stop();
if (last != null && last.image != null) {
System.out.println("存储最后一张图片 ");
BufferedImage fecthedImage = converter.getBufferedImage(last);
File screenshotFile = new File("D:\\tmp-data\\", System.currentTimeMillis() + ".jpg");
ImageIO.write(fecthedImage, "jpg", screenshotFile);
}
} catch (Exception e) {
log.error("提取最近一个图片异常\n",e);
}
}
}