Android端使用FFmpeg的AVFilter滤镜
现在各种直播平台如雨后春笋般出现，游戏、女主播等直播特受欢迎。开发人员往往会在推流端加上各种滤镜特效，不仅限于美颜美肤。本篇文章探讨使用FFmpeg的AVFilter实现滤镜，它可以实现多种花式特效。更多信息请查看FFmpeg官网文档：
https://ffmpeg.org/ffmpeg-filters.html。
AVFilter基本使用流程是：初始化滤波器、传入待滤波的视频帧、取出滤波后的视频帧。初始化需要用到的方法包括：获取输入输出滤波器avfilter_get_by_name、分配输入输出流avfilter_inout_alloc、分配滤波图层avfilter_graph_alloc、创建滤波器avfilter_graph_create_filter、解析滤波指令avfilter_graph_parse_ptr、配置滤波图层avfilter_graph_config。
滤波器又分为简单和复杂两种。其中简单滤波器是一对一关系，如下图所示：
而复杂滤波器往往是多对多关系：
滤波器初始化代码如下，有参考FFmpeg官方文档：
//初始化滤波器
int init_filters(const char *filters_descr) {
char args[512];
int ret = 0;
AVFilter *buffersrc = avfilter_get_by_name("buffer");
AVFilter *buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut *outputs = avfilter_inout_alloc();
AVFilterInOut *inputs = avfilter_inout_alloc();
AVRational time_base = pFormatCtx->streams[video_stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt,
time_base.num, time_base.den,
pCodecCtx->sample_aspect_ratio.num, pCodecCtx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
LOGE("Cannot create buffer source\n");
goto end;
}
/* buffer video sink: to terminate the filter chain. */
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
LOGE("Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
LOGE("Cannot set output pixel format\n");
goto end;
}
/*
* The buffer source output must be connected to the input pad of
* the first filter described by filters_descr; since the first
* filter input label is not specified, it is set to "in" by
* default.
*/
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
/*
* The buffer sink input must be connected to the output pad of
* the last filter described by filters_descr; since the last
* filter output label is not specified, it is set to "out" by
* default.
*/
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}打开视频流操作包括：注册相关函数、获取上下文、打开输入文件、寻找视频流、寻找并打开解码器，还有就是NativeWindow相关初始化。
int open_input(JNIEnv * env, const char* file_name, jobject surface){
LOGI("open file:%s\n", file_name);
//注册所有组件
av_register_all();
//分配上下文
pFormatCtx = avformat_alloc_context();
//打开视频文件
if(avformat_open_input(&pFormatCtx, file_name, NULL, NULL)!=0) {
LOGE("Couldn't open file:%s\n", file_name);
return -1;
}
//检索多媒体流信息
if(avformat_find_stream_info(pFormatCtx, NULL)<0) {
LOGE("Couldn't find stream information.");
return -1;
}
//寻找视频流的第一帧
int i;
for (i = 0; i < pFormatCtx->nb_streams; i++) {
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO
&& video_stream_index < 0) {
video_stream_index = i;
}
}
if(video_stream_index == -1) {
LOGE("couldn't find a video stream.");
return -1;
}
//获取codec上下文指针
pCodecCtx = pFormatCtx->streams[video_stream_index]->codec;
//寻找视频流的解码器
AVCodec * pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if(pCodec==NULL) {
LOGE("couldn't find Codec.");
return -1;
}
if(avcodec_open2(pCodecCtx, pCodec, NULL) < 0) {
LOGE("Couldn't open codec.");
return -1;
}
// 获取native window
nativeWindow = ANativeWindow_fromSurface(env, surface);
// 设置native window的buffer大小,可自动拉伸
ANativeWindow_setBuffersGeometry(nativeWindow, pCodecCtx->width, pCodecCtx->height, WINDOW_FORMAT_RGBA_8888);
//申请内存
pFrame = av_frame_alloc();
pFrameRGBA = av_frame_alloc();
if(pFrameRGBA == NULL || pFrame == NULL) {
LOGE("Couldn't allocate video frame.");
return -1;
}
// buffer中数据用于渲染,且格式为RGBA
int numBytes=av_image_get_buffer_size(AV_PIX_FMT_RGBA, pCodecCtx->width, pCodecCtx->height, 1);
buffer=(uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
av_image_fill_arrays(pFrameRGBA->data, pFrameRGBA->linesize, buffer, AV_PIX_FMT_RGBA,
pCodecCtx->width, pCodecCtx->height, 1);
// 由于解码出来的帧格式不是RGBA的,在渲染之前需要进行格式转换
sws_ctx = sws_getContext(pCodecCtx->width,
pCodecCtx->height,
pCodecCtx->pix_fmt,
pCodecCtx->width,
pCodecCtx->height,
AV_PIX_FMT_RGBA,
SWS_BILINEAR,
NULL,
NULL,
NULL);
return 0;
} // 循环读取一帧视频
while(av_read_frame(pFormatCtx, &packet)>=0 && !release) {
//切换滤波器，退出当初播放
if(again){
goto again;
}
//判断是否为视频流
if(packet.stream_index == video_stream_index) {
//对该帧进行解码
avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);
if (frameFinished) {
//把解码后视频帧添加到filter graph
if (av_buffersrc_add_frame_flags(buffersrc_ctx, pFrame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
LOGE("Error while feeding the filter_graph\n");
break;
}
//把滤波后的视频帧从filter graph取出来
ret = av_buffersink_get_frame(buffersink_ctx, filter_frame);
if (ret >= 0){
// lock native window
ANativeWindow_lock(nativeWindow, &windowBuffer, 0);
// 格式转换
sws_scale(sws_ctx, (uint8_t const * const *)filter_frame->data,
filter_frame->linesize, 0, pCodecCtx->height,
pFrameRGBA->data, pFrameRGBA->linesize);
// 获取stride
uint8_t * dst = windowBuffer.bits;
int dstStride = windowBuffer.stride * 4;
uint8_t * src = pFrameRGBA->data[0];
int srcStride = pFrameRGBA->linesize[0];
// 由于window的stride和帧的stride不同,因此需要逐行复制
int h;
for (h = 0; h < pCodecCtx->height; h++) {
memcpy(dst + h * dstStride, src + h * srcStride, (size_t) srcStride);
}
ANativeWindow_unlockAndPost(nativeWindow);
}
av_frame_unref(filter_frame);
}
av_frame_unref(pFrame);
//延迟等待
usleep((unsigned long) (1000 * 40));
}
av_packet_unref(&packet);
}
end:
is_playing = 0;
//释放内存以及关闭文件
av_free(buffer);
av_free(pFrameRGBA);
av_free(filter_frame);
av_free(pFrame);
avcodec_close(pCodecCtx);
avformat_close_input(&pFormatCtx);
avfilter_free(buffersrc_ctx);
avfilter_free(buffersink_ctx);
avfilter_graph_free(&filter_graph);
(*env)->ReleaseStringUTFChars(env, filePath, file_name);
(*env)->ReleaseStringUTFChars(env, filterDescr, filter_descr);
LOGE("do release...");
again:
again = 0;
LOGE("play again...");
return ret;
}最后看下几种滤镜效果：
详细代码地址：https://github.com/xufuji456/FFmpegAndroid。如果大家有问题，欢迎提出，也欢迎大家star和fork。