音视频开发-ffmpeg介绍-系列三个

根据上篇文章的介绍，这篇文章进行具体的实现，配置ffmpeg到AS的过程就不讲解了

目录

一、创建封装格式上下文

二、读取包

三、音频重采样&音频播放以及解码

四、视频解码&视频播放

五、视频渲染

---

一、创建封装格式上下文

avformat_network_init();
//todo 最新版本好像不用 regiest_all了
    avformat_network_init();
    // 代表一个 视频/音频 包含了视频、音频的各种信息
    formatContext = avformat_alloc_context();
    //1、打开URL
    AVDictionary *opts = NULL;
    //设置超时3秒
    av_dict_set(&opts, "timeout", "3000000", 0);
    //强制指定AVFormatContext中AVInputFormat的。这个参数一般情况下可以设置为NULL，这样FFmpeg可以自动检测AVInputFormat。
    //输入文件的封装格式
//    av_find_input_format("avi")
    int ret = avformat_open_input(&formatContext, url, NULL, &opts);
    //av_err2str(ret)
    LOGE("%s open %d  %s", url, ret, av_err2str(ret));
    if (ret != 0) {
        if (javaCallHelper) {
            javaCallHelper->onError(THREAD_CHILD, (jstring) FFMPEG_CAN_NOT_OPEN_URL);
        }
        return;
    }
    //2.查找流
    if (avformat_find_stream_info(formatContext, NULL) < 0) {
        if (javaCallHelper) {
            javaCallHelper->onError(THREAD_CHILD, (jstring) FFMPEG_CAN_NOT_FIND_STREAMS);
        }
        return;
    }
    //视频时长（单位：微秒us，转换为秒需要除以1000000）
    duration = formatContext->duration / 1000000;
    for (int i = 0; i < formatContext->nb_streams; ++i) {
        AVCodecParameters *codecpar = formatContext->streams[i]->codecpar;
        //找到解码器
        AVCodec *dec = avcodec_find_decoder(codecpar->codec_id);
        if (!dec) {
            if (javaCallHelper) {
                javaCallHelper->onError(THREAD_CHILD, (jstring) FFMPEG_FIND_DECODER_FAIL);
            }
            return;
        }
        //创建上下文
        AVCodecContext *codecContext = avcodec_alloc_context3(dec);
        if (!codecContext) {
            if (javaCallHelper)
                javaCallHelper->onError(THREAD_CHILD, (jstring) FFMPEG_ALLOC_CODEC_CONTEXT_FAIL);
            return;
        }
        //复制参数
        if (avcodec_parameters_to_context(codecContext, codecpar) < 0) {
            if (javaCallHelper)
                javaCallHelper->onError(THREAD_CHILD,
                                        (jstring) FFMPEG_CODEC_CONTEXT_PARAMETERS_FAIL);
            return;
        }
        //打开解码器
        if (avcodec_open2(codecContext, dec, 0) != 0) {
            if (javaCallHelper)
                javaCallHelper->onError(THREAD_CHILD,
                                        reinterpret_cast(FFMPEG_OPEN_DECODER_FAIL));
            return;
        }
        //时间基
        AVRational base = formatContext->streams[i]->time_base;
        //音频
        if (codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
            LOGE("创建audio");
            audioChannel = new AudioChannel(i, javaCallHelper, codecContext, base);
        } else if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
            LOGE("创建video");
            //视频
//            int num = formatContext->streams[i]->avg_frame_rate.num;
//            int den = formatContext->streams[i]->avg_frame_rate.den;
            //帧率 num：分子
            int fps = av_q2d(formatContext->streams[i]->avg_frame_rate);
            videoChannel = new VideoChannel(i, javaCallHelper, codecContext, base, fps);
            videoChannel->setRenderCallback(renderFrame);
        }
    }

    //音视频都没有
    if (!audioChannel && !videoChannel) {
        if (javaCallHelper)
            javaCallHelper->onError(THREAD_CHILD, (jstring) FFMPEG_NOMEDIA);
        return;
    }
    if (javaCallHelper)
        javaCallHelper->onPrepare(THREAD_CHILD);

上述核心代码主要作用就是解封装，创建封装格式上下文，获取音视频流信息和音视频流参数

二、读取包

//锁住formatContext
        pthread_mutex_lock(&seekMutex);
        //读取包
        AVPacket *packet = av_packet_alloc();
        // 从媒体中读取音频、视频包
        ret = av_read_frame(formatContext, packet);
        pthread_mutex_unlock(&seekMutex);
        if (ret == 0) {
            //将数据包加入队列
            if (audioChannel && packet->stream_index == audioChannel->channleId) {
                audioChannel->pkt_queue.push(packet);
            } else if (videoChannel && packet->stream_index == videoChannel->channleId) {
                videoChannel->pkt_queue.push(packet);
            }
        } else if (ret == AVERROR_EOF) {
            //读取完毕 但是不一定播放完毕
            if (videoChannel->pkt_queue.empty() && videoChannel->frame_queue.empty() &&
                audioChannel->pkt_queue.empty() && audioChannel->frame_queue.empty()) {
                LOGE("播放完毕。。。");
                break;
            }
            //因为seek 的存在，就算读取完毕，依然要循环 去执行av_read_frame(否则seek了没用...)
        } else {
            break;
        }

    }
    isPlaying = 0;
    audioChannel->stop();
    videoChannel->stop();

上述代码的主要作用就是读取音视频数据到对应的队列里面，

三、音频重采样&音频播放以及解码

//重采样为固定的双声道 16位 44100
    swr_ctx = swr_alloc_set_opts(0, AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_S16, out_sample_rate,
                                 avCodecContext->channel_layout,
                                 avCodecContext->sample_fmt,
                                 avCodecContext->sample_rate, 0, 0);
    swr_init(swr_ctx);
    startWork();
    isPlaying = true;
    pthread_create(&pid_audio_play, NULL, audioPlay, this);
    pthread_create(&pid_audio_decode, NULL, audioDecode, this);

 音频解码

AVPacket *packet = 0;
    while (isPlaying) {
        int ret = pkt_queue.pop(packet);
        if (!isPlaying) {
            break;
        }
        if (!ret) {
            continue;
        }
        ret = avcodec_send_packet(avCodecContext, packet);
        releaseAvPacket(packet);
        if (ret == AVERROR(EAGAIN)) {
            //需要更多数据
            continue;
        } else if (ret < 0) {
            //失败
            break;
        }
        AVFrame *frame = av_frame_alloc();
        ret = avcodec_receive_frame(avCodecContext, frame);
        if (ret == AVERROR(EAGAIN)) {
            //需要更多数据
            continue;
        } else if (ret < 0) {
            break;
        }
        while (frame_queue.size() > 100 && isPlaying) {
            av_usleep(1000 * 10);
            continue;
        }
        frame_queue.push(frame);
    }
    releaseAvPacket(packet);

音频播放，采用Android 自带的openSl es

void AudioChannel::initOpenSL() {
    //创建引擎
    SLresult result;
    // 创建引擎engineObject
    result = slCreateEngine(&engineObject, 0, NULL, 0, NULL, NULL);
    if (SL_RESULT_SUCCESS != result) {
        return;
    }
    // 初始化引擎engineObject
    result = (*engineObject)->Realize(engineObject, SL_BOOLEAN_FALSE);
    if (SL_RESULT_SUCCESS != result) {
        return;
    }
    // 获取引擎接口engineEngine
    result = (*engineObject)->GetInterface(engineObject, SL_IID_ENGINE,
                                           &engineInterface);
    if (SL_RESULT_SUCCESS != result) {
        return;
    }

    // 创建混音器outputMixObject
    result = (*engineInterface)->CreateOutputMix(engineInterface, &outputMixObject, 0, 0, 0);
    if (SL_RESULT_SUCCESS != result) {
        return;
    }

    // 初始化混音器outputMixObject
    result = (*outputMixObject)->Realize(outputMixObject, SL_BOOLEAN_FALSE);
    if (SL_RESULT_SUCCESS != result) {
        return;
    }


    /**
     * 配置输入声音信息
     */
    //创建buffer缓冲类型的队列 2个队列
    SLDataLocator_AndroidSimpleBufferQueue android_queue = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
                                                            2};
    //pcm数据格式
    SLDataFormat_PCM pcm = {SL_DATAFORMAT_PCM, 2, SL_SAMPLINGRATE_44_1, SL_PCMSAMPLEFORMAT_FIXED_16,
                            SL_PCMSAMPLEFORMAT_FIXED_16,
                            SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT,
                            SL_BYTEORDER_LITTLEENDIAN};

    //数据源 将上述配置信息放到这个数据源中
    SLDataSource slDataSource = {&android_queue, &pcm};


    //设置混音器
    SLDataLocator_OutputMix outputMix = {SL_DATALOCATOR_OUTPUTMIX, outputMixObject};
    SLDataSink audioSnk = {&outputMix, NULL};
    //需要的接口
    const SLInterfaceID ids[1] = {SL_IID_BUFFERQUEUE};
    const SLboolean req[1] = {SL_BOOLEAN_TRUE};
    //创建播放器
    (*engineInterface)->CreateAudioPlayer(engineInterface, &bqPlayerObject, &slDataSource,
                                          &audioSnk, 1,
                                          ids, req);
    //初始化播放器
    (*bqPlayerObject)->Realize(bqPlayerObject, SL_BOOLEAN_FALSE);

//    得到接口后调用  获取Player接口
    (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_PLAY, &bqPlayerInterface);

//    获得播放器接口
    (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_BUFFERQUEUE,
                                    &bqPlayerBufferQueue);
    //设置回调
    (*bqPlayerBufferQueue)->RegisterCallback(bqPlayerBufferQueue, bqPlayerCallback, this);
//    设置播放状态
    (*bqPlayerInterface)->SetPlayState(bqPlayerInterface, SL_PLAYSTATE_PLAYING);

    bqPlayerCallback(bqPlayerBufferQueue, this);
}

四、视频解码&视频播放

视频解码

void VideoChannel::decodePacket() {
    AVPacket *packet = 0;
    while (isPlaying) {
        int ret = pkt_queue.pop(packet);
        if (!isPlaying) {
            break;
        }
        if (!ret) {
            continue;
        }

        ret = avcodec_send_packet(avCodecContext, packet);
        releaseAvPacket(packet);
        if (ret == AVERROR(EAGAIN)) {
            //需要更多数据
            continue;
        } else if (ret < 0) {
            //失败
            break;
        }
        AVFrame *frame = av_frame_alloc();
        ret = avcodec_receive_frame(avCodecContext, frame);
        if (ret == AVERROR(EAGAIN)) {
            //需要更多数据
            continue;
        } else if (ret < 0) {
            break;
        }
        while (frame_queue.size() > 100 && isPlaying) {
            av_usleep(1000 * 10);
            continue;
        }
        frame_queue.push(frame);
    }
    releaseAvPacket(packet);
}

视频播放，同时还解决音视频同步问题

void VideoChannel::synchronizeFrame() {
    //转换rgba
    SwsContext *sws_ctx = sws_getContext(
            avCodecContext->width, avCodecContext->height, avCodecContext->pix_fmt,
            avCodecContext->width, avCodecContext->height, AV_PIX_FMT_RGBA,
            SWS_BILINEAR, 0, 0, 0);

    //1s
    double frame_delay = 1.0 / fps;
    uint8_t *dst_data[4];
    int dst_linesize[4];
    av_image_alloc(dst_data, dst_linesize,
                   avCodecContext->width, avCodecContext->height, AV_PIX_FMT_RGBA, 1);
    AVFrame *frame = 0;
    while (isPlaying) {
        int ret = frame_queue.pop(frame);
        if (!isPlaying) {
            break;
        }
        if (!ret) {
            continue;
        }
#if 1
        /**
         *  seek需要注意的点：编码器中存在缓存
         *  100s 的图像,用户seek到第 50s 的位置releaseAvFrame
         *  音频是50s的音频，但是视频 你获得的是100s的视频
         */

        //显示时间戳 什么时候显示这个frame
        if ((clock = frame->best_effort_timestamp) == AV_NOPTS_VALUE) {
            clock = 0;
        }
        //pts 单位就是time_base
        //av_q2d转为双精度浮点数 乘以 pts 得到pts --- 显示时间:秒
        clock = clock * av_q2d(time_base);
        //frame->repeat_pict = 当解码时，这张图片需要要延迟多久显示
        //需要求出扩展延时：
        //extra_delay = repeat_pict / (2*fps) 需要延迟这么久来显示
        double repeat_pict = frame->repeat_pict;
        double extra_delay = repeat_pict / (2 * fps);

        // double frame_base = av_q2d(time_base);
        double delay = extra_delay + frame_delay;
        if (clock == 0) {
            //正常播放
            av_usleep(delay * 1000000);
        } else {
            double audioClock = audioChannel ? audioChannel->clock : 0;
            double diff = fabs(clock - audioClock);
            LOGE("当前和音频比较:%f - %f = %f", clock, audioClock, diff);
            //允许误差 diff > 0.04 &&
            if (audioChannel) {
                //如果视频比音频快，延迟差值播放，否则直接播放
                if (clock > audioClock) {
                    if (diff > 1) {
                        //差的太久了， 那只能慢慢赶 不然就是卡好久
                        av_usleep((delay * 2) * 1000000);
                    } else {
                        //差的不多，尝试一次赶上去
                        av_usleep((delay + diff) * 1000000);
                    }
                } else {
                    //音频比视频快
                    //视频慢了 0.05s 已经比较明显了 (丢帧)
                    if (diff > 1) {
                        //一种可能： 快进了(因为解码器中有缓存数据，这样获得的avframe就和seek的匹配了)
                    } else if (diff >= 0.05) {
                        releaseAvFrame(frame);
                        //执行同步操作 删除到最近的key frame
                        frame_queue.sync();
                        continue;
                    } else {
                        //不休眠 加快速度赶上去
                    }
                }
            } else {
                //正常播放
                av_usleep(delay * 1000000);
            }
        }
#endif
        //diff太大了不回调了
        if (javaCallHelper && !audioChannel) {
            javaCallHelper->onProgress(THREAD_CHILD, clock);
        }
        sws_scale(sws_ctx,
                  reinterpret_cast(frame->data), frame->linesize, 0,
                  frame->height,
                  dst_data, dst_linesize);
        //绘画
        renderFrame(dst_data[0], dst_linesize[0], avCodecContext->width, avCodecContext->height);
        releaseAvFrame(frame);
    }
    av_freep(&dst_data[0]);
    isPlaying = false;
    releaseAvFrame(frame);
    sws_freeContext(sws_ctx);
}

五、视频渲染

使用Android自带的ANativeWindow

void renderFrame(uint8_t *data, int linesize, int w, int h) {
    pthread_mutex_lock(&mutext);
    if (!window) {
        pthread_mutex_unlock(&mutext);
        return;
    }
    //设置窗口属性
    ANativeWindow_setBuffersGeometry(window, w,
                                     h,
                                     WINDOW_FORMAT_RGBA_8888);

    ANativeWindow_Buffer window_buffer;
    if (ANativeWindow_lock(window, &window_buffer, 0)) {
        ANativeWindow_release(window);
        window = 0;
        pthread_mutex_unlock(&mutext);
        return;
    }
    uint8_t *dst_data = static_cast(window_buffer.bits);
    //一行需要多少像素 * 4(RGBA)
    int dst_linesize = window_buffer.stride * 4;
    uint8_t *src_data = data;
    int src_linesize = linesize;
    //一次拷贝一行
    for (int i = 0; i < window_buffer.height; ++i) {
        memcpy(dst_data + i * dst_linesize, src_data + i * src_linesize, dst_linesize);
    }
    ANativeWindow_unlockAndPost(window);
    pthread_mutex_unlock(&mutext);
}

根据以上步骤就可以完成一个具体视频播放器了