Windows平台Unity下实现camera场景推送RTMP|轻量级RTSP服务|实时录像

技术背景

我们在对接Unity平台camera场景采集的时候,除了常规的RTMP推送、录像外,还有一些开发者,需要能实现轻量级RTSP服务,对外提供个拉流的RTSP URL。

目前我们在Windows平台Unity下数据源可采集到以下部分:

  • 采集Unity camera场景;
  • 采集摄像头;
  • 采集屏幕;
  • 采集Unity声音;
  • 采集麦克风;
  • 采集扬声器;
  • Unity PCM混音;

对外提供的技术能力有:

  • RTMP直播推送;
  • 轻量级RTSP服务;
  • 实时录像、暂停|恢复录像;
  • 实时预览。

以下录制下来的MP4文件是采集Unity camera场景,音频是unity声音。

Unity平台实现camera场景实时录像

技术实现

实际上,在实现Unity平台音视频能力之前,我们原生模块已经有非常成熟的技术积累,Unity下还是调用的原生的推送模块,不同的是,数据源需要采集Unity的audio、video,然后高效的投递到底层模块,底层模块负责编码打包,并投递到RTMP或RTSP服务。

Windows平台Unity下实现camera场景推送RTMP|轻量级RTSP服务|实时录像_第1张图片

先说支持的音视频类型:

    public void SelVideoPushType(int type)
    {
        switch (type)
        {
            case 0:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER;    //采集Unity窗体
                break;
            case 1:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA;   //采集摄像头
                break;
            case 2:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_SCREEN;   //采集屏幕
                break;
            case 3:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_NO_VIDEO; //不采集视频
                break;
        }

        Debug.Log("SelVideoPushType type: " + type + " video_push_type: " + video_push_type_);
    }

    public void SelAudioPushType(int type)
    {
        switch (type)
        {
            case 0:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA;    //采集Unity声音
                break;
            case 1:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_MIC;  //采集麦克风
                break;
            case 2:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_SPEAKER;  //采集扬声器
                break;
            case 3:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER;  //两路Unity AudioClip混音测试
                break;
            case 4:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_NO_AUDIO;   //不采集音频
                break;
        }

        Debug.Log("SelAudioPushType type: " + type + " audio_push_type: " + audio_push_type_);
    }

采集音视频数据:

    private void StartCaptureAvData()
    {
        if (audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA
            || audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER)
        {
            PostUnityAudioClipData();
        }

        textures_poll_ = new TexturesPool();
        post_image_worker_ = new PostImageWorker(textures_poll_, publisher_wrapper_);
        post_worker_thread_ = new Thread(post_image_worker_.run);
        post_worker_thread_.Start();
    }

    private void StopCaptureAvData()
    {
        if (post_image_worker_ != null)
        {
            post_image_worker_.requestStop();
            post_image_worker_ = null;
        }

        if (post_worker_thread_ != null)
        {
            post_worker_thread_.Join();
            post_worker_thread_ = null;
        }

        if (textures_poll_ != null)
        {
            textures_poll_.clear();
            textures_poll_ = null;
        }

        StopAudioSource();
    }

RTMP推送:

    public void btn_start_rtmp_pusher_Click()
    {
        if (publisher_wrapper_.IsPushingRtmp())
        {
            StopPushRTMP();
            btn_rtmp_pusher_.GetComponentInChildren().text = "推送RTMP";

            return;
        }

        String url = rtmp_pusher_url_.text;

        if (url.Length < 8)
        {
            publisher_wrapper_.Close();

            Debug.LogError("请输入RTMP推送地址");
            return;
        }

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
        {
            publisher_wrapper_.SetVideoPushType(video_push_type_);
            publisher_wrapper_.SetAudioPushType(audio_push_type_);
        }

        if (!publisher_wrapper_.StartRtmpPusher(url))
        {
            Debug.LogError("调用StartPublisher失败..");
            return;
        }

        btn_rtmp_pusher_.GetComponentInChildren().text = "停止推送";

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
        {
            StartCaptureAvData();
            coroutine_ = StartCoroutine(OnPostVideo());
        }
    }

轻量级RTSP服务相关调用:

    public void btn_rtsp_service_Click()
    {
        if (publisher_wrapper_.IsRtspServiceRunning())
        {
            publisher_wrapper_.StopRtspService();
            btn_rtsp_service_.GetComponentInChildren().text = "启动RTSP服务";

            btn_rtsp_publisher_.interactable = false;
            return;
        }

        if (!publisher_wrapper_.StartRtspService())
        {
            Debug.LogError("调用StartRtspService失败..");
            return;
        }

        btn_rtsp_publisher_.interactable = true;

        btn_rtsp_service_.GetComponentInChildren().text = "停止RTSP服务";
    }

    public void btn_rtsp_publisher_Click()
    {
        if (publisher_wrapper_.IsRtspPublisherRunning())
        {
            publisher_wrapper_.StopRtspStream();

            if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
            {
                StopCaptureAvData();

                if (coroutine_ != null)
                {
                    StopCoroutine(coroutine_);
                    coroutine_ = null;
                }
            }

            btn_rtsp_service_.interactable = true;

            btn_rtsp_publisher_.GetComponentInChildren().text = "发布RTSP";
        }
        else
        {
            if (!publisher_wrapper_.IsRtspServiceRunning())
            {
                Debug.LogError("RTSP service is not running..");
                return;
            }


            if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
            {
                publisher_wrapper_.SetVideoPushType(video_push_type_);
                publisher_wrapper_.SetAudioPushType(audio_push_type_);
            }

            publisher_wrapper_.StartRtspStream();

            if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
            {
                StartCaptureAvData();
                coroutine_ = StartCoroutine(OnPostVideo());
            }

            btn_rtsp_publisher_.GetComponentInChildren().text = "停止RTSP";

            btn_rtsp_service_.interactable = false;
        }
    }

    public void btn_get_rtsp_session_numbers_Click()
    {
        if (publisher_wrapper_.IsRtspServiceRunning())
        {
            btn_get_rtsp_session_numbers_.GetComponentInChildren().text = "RTSP会话数:" + publisher_wrapper_.GetRtspSessionNumbers();
        }
    }

实时录像调用:

    public void btn_record_Click()
    {
        if (publisher_wrapper_.IsRecording())
        {
            StopRecord();
            btn_record_.GetComponentInChildren().text = "开始录像";

            return;
        }

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
        {
            publisher_wrapper_.SetVideoPushType(video_push_type_);
            publisher_wrapper_.SetAudioPushType(audio_push_type_);
        }

        if (!publisher_wrapper_.StartRecorder())
        {
            Debug.LogError("调用StartRecorder失败..");
            return;
        }

        btn_record_.GetComponentInChildren().text = "停止录像";

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
        {
            StartCaptureAvData();
            coroutine_ = StartCoroutine(OnPostVideo());
        }
    }

    public void StopRecord()
    {
        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
        {
            StopCaptureAvData();

            if (coroutine_ != null)
            {
                StopCoroutine(coroutine_);
                coroutine_ = null;
            }
        }

        publisher_wrapper_.StopRecorder();
    }

    private void btn_pause_record_Click()
    {
        if (!publisher_wrapper_.IsPublisherHandleAvailable())
        {
            return;
        }

        String btn_pause_rec_text = btn_pause_record_.GetComponentInChildren().text;

        if ("暂停录像" == btn_pause_rec_text)
        {
            UInt32 ret = publisher_wrapper_.PauseRecorder(true);

            if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
            {
                Debug.LogError("暂停录像失败, 请重新尝试!");
                return;
            }
            else if (NTBaseCodeDefine.NT_ERC_OK == ret)
            {
                btn_pause_record_.GetComponentInChildren().text = "恢复录像";
            }
        }
        else
        {
            UInt32 ret = publisher_wrapper_.PauseRecorder(false);
            if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
            {
                Debug.LogError("恢复录像失败, 请重新尝试!");
                return;
            }
            else if (NTBaseCodeDefine.NT_ERC_OK == ret)
            {
                btn_pause_record_.GetComponentInChildren().text = "暂停录像";
            }
        }
    }

实时预览相关:

    public void btn_preview_Click()
    {
        if (btn_preview_.GetComponentInChildren().text.Equals("本地预览"))
        {
            if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
            {
                publisher_wrapper_.SetVideoPushType(video_push_type_);
            }

            if (publisher_wrapper_.StartPreview())
            {
                btn_preview_.GetComponentInChildren().text = "停止预览";
            }

            if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
            {
                StartCaptureAvData();
                coroutine_ = StartCoroutine(OnPostVideo());
            }
        }
        else
        {
            if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
            {
                StopCaptureAvData();

                if (coroutine_ != null)
                {
                    StopCoroutine(coroutine_);
                    coroutine_ = null;
                }
            }

            publisher_wrapper_.StopPreview();
            btn_preview_.GetComponentInChildren().text = "本地预览";
        }
    }

总结

Unity平台下RTMP推送、录像、轻量级RTSP服务,在虚拟仿真、医疗、教育等场景下,应用非常广泛。要实现低延迟,除了需要高效率的音视频数据采集,编码和数据投递外,还需要好的直播播放器支持。配合我们的SmartPlayer,可轻松实现毫秒级体验,满足绝大多数应用场景技术诉求。

你可能感兴趣的:(Unity,RTMP,RTSP直播推送播放,windows,unity,游戏引擎,实时音视频,大牛直播SDK,Unity,RTMP,Unity,RTSP)