TDA4VM EVM学习笔记（2）:基于v4l2用USB摄像头采集图像并显示

    本文要实现的功能是在A72上用USB摄像头采集图像数据（格式YUYV4:2:2），然后在DSP上将YUV422格式的图像转化为RGB格式的图像，之后在Display上显示该图像数据。

完整代码

1. Kernel：https://download.csdn.net/download/walker_bk/87747967

2. Use case：https://download.csdn.net/download/walker_bk/87747987

目录

1. 创建USB摄像头采集数据的Kernel

2. 组织各个Node并创建Graph

3. 运行和结果

4. 需要改进

---

1. 创建USB摄像头采集数据的Kernel

1.1 Linux上基于v4l2驱动使用USB摄像头

参考文章：v4l2接口解析和摄像头数据采集_v4l2摄像头采集_liujun3512159的博客-CSDN博客

v4l2官方例程：11.1. file: media/v4l/capture.c — The Linux Kernel documentation

将官方例程的process_image函数替换为

        FILE* fp = fopen("pic.yuv", "w");
        if(!fp)
        {
                printf("failed to open picture\n");
                return;
        }

        fwrite(p, 1, length, fp);
        fclose(fp);
        return;

在linux上编译并运行修改后的例程，程序将采集USB摄像头数据并将数据存储到文件pic.yuv中。使用以下Pyhon文件解析yuv数据并显示图像内容。按任意键退出显示。

import cv2
import numpy as np
 
#读取YUV格式图像文件
def read_yuv422(image_path, rows, cols):
    # 创建y分量
    img_y_1 = np.zeros((rows, int(cols/2)), np.uint8)
    img_y_2 = np.zeros((rows, int(cols / 2)), np.uint8)
    img_y = np.zeros((rows, cols), np.uint8)
 
    # 创建u分量
    img_u = np.zeros((rows, int(cols / 2)), np.uint8)
 
    # 创建v分量
    img_v = np.zeros((rows, int(cols / 2)), np.uint8)
 
    # 读取内存中数据
    with open(image_path, 'rb') as reader:
        for i in range(rows):
            for j in range(int(cols/2)):
 #               img_y_1[i, j] = ord(reader.read(1))
                img_y[i,2*j] = ord(reader.read(1))
                img_u[i, j] = ord(reader.read(1))
 #               img_y_2[i, j] = ord(reader.read(1))
                img_y[i,2*j+1] = ord(reader.read(1))
                img_v[i, j] = ord(reader.read(1))
 
#    for i in range(rows):
#        for j in range(int(cols/2)):
#            img_y[i, 2*j] = img_y_1[i, j]
#            img_y[i, 2*j+1] = img_y_2[i,j]
 
    return img_y, img_u, img_v
 
#把YUV格式数据转换为RGB格式
def yuv2rgb422(y, u, v):
 
    rows, cols = y.shape[:2]
    
    # 创建r,g,b分量
    r = np.zeros((rows, cols), np.uint8)
    g = np.zeros((rows, cols), np.uint8)
    b = np.zeros((rows, cols), np.uint8)
 
    for i in range(rows):
        for j in range(int(cols/2)):
            r[i, 2 * j] = max(0,min(255,y[i, 2 * j] + 1.402 * (v[i, j] - 128)))
            g[i, 2 * j] = max(0,min(255,y[i, 2 * j] - 0.34414 * (u[i, j] - 128) - 0.71414 * (v[i, j] - 128)))
            b[i, 2 * j] = max(0,min(255,y[i, 2 * j] + 1.772 * (u[i, j] - 128)))
 
            r[i, 2 * j+1] = max(0,min(255,y[i, 2 * j+1] + 1.402 * (v[i, j] - 128)))
            g[i, 2 * j+1] = max(0,min(255,y[i, 2 * j+1] - 0.34414 * (u[i, j] - 128) - 0.71414 * (v[i, j] - 128)))
            b[i, 2 * j+1] = max(0,min(255,y[i, 2 * j+1] + 1.772 * (u[i, j] - 128)))
 
    rgb = cv2.merge([b, g, r])
 
    return rgb, r, g, b
 
img_yuv=read_yuv422("pic.yuv",480,640)
 
#cv2.imshow("YUV_Y",img_yuv[0])#显示从文件提取出的Y分量

#img_yuv=cv2.imread("pic.yuv")
 
#img_rgb=cv2.cvtColor(img_yuv,cv2.COLOR_YUV2RGB_Y422)
img_rgb=yuv2rgb422(img_yuv[0],img_yuv[1],img_yuv[2])
 
cv2.imshow("RGB",img_rgb[0])
#cv2.imshow("RGB",img_rgb)
 
cv2.waitKey(0)

1.2 创建Custom Kernel，并将USB摄像头采集代码集成

1.2.1 安装PyTIOVX

请参考SDK路径下：tiovx/docs/user_guide/PYTIOVX.html

1.2.2 创建custom kernel的Python文件

from tiovx import *

code = KernelExportCode("UCAM_CAPTURE", Core.A72, "CUSTOM_APPLICATION_PATH")

kernel = Kernel("ucam_capture")

kernel.setParameter(Type.IMAGE, Direction.OUTPUT, ParamState.REQUIRED, "OUT")

kernel.setTarget(Target.A72_0)

code.export(kernel)
code.exportDiagram(kernel)

1.2.3 生成代码

在terminal中

 export CUSTOM_APPLICAION_PATH=‘Your Kernel Path’
 python 'Your python file'

1.2.4 修改代码，集成USB摄像头采集代码

1.2.4.1 在tivxUcamCaptureCreate中

static vx_status VX_CALLBACK tivxUcamCaptureCreate(
       tivx_target_kernel_instance kernel,
       tivx_obj_desc_t *obj_desc[],
       uint16_t num_params, void *priv_arg)
{
    vx_status status = (vx_status)VX_SUCCESS;
    /* < DEVELOPER_TODO: Uncomment if kernel context is needed > */
    #if 0
    tivxUcamCaptureParams *prms = NULL;
    #endif

    /* < DEVELOPER_TODO: (Optional) Add any target kernel create code here (e.g. allocating */
    /*                   local memory buffers, one time initialization, etc) > */
    if ( (num_params != TIVX_KERNEL_UCAM_CAPTURE_MAX_PARAMS)
        || (NULL == obj_desc[TIVX_KERNEL_UCAM_CAPTURE_OUT_IDX])
    )
    {
        status = (vx_status)VX_FAILURE;

    }
    else
    {


        /* < DEVELOPER_TODO: Uncomment if kernel context is needed > */
        #if 0
        prms = tivxMemAlloc(sizeof(tivxUcamCaptureParams), (vx_enum)TIVX_MEM_EXTERNAL);
        if (NULL != prms)
        {

        }
        else
        {
            status = (vx_status)VX_ERROR_NO_MEMORY;
            VX_PRINT(VX_ZONE_ERROR, "Unable to allocate local memory\n");
        }

        if ((vx_status)VX_SUCCESS == status)
        {
            tivxSetTargetKernelInstanceContext(kernel, prms,
                sizeof(tivxUcamCaptureParams));
        }
        else
        {
            status = (vx_status)VX_ERROR_NO_MEMORY;
            VX_PRINT(VX_ZONE_ERROR, "Unable to allocate local memory\n");
        }
        #endif
    }

    open_device ();
    init_device ();
    start_capturing ();

    return status;
}

1.2.4.2 在tivxUcamCaptureProcess中做如下修改

static vx_status VX_CALLBACK tivxUcamCaptureProcess(
       tivx_target_kernel_instance kernel,
       tivx_obj_desc_t *obj_desc[],
       uint16_t num_params, void *priv_arg)
{
    vx_status status = (vx_status)VX_SUCCESS;
    /* < DEVELOPER_TODO: Uncomment if kernel context is needed > */
    #if 0
    tivxUcamCaptureParams *prms = NULL;
    #endif
    tivx_obj_desc_image_t *out_desc;

    if ( (num_params != TIVX_KERNEL_UCAM_CAPTURE_MAX_PARAMS)
        || (NULL == obj_desc[TIVX_KERNEL_UCAM_CAPTURE_OUT_IDX])
    )
    {
        status = (vx_status)VX_FAILURE;
    }

    if((vx_status)VX_SUCCESS == status)
    {
        /* < DEVELOPER_TODO: Uncomment if kernel context is needed > */
        #if 0
        uint32_t size;
        #endif
        out_desc = (tivx_obj_desc_image_t *)obj_desc[TIVX_KERNEL_UCAM_CAPTURE_OUT_IDX];

        /* < DEVELOPER_TODO: Uncomment if kernel context is needed > */
        #if 0
        status = tivxGetTargetKernelInstanceContext(kernel,
            (void **)&prms, &size);
        if (((vx_status)VX_SUCCESS != status) || (NULL == prms) ||
            (sizeof(tivxUcamCaptureParams) != size))
        {
            status = (vx_status)VX_FAILURE;
        }
        #endif
    }

    if((vx_status)VX_SUCCESS == status)
    {

        void *out_target_ptr;

        out_target_ptr = tivxMemShared2TargetPtr(&out_desc->mem_ptr[0]);
        tivxCheckStatus(&status, tivxMemBufferMap(out_target_ptr,
           out_desc->mem_size[0], (vx_enum)VX_MEMORY_TYPE_HOST,
           (vx_enum)VX_WRITE_ONLY));



        {
            VXLIB_bufParams2D_t vxlib_out;
            uint8_t *out_addr = NULL;

            tivxInitBufParams(out_desc, &vxlib_out);
            tivxSetPointerLocation(out_desc, &out_target_ptr, &out_addr);

            /* call kernel processing function */

            /* < DEVELOPER_TODO: Add target kernel processing code here > */
   

            mainloop (out_target_ptr);


            /* kernel processing function complete */

        }
        tivxCheckStatus(&status, tivxMemBufferUnmap(out_target_ptr,
           out_desc->mem_size[0], (vx_enum)VX_MEMORY_TYPE_HOST,
            (vx_enum)VX_WRITE_ONLY));



    }

   
    return status;
}

值得注意的是，基于原capture.c，需要做些小的改动，即通过mainloop把捕获到的图像存储到kernel所指定的图像地址上

1.2.4.3 在tivxUcamCaptureDelete中

static vx_status VX_CALLBACK tivxUcamCaptureDelete(
       tivx_target_kernel_instance kernel,
       tivx_obj_desc_t *obj_desc[],
       uint16_t num_params, void *priv_arg)
{
    vx_status status = (vx_status)VX_SUCCESS;
    /* < DEVELOPER_TODO: Uncomment if kernel context is needed > */
    #if 0
    tivxUcamCaptureParams *prms = NULL;
    uint32_t size;
    #endif

    /* < DEVELOPER_TODO: (Optional) Add any target kernel delete code here (e.g. freeing */
    /*                   local memory buffers, etc) > */
    /* < DEVELOPER_TODO: Uncomment if kernel context is needed > */
    #if 0
    if ( (num_params != TIVX_KERNEL_UCAM_CAPTURE_MAX_PARAMS)
        || (NULL == obj_desc[TIVX_KERNEL_UCAM_CAPTURE_OUT_IDX])
    )
    {
        status = (vx_status)VX_FAILURE;
    }
    else
    {
        tivxGetTargetKernelInstanceContext(kernel, (void **)&prms, &size);

        if ((NULL != prms) &&
            (sizeof(tivxUcamCaptureParams) == size))
        {
            tivxMemFree(prms, size, (vx_enum)TIVX_MEM_EXTERNAL);
        }
    }
    #endif
    stop_capturing ();
    uninit_device ();
    close_device ();

    return status;
}

特别注意

a. 要在核上注册

b. 如果要使用虚拟节点，需要添加Meta数据

2. 组织各个Node并创建Graph

2.1 创建graph

    vx_status status = VX_SUCCESS;
    APP_PRINTF("ucam_disp: Creating graph ... \n");

    /* Create OpenVx Graph */
    obj->graph = vxCreateGraph(obj->context);
    APP_ASSERT_VALID_REF(obj->graph)
    vxSetReferenceName((vx_reference)obj->graph, "Ucam_Display");

    obj->cap_img = vxCreateVirtualImage(obj->graph, CAP_IMG_WIDTH, CAP_IMG_HEIGHT, VX_DF_IMAGE_YUYV);
    APP_ASSERT_VALID_REF(obj->cap_img)
   
    obj->disp_img = vxCreateVirtualImage(obj->graph, CAP_IMG_WIDTH, CAP_IMG_HEIGHT, VX_DF_IMAGE_RGB);
    APP_ASSERT_VALID_REF(obj->disp_img)
    
    /* ucam node */
    obj->ucam_cap_node = tivxUcamCaptureNode(obj->graph, obj->cap_img);
    
    APP_ASSERT_VALID_REF(obj->ucam_cap_node)
    vxSetNodeTarget(obj->ucam_cap_node, VX_TARGET_STRING, TIVX_TARGET_A72_0);


    obj->color_cvt_node = vxColorConvertNode(obj->graph, obj->cap_img, obj->disp_img);
    /*vx_node VX_API_CALL vxColorConvertNode 	( 	vx_graph  	graph,
		vx_image  	input,
		vx_image  	output 
	) 		*/
    APP_ASSERT_VALID_REF(obj->color_cvt_node)
    
    vxSetNodeTarget(obj->disp_node, VX_TARGET_STRING, TIVX_TARGET_DSP1);

    obj->disp_node = tivxDisplayNode(obj->graph, obj->disp_params_obj, obj->disp_img);
    APP_ASSERT_VALID_REF(obj->disp_node)

    vxSetNodeTarget(obj->disp_node, VX_TARGET_STRING, TIVX_TARGET_DISPLAY1);
    vxSetReferenceName((vx_reference)obj->disp_node, "DisplayNode");

 2.2 按指定时间间隔运行graph

        cur_time = tivxPlatformGetTimeInUsecs();
        APP_PRINTF("ucam_disp: Running display graph ...\n");


        if(status == VX_SUCCESS)
        {
            status = vxProcessGraph(obj->graph);
        }
        APP_PRINTF("ucam_disp: Running display graph ... Done.\n");
        cur_time = tivxPlatformGetTimeInUsecs() - cur_time;
        /* convert to msecs */
        cur_time = cur_time/1000;

        if(cur_time < obj->delay_in_msecs)
        {
            tivxTaskWaitMsecs(obj->delay_in_msecs - cur_time);
        }

        /* user asked to stop processing */
        if(obj->stop_task || (status != VX_SUCCESS))
        {
            break;
        }

3. 运行和结果

在做完以上的修改，编译生成可执行代码，将生成的代码放在目标板上，就可以在屏幕上看到USB摄像头所采集的数据了。

注意：由于生成了自定义的kernel，需要将新生成的libtivision_apps.so也更新到目标板上。

4. 需要改进

4.1. 当前USB摄像头所有的参数都是直接写在程序中的，可以通过Parameter来配置kernel的参数。

【函数说明】

◆ tivxDisplayNode()

VX_API_ENTRY vx_node VX_API_CALL tivxDisplayNode	(	vx_graph	graph,
		vx_user_data_object	configuration,
		vx_image	image
	)

[Graph] Creates a DSS Display Node.

Parameters

[in]	graph	The reference to the graph.
[in]	configuration	The input user data object of a single display params structure of type tivx_display_params_t.
[in]	image	The input image in one of the below formats: VX_DF_IMAGE_RGB, VX_DF_IMAGE_RGBX, VX_DF_IMAGE_BGRX, VX_DF_IMAGE_UYVY, VX_DF_IMAGE_NV12, VX_DF_IMAGE_U16, VX_DF_IMAGE_U8 or TIVX_DF_IMAGE_RGB565.

See also

TIVX_KERNEL_DISPLAY_NAME

Returns

vx_node.

Return values

vx_node

A node reference. Any possible errors preventing a successful creation should be checked using vxGetStatus

Display Kernel 's Operation Mode
Display kernel can be run in two modes: it can be used to display only one buffer repeatedly or application provides a new buffer every VSYNC. In case of one buffer, kernel needs to maintain a local copy of the buffer.
#define	TIVX_KERNEL_DISPLAY_ZERO_BUFFER_COPY_MODE   ((uint32_t) 0U)
	Display Kernel does not need to maintain buffer copy i.e. application gives new frame at every VSYNC.
#define	TIVX_KERNEL_DISPLAY_BUFFER_COPY_MODE   ((uint32_t) 1U)
	Display Kernel needs to maintain buffer copy.

【参考信息】

1. Ti SDK Linux version 8.0.5： Processor SDK Linux Software Developer’s Guide — Processor SDK Linux for J721e Documentation
2. Ti SDK RTOS version 8.0.5： Processor SDK RTOS J721E — Processor SDK RTOS J721E
3. Tiovx：TIOVX User Guide: Overview