视频多标签分类Conv3D实现

实现流程简要概括:

1. 抓取样本videos
2. 视频内容切片为frame（每帧或每几帧）
3. Conv3D神经网络（视频信息嵌入）
4. 全连层 sigmoid+binary CE 多标签分类

优点是实现端对端预测，可直接用于下游任务：分类、打标等等
缺点是未考虑frame的时序信息，切分类结果通常较general，且依赖大量样本

1. Frame 提取方式

import cv2
import numpy as np
import os

def mkdir(path):
    folder = os.path.exists(path)
    if not folder:
        os.makedirs(path)
        
def v2frame(videoPath, svPath, num_frame=450, size=120):
	# 保留所有帧，每个视频取450frame，不足的以黑画面补全
    cap = cv2.VideoCapture(videoPath)
    suc, frame = cap.read()
    frame_count = 0
    while(frame_count<num_frame):
        if(suc):
            frame=cv2.resize(frame,(size,size),interpolation=cv2.INTER_AREA)
        else:
            frame = np.zeros((size,size,3), np.uint8)
        cv2.imwrite(svPath+'/%d.jpg' % frame_count, frame)#, params)
        if(suc):
            suc, frame = cap.read()
        frame_count += 1
    cap.release()
    
filenames = tuple(os.listdir("videos"))
filenames = [x.split(".")[0] for x in filenames]
for filename in filenames:
    videoPath = "videos/"+filename+".mp4"
    svPath = "frames/"+filename
    mkdir(svPath)
    unlock_mv(videoPath, svPath)

2. Frame转array

def image2array(image_path, image_num=450):
    image_name = image_path + "0.jpg"
    Img = cv2.imread(image_name)
    Img = Img.reshape(1,Img.shape[0], Img.shape[1], Img.shape[2])
    Img_batch = Img
    for i in range(1, image_num):
        image_name = image_path + str(i) + ".jpg"
        Img = cv2.imread(image_name)
        Img =Img.reshape(1,Img.shape[0], Img.shape[1], Img.shape[2])
        Img_batch = np.concatenate((Img_batch, Img), axis=0)
    return Img_batch

filenames = tuple(os.listdir("videos"))
filenames = [x.split(".")[0] for x in filenames if x.split(".")[0]!='']
for filename in filenames:
    image_path = "frames/"+filename+"/"
    sv_path = "cube/"+filename+".npy"
    cube = image2array(image_path)
    np.save(sv_path, cube)

3. Conv3D网络的实现

def cnn3d(n_classes, input_shape):
    # Create the model
    main_input = Input(shape=input_shape)
    x=Conv3D(8,(3,3,3),activation='relu',strides=(1,1,2),padding="same")(main_input)
    x=MaxPooling3D(pool_size=(2,2,2), strides=2)(x) 

    x=Conv3D(16,(5,5,5),activation='relu',strides=(2,2,2),padding="same")(x)
    x=MaxPooling3D(pool_size=(2,2,2), strides=2)(x) 

    x=Conv3D(16,(3,3,3),activation='relu',strides=(1,1,1),padding="same")(x)
    x=MaxPooling3D(pool_size=(2,2,2), strides=2)(x) 

    x=Conv3D(8,(3,3,3),activation='relu',strides=(1,1,1),padding="same")(x)
    x=MaxPooling3D(pool_size=(2,2,2), strides=2)(x) 

    x=Conv3D(128,(3,3,3),activation='relu',strides=(1,1,1),padding="same")(x)
    x=MaxPooling3D(pool_size=(2,2,2), strides=2)(x) 

    x = Flatten()(x)

    x = Dense(512)(x)
    x = LeakyReLU(alpha=Leaky_alpha)(x)
    x = Dropout(dropout)(x)
    
    x = Dense(256)(x)
    x = LeakyReLU(alpha=Leaky_alpha)(x)
    x = Dropout(dropout)(x)

    output = Dense(n_classes, activation='sigmoid', name='output')(x)
    model = Model(inputs=main_input, outputs=output)
    
    # Compile the model
    model.compile(loss=keras.losses.binary_crossentropy,
                  optimizer=keras.optimizers.Adam(lr=0.001),
                  metrics=['accuracy'])
    
    return model

model = cnn3d(n_classes, input_shape)
# checkpoint
filepath="checkpoints/conv3d_best.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='auto')
callbacks_list = [checkpoint]

# Parameters
n_classes = y_train.shape[1]
batch_size = 4
n_epochs = 3
validation_split = 0.1
verbosity = 1
input_shape =(450, 120, 120, 3)

# Fit the model
history = model.fit(X_train, y_train,
                    batch_size=batch_size,
                    epochs=n_epochs,
                    verbose=verbosity,
                    validation_split=validation_split, 
                    callbacks=callbacks_list)