划分训练集、测试集，制作自己的数据集

从文件路径读取图片，将图片的数组存为npz格式。

数据集：人脸卡通表情FERG数据，包含6个卡通人物，每个卡通人物7个表情，每张图片256*256的png格式，数据集压缩包大小2.95GB。

图片路径tree：

数据集的85%为训练集，15%为测试集

代码如下：

'''
id存索引,ep存索引，image存矩阵
将图片预处理为numpy数组，存储于.npz

'''
import time
from numpy import *
import cv2
from keras.utils.np_utils import *
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"]="3"
from keras.preprocessing.image import img_to_array
import matplotlib.pyplot as plt
ratio = 0.15  #按照ratio划分测试集、训练集，ratio:测试集的比例 15%
ep = ['anger', 'disgust', 'fear', 'joy', 'neutral', 'sadness', 'surprise']
id=['aia','bonnie','jules','malcolm','mery','ray']

dataset_dir = 'E:/code/dataset/FERG_DB_256/'
def get_file(file_dir,id_item):
    anger = []
    label_anger = []
    disgusted = []
    label_disgusted = []
    fearful = []
    label_fearful = []
    joy = []
    label_joy = []
    neutral = []
    label_neutral = []
    sadness = []
    label_sadness = []
    surprised = []
    label_surprised = []

    # 定义存放测试集、训练集图片地址、标签的list
    x_train = []
    ep_train = []

    x_test = []
    ep_test = []
    count = 0

    # step1：获取路径下所有的图片路径名，存放到
    for file in os.listdir(file_dir + '/'+id_item+'_anger'):
        if not file.endswith('.png'):
            continue
        count = count + 1
        if count > 1000:
            break
        anger.append(file_dir + '/'+id_item + '_anger' + '/' + file)
        label_anger.append(0)
    count = 0
    for file in os.listdir(file_dir + '/'+id_item+'_disgust'):
        if not file.endswith('.png'):
            continue
        count = count + 1
        if count > 1000:
            break
        disgusted.append(file_dir + '/'+id_item+'_disgust' + '/' + file)
        label_disgusted.append(1)
    count = 0
    for file in os.listdir(file_dir+ '/'+id_item+'_fear'):
        if not file.endswith('.png'):
            continue
        count = count + 1
        if count > 1000:
            break
        fearful.append(file_dir+ '/'+id_item+'_fear' + '/' + file)
        label_fearful.append(2)
    count = 0
    for file in os.listdir(file_dir+ '/'+id_item+'_joy'):
        if not file.endswith('.png'):
            continue
        count = count + 1
        if count > 1000:
            break
        joy.append(file_dir+ '/'+id_item+'_joy'+ '/' + file)
        label_joy.append(3)
    count = 0
    for file in os.listdir(file_dir+ '/'+id_item+'_neutral'):
        if not file.endswith('.png'):
            continue
        count = count + 1
        if count > 1000:
            break
        neutral.append(file_dir+ '/'+id_item+'_neutral' + '/' + file)
        label_neutral.append(4)
    count = 0
    for file in os.listdir(file_dir+ '/'+id_item+'_sadness'):
        if not file.endswith('.png'):
            continue
        count = count + 1
        if count > 1000:
            break
        sadness.append(file_dir+ '/'+id_item+'_sadness' + '/' + file)
        label_sadness.append(5)
    count = 0
    for file in os.listdir(file_dir+ '/'+id_item+'_surprise'):
        if not file.endswith('.png'):
            continue
        count = count + 1
        if count > 1000:
            break
        surprised.append(file_dir+ '/'+id_item+'_surprise' + '/' + file)
        label_surprised.append(6)

    imageall_list = np.hstack((anger, disgusted, fearful, joy, neutral,sadness, surprised))
    labelall_list = np.hstack((label_anger, label_disgusted, label_fearful, label_joy,label_neutral, label_sadness, label_surprised))
    #按照ratio划分测试集、训练集，ratio:测试集的比例
    ep_len = [len(anger),len(disgusted) ,len(fearful) ,len(joy) , len(neutral), len(sadness), len(surprised)]
    pre_index = 0
    for ep_index in ep_len:
        image_list = imageall_list[pre_index:pre_index + ep_index]
        label_list = labelall_list[pre_index:pre_index + ep_index]
        pre_index = ep_index + pre_index
        # 利用shuffle，转置、随机打乱
        temp = np.array([image_list, label_list])  # 转换成2维矩阵
        temp = temp.transpose()  # 转置
        np.random.shuffle(temp)  # 按行随机打乱顺序函数
        # 将所有的img和lab转换成list
        all_image_list = list(temp[:, 0])  # 取出第0列数据，即图片路径
        all_label_list = list(temp[:, 1])  # 取出第1列数据，即图片标签
        all_label_list = [int(float(i)) for i in all_label_list]  # 转换成int数据类型

     # 将所得List分为两部分，一部分用来训练tra，一部分用来测试val
        n_sample = len(all_label_list)
        n_val = int(math.ceil(n_sample * ratio))  # 测试样本数, ratio是测试集的比例
        n_train = n_sample - n_val  # 训练样本数
        # print("%s的训练样本数：%s" % (ep[ep_len.index(ep_index)], str(n_train)))
        tra_images = all_image_list[0:n_train]
        tra_labels = all_label_list[0:n_train]
        val_images = all_image_list[n_train:]
        val_labels = all_label_list[n_train:]

        #将训练集和测试集的图片地址和标签存入列表
        x_train.extend(tra_images)
        ep_train.extend(tra_labels)
        x_test.extend(val_images)
        ep_test.extend(val_labels)
    id_train = [id.index(id_item) for k in range(len(ep_train))]
    id_test = [id.index(id_item) for k in range(len(ep_test))]
    return x_train, id_train,ep_train, x_test, id_test, ep_test

def text_save(filename, data):
    file = open(filename,'a')
    for i in range(len(data)):
        s = str(data[i]).replace('[','').replace(']','')#去除[],这两行按数据不同，可以选择
        s = s.replace("'",'').replace(',','') +'\n'   #去除单引号，逗号，每行末尾追加换行符
        file.write(s)
    file.close()


def listdata(dataset_dir):
    train_data = []
    test_data = []
    train_ep =[]
    train_id = []
    test_ep = []
    test_id = []
    global  id
    count = 0
    norm_size = 64
    for id_item in id:
        train_dir = os.path.join(dataset_dir,id_item)
        x_train, id_train,ep_train, x_test, id_test, ep_test = get_file(train_dir,id_item)
        train_data.extend(x_train)
        train_id.extend(id_train)
        train_ep.extend(ep_train)
        test_data.extend(x_test)
        test_id.extend(id_test)
        test_ep.extend(ep_test)
    text_save('./datapath/train_data.txt', train_data)
    text_save('./datapath/test_data.txt', test_data)
    print("训练集一共有%d张图\n" % len(train_data))
    print("测试集一共有%d张图\n" % len(test_data))
    return  train_data,test_data,train_ep,test_ep,train_id,test_id


def numpydata(norm_size=64,dir=dataset_dir):
    train_data, test_data, train_ep, test_ep, train_id, test_id = listdata(dataset_dir)
    train_image_list = []
    test_image_list = []
    for m in range(len(train_data)):
        image = cv2.imread(train_data[m])
        image = cv2.resize(image, (norm_size, norm_size))
        b, g, r = cv2.split(image)
        image = cv2.merge([r, g, b])
        image = img_to_array(image)
        train_image_list.append(image)

    for m in range(len(test_data)):
        image1 = cv2.imread(test_data[m])
        image1 = cv2.resize(image1, (norm_size, norm_size))
        b, g, r = cv2.split(image1)
        image1 = cv2.merge([r, g, b])
        image1 = img_to_array(image1)
        test_image_list.append(image1)

    # 标准化：提高模型预测精准度，加快收敛
    train_image_list = np.array(train_image_list, dtype="float") / 255.0
    test_image_list = np.array(test_image_list, dtype="float") / 255.0

    # convert the labels from integers to vectors one-hot编码
    train_id_list = to_categorical(train_id, num_classes=6)
    test_id_list = to_categorical(test_id, num_classes=6)
    train_ep_list = to_categorical(train_ep, num_classes=7)
    test_ep_list = to_categorical(test_ep, num_classes=7)
    # 第一运行 把处理好的数据保存下来
    np.save('./numpyrgbdata/train_image', train_image_list)
    np.save('./numpyrgbdata/test_image', test_image_list)
    np.save('./numpyrgbdata/train_id_list', train_id_list)
    np.save('./numpyrgbdata/test_id_list', test_id_list)
    np.save('./numpyrgbdata/train_ep_list', train_ep_list)
    np.save('./numpyrgbdata/test_ep_list', test_ep_list)

    print('train_image_list.shape: ',train_image_list.shape)
    print('train_id_list.shape: ',train_id_list.shape)
    print('train_ep_list.shape: ',train_ep_list.shape)

    print('test_image_list.shape: ',test_image_list.shape)

    return train_image_list, train_id_list, train_ep_list,test_image_list, test_id_list,test_ep_list

if __name__ == '__main__':
    start_time = time.time()
    numpydata(norm_size=64, dir=dataset_dir)
    process_time = time.time() - start_time
    print("Elapsed: %s " % (process_time))