【工具代码】将Mnist和Cifar-10数据集转换为numpy格式

Cifar-10:

import pickle
import glob
import numpy as np


def Dataloader():
	data_list = glob.glob("data_batch_*")

	for data in data_list:
		data = pickle.load(open(data, 'rb'), encoding='bytes')
		labels, data, filenames = data[b'labels'], data[b'data'], data[b'filenames']
		labels, data = map(np.array, [labels, data])
		try:
			Data = np.r_[Data, data]
			Labels = np.r_[Labels, labels]
		except:
			Data = data
			Labels = labels
	
	np.save("data/data.npy", Data)
	np.save("data/label.npy", Labels)
			
if __name__ == "__main__":
	Dataloader()

Mnist:

import numpy as np
import struct
 
def loadImageSet(filename):
 
    binfile = open(filename, 'rb') # 读取二进制文件
    buffers = binfile.read()
 
    head = struct.unpack_from('>IIII', buffers, 0) # 取前4个整数，返回一个元组
 
    offset = struct.calcsize('>IIII')  # 定位到data开始的位置
    imgNum = head[1]
    width = head[2]
    height = head[3]
 
    bits = imgNum * width * height  # data一共有60000*28*28个像素值
    bitsString = '>' + str(bits) + 'B'  # fmt格式：'>47040000B'
 
    imgs = struct.unpack_from(bitsString, buffers, offset) # 取data数据，返回一个元组
 
    binfile.close()
    imgs = np.reshape(imgs, [imgNum, width * height]) # reshape为[60000,784]型数组
 
    return imgs,head
 
 
def loadLabelSet(filename):
 
    binfile = open(filename, 'rb') # 读二进制文件
    buffers = binfile.read()
 
    head = struct.unpack_from('>II', buffers, 0) # 取label文件前2个整形数
 
    labelNum = head[1]
    offset = struct.calcsize('>II')  # 定位到label数据开始的位置
 
    numString = '>' + str(labelNum) + "B" # fmt格式：'>60000B'
    labels = struct.unpack_from(numString, buffers, offset) # 取label数据
 
    binfile.close()
    labels = np.reshape(labels, [labelNum]) # 转型为列表(一维数组)
 
    return labels,head
 
 
if __name__ == "__main__":
    file1= './train-images.idx3-ubyte'
    file2= './train-labels.idx1-ubyte'
 
    imgs,data_head = loadImageSet(file1)
 
    labels,labels_head = loadLabelSet(file2)

    np.save("data/data.npy", imgs)
    np.save("data/label.npy", labels)