16 利用scipy.misc等库对jpg以及png等图像数据预处理（用于深度学习喂数据）

import os
import glob
image_dir = "D:\\data\\ADEChallengeData2016"
file_glob = os.path.join(image_dir, "images", "training", '*.' + 'jpg')
file_list = []
file_list.extend(glob.glob(file_glob))

file_list_test = file_list[0:10]

import random
random.shuffle(file_list_test)

import numpy as np
import scipy.misc as misc

def read_and_transform(filename):
    image = misc.imread(filename)
    resize_image = misc.imresize(image,[224, 224], interp='nearest')
    return np.array(resize_image)

images = np.array([read_and_transform(filename) for filename in file_list_test])

def find_annotations_image_path(image_path):
    return image_path.replace("images","annotations").replace("jpg","png")

annotations_file_list_test = [find_annotations_image_path(image_path) for image_path in file_list_test]

annotations = np.array([np.expand_dims(read_and_transform(filename), axis=3) for filename in annotations_file_list_test])

batch_offset = 0

def next_batch(batch_size,batch_offset,images,annotations,epochs_completed):
    start = batch_offset
    batch_offset += batch_size
    if batch_offset > images.shape[0]:
        # Finished epoch
        epochs_completed += 1
        print("****************** Epochs completed: " + str(epochs_completed) + "******************")
        # Shuffle the data
        perm = np.arange(images.shape[0])
        np.random.shuffle(perm)
        images = images[perm]
        annotations = annotations[perm]
        # Start next epoch
        start = 0
        batch_offset = batch_size

    end = batch_offset
    print start,end
#     return images[start:end], self.annotations[start:end]

2 测试

2.1 测试next_batch送数据

next_batch(2,0,images,annotations,0)

0 2

next_batch(2,2,images,annotations,0)

2 4

next_batch(2,10,images,annotations,0)

****************** Epochs completed: 1******************
0 2

2.2 测试shuffle打乱数据

perm = np.arange(images.shape[0])

print perm

[0 1 2 3 4 5 6 7 8 9]

np.random.shuffle(perm)

print perm

[5 3 9 4 8 6 2 1 7 0]

images[0]

array([[[ 92,  57,  27],
        [181, 145, 113],
        [227, 189, 153],
        ..., 
        [100,  77,  71],
        [200, 177, 171],
        [194, 174, 167]],

       [[ 82,  45,  18],
        [ 85,  46,  17],
        [176, 136, 101],
        ..., 
        [200, 177, 171],
        [192, 172, 165],
        [180, 160, 153]],

       [[ 88,  47,  25],
        [ 93,  53,  28],
        [131,  90,  60],
        ..., 
        [202, 182, 175],
        [201, 181, 174],
        [163, 144, 137]],

       ..., 
       [[ 99,  48,  31],
        [ 84,  33,  16],
        [ 81,  30,  13],
        ..., 
        [170, 134, 108],
        [166, 130, 104],
        [163, 127, 101]],

       [[ 67,  16,   0],
        [ 98,  47,  30],
        [ 97,  46,  29],
        ..., 
        [168, 132, 106],
        [173, 137, 111],
        [168, 132, 106]],

       [[ 62,  11,   0],
        [ 83,  32,  15],
        [ 65,  14,   0],
        ..., 
        [169, 133, 107],
        [164, 128, 102],
        [165, 129, 103]]], dtype=uint8)

images = images[perm]
annotations = annotations[perm]

images[0]

array([[[ 50,  75,  80],
        [ 50,  75,  80],
        [ 51,  76,  81],
        ..., 
        [135, 165, 176],
        [133, 163, 174],
        [133, 163, 174]],

       [[ 50,  75,  80],
        [ 50,  75,  80],
        [ 51,  76,  81],
        ..., 
        [135, 165, 176],
        [133, 163, 174],
        [133, 163, 174]],

       [[ 49,  74,  79],
        [ 49,  74,  79],
        [ 50,  75,  80],
        ..., 
        [135, 165, 176],
        [133, 163, 174],
        [133, 163, 174]],

       ..., 
       [[  4,  16,  12],
        [  4,  16,  12],
        [  4,  16,  12],
        ..., 
        [176, 196, 205],
        [177, 197, 206],
        [178, 198, 207]],

       [[  4,  16,  12],
        [  4,  16,  12],
        [  4,  16,  12],
        ..., 
        [175, 195, 204],
        [176, 196, 205],
        [176, 196, 205]],

       [[  4,  16,  12],
        [  4,  16,  12],
        [  4,  16,  12],
        ..., 
        [177, 197, 206],
        [176, 196, 205],
        [176, 196, 205]]], dtype=uint8)

2.3 测试np.expand_dims

annotations1 = np.array([read_and_transform(filename) for filename in annotations_file_list_test])

annotations1.shape

(10L, 224L, 224L)

annotations.shape

(10L, 224L, 224L, 1L)

annotations_file_list_test[0]

'D:\\data\\ADEChallengeData2016\\annotations\\training\\ADE_train_00000007.png'

file_list_test[0]

'D:\\data\\ADEChallengeData2016\\images\\training\\ADE_train_00000007.jpg'

image_path = annotations_file_list_test[0]

annotations_image = read_and_transform(image_path)

annotations_image.shape

(224L, 224L)

annotations_image

array([[  0,   1,   1, ...,   6,   6,   6],
       [135, 135, 135, ...,   6,   6,   6],
       [  0, 135, 135, ...,   6,   6,   6],
       ..., 
       [  0,  11,  11, ...,   4,   4,   4],
       [  0,  11,  11, ...,   4,   4,   4],
       [  0,  11,  11, ...,   4,   4,   4]], dtype=uint8)

np.expand_dims(annotations_image,axis=3)

array([[[  0],
        [  1],
        [  1],
        ..., 
        [  6],
        [  6],
        [  6]],

       [[135],
        [135],
        [135],
        ..., 
        [  6],
        [  6],
        [  6]],

       [[  0],
        [135],
        [135],
        ..., 
        [  6],
        [  6],
        [  6]],

       ..., 
       [[  0],
        [ 11],
        [ 11],
        ..., 
        [  4],
        [  4],
        [  4]],

       [[  0],
        [ 11],
        [ 11],
        ..., 
        [  4],
        [  4],
        [  4]],

       [[  0],
        [ 11],
        [ 11],
        ..., 
        [  4],
        [  4],
        [  4]]], dtype=uint8)

image_path.replace("images","annotations").replace("jpg","png")

'D:\\data\\ADEChallengeData2016\\annotations\\training\\ADE_train_00000007.png'

annotations_image_path = image_path.replace("images","annotations").replace("jpg","png")

annotations_image_dir = "D:\\data\\ADEChallengeData2016"
annotations_file_glob = os.path.join(image_dir, "annotations", "training", '*.' + 'png')
annotations_file_list = []
annotations_file_list.extend(glob.glob(file_glob))

len(annotations_file_list)

20210

annotations = np.array([np.expand_dims(read_and_transform(filename), axis=3) for filename in file_list_test])

annotations.shape

(10L, 224L, 224L, 3L, 1L)

len(images)

10

image_array = read_and_transform(file_list_test[0])

len(image_array)

224

image_array[0][:][:].shape

(224L, 3L)

image_array.shape

(224L, 224L, 3L)

2.4 测试misc的imread以及imresize

image = misc.imread(file_list_test[0])

image.shape

(735L, 512L, 3L)

resize_image = misc.imresize(image,[224, 224], interp='nearest')

import matplotlib.pyplot as plt
plt.figure()
plt.imshow(image)
plt.show()

import matplotlib.pyplot as plt
plt.figure()
plt.imshow(resize_image)
plt.show()

misc.imshow(image)

---------------------------------------------------------------------------

RuntimeError                              Traceback (most recent call last)

 in ()
----> 1 misc.imshow(image)


D:\software\Anaconda2\lib\site-packages\scipy\misc\pilutil.pyc in imshow(arr)
    440     os.unlink(fname)
    441     if status != 0:
--> 442         raise RuntimeError('Could not execute image viewer.')
    443 
    444 


RuntimeError: Could not execute image viewer.