theano中的concolutional_mlp.py学习

 

(1) evaluate _lenet5中的导入数据部分

 1     # 导入数据集，该函数定义在logistic_sgd中，返回的是一个list
 2     datasets = load_data(dataset)
 3 
 4     # 从list中提取三个元素，每个元素都是一个tuple（每个tuple含有2个元素，分别为images数据和label数据）
 5     train_set_x, train_set_y = datasets[0] #训练集
 6     valid_set_x, valid_set_y = datasets[1] #校验集
 7     test_set_x, test_set_y = datasets[2]   #测试集
 8 
 9 
10     # 训练集、校验集、测试集分别含有的样本个数
11     n_train_batches = train_set_x.get_value(borrow=True).shape[0]
12     n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
13     n_test_batches = test_set_x.get_value(borrow=True).shape[0]
14     # 训练集、校验集、测试集中包含的minibatch个数（每个iter，只给一个minibatch，而不是整个数据集）
15     n_train_batches /= batch_size
16     n_valid_batches /= batch_size
17     n_test_batches /= batch_size

（2）evaluate _lenet5中的building model部分

  1     # 首先，定义一些building model用到的符号变量
  2     index = T.lscalar()  # 用于指定具体哪个minibatch的指标
  3 
  4     # start-snippet-1
  5     x = T.matrix('x')   # 存储图像的像素数据
  6     y = T.ivector('y')  # 存储每幅图像对应的label
  7 
  8     # 开始build model
  9     print '... building the model'
 10 
 11     # 将输入的数据(batch_size, 28 * 28)reshape为4D tensor（batch_size是每个mini-batch包含的image个数）
 12     layer0_input = x.reshape((batch_size, 1, 28, 28))
 13 
 14     # 构造第一个卷积层
 15     # （1）卷积核大小为5*5、个数为nkerns[0]、striding =1，padding=0
 16     #     输出的feature map大小为：(28-5+1 , 28-5+1) = (24, 24)
 17     # （2）含有max-pooling，pooling的大小为2*2、striding =1，padding=0
 18     #     输出的map大小为：(24/2, 24/2) = (12, 12)
 19     # （3）综上，第一个卷积层输出的feature map为一个4D tensor，形状为：(batch_size, nkerns[0], 12, 12)
 20     layer0 = LeNetConvPoolLayer(
 21         rng,
 22         input=layer0_input,
 23         image_shape=(batch_size, 1, 28, 28),
 24         filter_shape=(nkerns[0], 1, 5, 5),
 25         poolsize=(2, 2)
 26     )
 27 
 28     # 构造第二个卷积层，卷积核大小为5*5
 29     # （1）卷积核大小为5*5、个数为nkerns[0]、striding =1，padding=0
 30     #     输出的feature map大小为：(12-5+1, 12-5+1) = (8, 8)
 31     # （2）含有max-pooling，pooling的大小为2*2、striding =1，padding=0
 32     #     输出的map大小为：(8/2, 8/2) = (4, 4)
 33     # （3）综上，第二个卷积层输出的feature map为一个4D tensor，形状为：(batch_size, nkerns[1], 4, 4)
 34     layer1 = LeNetConvPoolLayer(
 35         rng,
 36         input=layer0.output,
 37         image_shape=(batch_size, nkerns[0], 12, 12),
 38         filter_shape=(nkerns[1], nkerns[0], 5, 5),
 39         poolsize=(2, 2)
 40     )
 41 
 42     # 将第二个卷积层的输出map（形状为(batch_size, nkerns[1], 4, 4)）转化为一个matrix的形式
 43     # 该矩阵的形状为：(batch_size, nkerns[1] * 4 * 4)，每一行为一个图形对应的feature map
 44     layer2_input = layer1.output.flatten(2)
 45 
 46     # 第一个全链接层
 47     # （1）输入的大小固定，即第二个卷积层的输出
 48     # （2）输出大小自己选的，这里选定为500
 49     # （3）sigmoid函数为tan函数
 50     layer2 = HiddenLayer(
 51         rng,
 52         input=layer2_input,
 53         n_in=nkerns[1] * 4 * 4,
 54         n_out=500,
 55         activation=T.tanh
 56     )
 57 
 58     # 输出层，即逻辑回归层
 59     layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
 60 
 61     # 代价函数的计算
 62     cost = layer3.negative_log_likelihood(y)
 63 
 64     # 测试model，输入为具体要测试的test集中的某个mini-batch
 65     # 输出为训练得到的model在该mini-batch上的error
 66     test_model = theano.function(
 67         [index],
 68         layer3.errors(y),
 69         givens={
 70             x: test_set_x[index * batch_size: (index + 1) * batch_size],
 71             y: test_set_y[index * batch_size: (index + 1) * batch_size]
 72         }
 73     )
 74 
 75     # 校验model，输入为具体要测试的校验集中的某个mini-batch
 76     # 输出为训练得到的model在该mini-batch上的error
 77     validate_model = theano.function(
 78         [index],
 79         layer3.errors(y),
 80         givens={
 81             x: valid_set_x[index * batch_size: (index + 1) * batch_size],
 82             y: valid_set_y[index * batch_size: (index + 1) * batch_size]
 83         }
 84     )
 85 
 86     # 创建一个list，该list存放的是该CNN网络的所有待利用梯度下降法优化的参数
 87     params = layer3.params + layer2.params + layer1.params + layer0.params
 88 
 89     # 创建一个list，该list存放的是代价函数对该CNN网络的所有待利用梯度下降法优化的参数的梯度
 90     grads = T.grad(cost, params)
 91 
 92     # 为train模型创建更新规则，即创建一个list，自动更新params、grads中每一组值
 93     updates = [
 94         (param_i, param_i - learning_rate * grad_i)
 95         for param_i, grad_i in zip(params, grads)
 96     ]
 97 
 98     # 训练model，输入为具体要训练集中的某个mini-batch
 99     # 输出为训练得到的model在该mini-batch上的error
100     train_model = theano.function(
101         [index],
102         cost,
103         updates=updates,
104         givens={
105             x: train_set_x[index * batch_size: (index + 1) * batch_size],
106             y: train_set_y[index * batch_size: (index + 1) * batch_size]
107         }
108     )

（3）Lenet-5中的training model部分

 1     # 开始训练模型
 2     print '... training'
 3 
 4     # 定义一些进行early-stopping的相关参数
 5     # look as this many examples regardless
 6     patience = 10000
 7     # wait this much longer when a new best is found
 8     patience_increase = 2
 9     # a relative improvement of this much is considered significant
10     improvement_threshold = 0.995
11     # go through this many minibatche before checking the network on the validation set; in this case we check every epoch
12     validation_frequency = min(n_train_batches, patience / 2)
13 
14     # 训练过程中需要的其他参数
15     best_validation_loss = numpy.inf
16     best_iter = 0
17     test_score = 0.
18     start_time = timeit.default_timer()
19 
20     epoch = 0
21     done_looping = False
22 
23     while (epoch < n_epochs) and (not done_looping):
24 
25         #epoch次数增加1，每轮epoch，利用所有组mini-batch进行一次模型训练
26         # 每轮epoch，整体的迭代次数iter增加n_train_batches次
27         epoch = epoch + 1
28 
29         # 对于整个训练集中的第minibatch_index 个mini-batch
30         # minibatch_index=0,1,...,n_train_batches-1
31         for minibatch_index in xrange(n_train_batches):
32 
33             # 总的iter次数（每一轮epoch，iter个数都增加n_train_batches）
34             # 即每一个iter，只利用一个mini-batch进行训练
35             # 而每一个epoch，利用了所有的mini-batch进行训练
36             iter = (epoch - 1) * n_train_batches + minibatch_index
37 
38             # 整体的迭代次数可以被100整除时，显示一次迭代次数
39             if iter % 100 == 0:
40                 print 'training @ iter = ', iter
41 
42             # 利用第minibatch_index个mini-batch训练model，得到model的代价函数
43             cost_ij = train_model(minibatch_index)
44 
45             # 如果整体的迭代次数满足需要进行校验的条件，则对该次iter对应的model进行校验
46             if (iter + 1) % validation_frequency == 0:
47 
48                 # 计算该model在校验集上的loss函数值
49                 validation_losses = [validate_model(i) for i
50                                      in xrange(n_valid_batches)]
51                 this_validation_loss = numpy.mean(validation_losses)
52                 print('epoch %i, minibatch %i/%i, validation error %f %%' %
53                       (epoch, minibatch_index + 1, n_train_batches,
54                        this_validation_loss * 100.))
55 
56                 # 如果该model在校验集的loss值小于之前的值
57                 if this_validation_loss < best_validation_loss:
58 
59                     # 增加patience的值，目的是为了进行更多次的iter
60                     # 也就是说，如果在测试集上的性能不如之前好，证明模型开始恶化，那么，不再进行那么多次的training了
61                     if this_validation_loss < best_validation_loss *  \
62                        improvement_threshold:
63                         patience = max(patience, iter * patience_increase)
64 
65                     # save best validation score and iteration number
66                     best_validation_loss = this_validation_loss
67                     best_iter = iter
68 
69                     # 利用测试集测试该模型
70                     test_losses = [
71                         test_model(i)
72                         for i in xrange(n_test_batches)
73                     ]
74                     # 计算测试集的loss值
75                     test_score = numpy.mean(test_losses)
76                     print(('     epoch %i, minibatch %i/%i, test error of '
77                            'best model %f %%') %
78                           (epoch, minibatch_index + 1, n_train_batches,
79                            test_score * 100.))
80 
81             if patience <= iter:
82                 done_looping = True
83                 break
84                 
85     # 整个训练过程结束，记录training时间
86     end_time = timeit.default_timer()
87     print('Optimization complete.')
88     print('Best validation score of %f %% obtained at iteration %i, '
89           'with test performance %f %%' %
90           (best_validation_loss * 100., best_iter + 1, test_score * 100.))
91     print >> sys.stderr, ('The code for file ' +
92                           os.path.split(__file__)[1] +
93                           ' ran for %.2fm' % ((end_time - start_time) / 60.))

 

（4）真个convolutional_mlp的原始代码

  1 """This tutorial introduces the LeNet5 neural network architecture
  2 using Theano.  LeNet5 is a convolutional neural network, good for
  3 classifying images. This tutorial shows how to build the architecture,
  4 and comes with all the hyper-parameters you need to reproduce the
  5 paper's MNIST results.
  6 
  7 
  8 
  9 This implementation simplifies the model in the following ways:
 10 
 11  - LeNetConvPool doesn't implement location-specific gain and bias parameters
 12  - LeNetConvPool doesn't implement pooling by average, it implements pooling
 13    by max.
 14  - Digit classification is implemented with a logistic regression rather than
 15    an RBF network
 16  - LeNet5 was not fully-connected convolutions at second layer
 17 
 18 References:
 19  - Y. LeCun, L. Bottou, Y. Bengio and P. Haffner:
 20    Gradient-Based Learning Applied to Document
 21    Recognition, Proceedings of the IEEE, 86(11):2278-2324, November 1998.
 22    http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf
 23 
 24 """
 25 import os
 26 import sys
 27 import timeit
 28 
 29 import numpy
 30 
 31 import theano
 32 import theano.tensor as T
 33 from theano.tensor.signal import downsample
 34 from theano.tensor.nnet import conv
 35 
 36 from logistic_sgd import LogisticRegression, load_data
 37 from mlp import HiddenLayer
 38 
 39 
 40 class LeNetConvPoolLayer(object):
 41     """Pool Layer of a convolutional network """
 42 
 43     def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
 44         """
 45         Allocate a LeNetConvPoolLayer with shared variable internal parameters.
 46 
 47         :type rng: numpy.random.RandomState
 48         :param rng: a random number generator used to initialize weights
 49 
 50         :type input: theano.tensor.dtensor4
 51         :param input: symbolic image tensor, of shape image_shape
 52 
 53         :type filter_shape: tuple or list of length 4
 54         :param filter_shape: (number of filters, num input feature maps,
 55                               filter height, filter width)
 56 
 57         :type image_shape: tuple or list of length 4
 58         :param image_shape: (batch size, num input feature maps,
 59                              image height, image width)
 60 
 61         :type poolsize: tuple or list of length 2
 62         :param poolsize: the downsampling (pooling) factor (#rows, #cols)
 63         """
 64 
 65         assert image_shape[1] == filter_shape[1]
 66         self.input = input
 67 
 68         # there are "num input feature maps * filter height * filter width"
 69         # inputs to each hidden unit
 70         fan_in = numpy.prod(filter_shape[1:])
 71         # each unit in the lower layer receives a gradient from:
 72         # "num output feature maps * filter height * filter width" /
 73         #   pooling size
 74         fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
 75                    numpy.prod(poolsize))
 76         # initialize weights with random weights
 77         W_bound = numpy.sqrt(6. / (fan_in + fan_out))
 78         self.W = theano.shared(
 79             numpy.asarray(
 80                 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
 81                 dtype=theano.config.floatX
 82             ),
 83             borrow=True
 84         )
 85 
 86         # the bias is a 1D tensor -- one bias per output feature map
 87         b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
 88         self.b = theano.shared(value=b_values, borrow=True)
 89 
 90         # convolve input feature maps with filters
 91         conv_out = conv.conv2d(
 92             input=input,
 93             filters=self.W,
 94             filter_shape=filter_shape,
 95             image_shape=image_shape
 96         )
 97 
 98         # downsample each feature map individually, using maxpooling
 99         pooled_out = downsample.max_pool_2d(
100             input=conv_out,
101             ds=poolsize,
102             ignore_border=True
103         )
104 
105         # add the bias term. Since the bias is a vector (1D array), we first
106         # reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
107         # thus be broadcasted across mini-batches and feature map
108         # width & height
109         self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
110 
111         # store parameters of this layer
112         self.params = [self.W, self.b]
113 
114         # keep track of model input
115         self.input = input
116 
117 
118 def evaluate_lenet5(learning_rate=0.1, n_epochs=200,
119                     dataset='mnist.pkl.gz',
120                     nkerns=[20, 50], batch_size=500):
121     """ Demonstrates lenet on MNIST dataset
122 
123     :type learning_rate: float
124     :param learning_rate: learning rate used (factor for the stochastic
125                           gradient)
126 
127     :type n_epochs: int
128     :param n_epochs: maximal number of epochs to run the optimizer
129 
130     :type dataset: string
131     :param dataset: path to the dataset used for training /testing (MNIST here)
132 
133     :type nkerns: list of ints
134     :param nkerns: number of kernels on each layer
135     """
136 
137     rng = numpy.random.RandomState(23455)
138 
139     datasets = load_data(dataset)
140 
141     train_set_x, train_set_y = datasets[0]
142     valid_set_x, valid_set_y = datasets[1]
143     test_set_x, test_set_y = datasets[2]
144 
145     # compute number of minibatches for training, validation and testing
146     n_train_batches = train_set_x.get_value(borrow=True).shape[0]
147     n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
148     n_test_batches = test_set_x.get_value(borrow=True).shape[0]
149     n_train_batches /= batch_size
150     n_valid_batches /= batch_size
151     n_test_batches /= batch_size
152 
153     # allocate symbolic variables for the data
154     index = T.lscalar()  # index to a [mini]batch
155 
156     # start-snippet-1
157     x = T.matrix('x')   # the data is presented as rasterized images
158     y = T.ivector('y')  # the labels are presented as 1D vector of
159                         # [int] labels
160 
161     ######################
162     # BUILD ACTUAL MODEL #
163     ######################
164     print '... building the model'
165 
166     # Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
167     # to a 4D tensor, compatible with our LeNetConvPoolLayer
168     # (28, 28) is the size of MNIST images.
169     layer0_input = x.reshape((batch_size, 1, 28, 28))
170 
171     # Construct the first convolutional pooling layer:
172     # filtering reduces the image size to (28-5+1 , 28-5+1) = (24, 24)
173     # maxpooling reduces this further to (24/2, 24/2) = (12, 12)
174     # 4D output tensor is thus of shape (batch_size, nkerns[0], 12, 12)
175     layer0 = LeNetConvPoolLayer(
176         rng,
177         input=layer0_input,
178         image_shape=(batch_size, 1, 28, 28),
179         filter_shape=(nkerns[0], 1, 5, 5),
180         poolsize=(2, 2)
181     )
182 
183     # Construct the second convolutional pooling layer
184     # filtering reduces the image size to (12-5+1, 12-5+1) = (8, 8)
185     # maxpooling reduces this further to (8/2, 8/2) = (4, 4)
186     # 4D output tensor is thus of shape (batch_size, nkerns[1], 4, 4)
187     layer1 = LeNetConvPoolLayer(
188         rng,
189         input=layer0.output,
190         image_shape=(batch_size, nkerns[0], 12, 12),
191         filter_shape=(nkerns[1], nkerns[0], 5, 5),
192         poolsize=(2, 2)
193     )
194 
195     # the HiddenLayer being fully-connected, it operates on 2D matrices of
196     # shape (batch_size, num_pixels) (i.e matrix of rasterized images).
197     # This will generate a matrix of shape (batch_size, nkerns[1] * 4 * 4),
198     # or (500, 50 * 4 * 4) = (500, 800) with the default values.
199     layer2_input = layer1.output.flatten(2)
200 
201     # construct a fully-connected sigmoidal layer
202     layer2 = HiddenLayer(
203         rng,
204         input=layer2_input,
205         n_in=nkerns[1] * 4 * 4,
206         n_out=500,
207         activation=T.tanh
208     )
209 
210     # classify the values of the fully-connected sigmoidal layer
211     layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
212 
213     # the cost we minimize during training is the NLL of the model
214     cost = layer3.negative_log_likelihood(y)
215 
216     # create a function to compute the mistakes that are made by the model
217     test_model = theano.function(
218         [index],
219         layer3.errors(y),
220         givens={
221             x: test_set_x[index * batch_size: (index + 1) * batch_size],
222             y: test_set_y[index * batch_size: (index + 1) * batch_size]
223         }
224     )
225 
226     validate_model = theano.function(
227         [index],
228         layer3.errors(y),
229         givens={
230             x: valid_set_x[index * batch_size: (index + 1) * batch_size],
231             y: valid_set_y[index * batch_size: (index + 1) * batch_size]
232         }
233     )
234 
235     # create a list of all model parameters to be fit by gradient descent
236     params = layer3.params + layer2.params + layer1.params + layer0.params
237 
238     # create a list of gradients for all model parameters
239     grads = T.grad(cost, params)
240 
241     # train_model is a function that updates the model parameters by
242     # SGD Since this model has many parameters, it would be tedious to
243     # manually create an update rule for each model parameter. We thus
244     # create the updates list by automatically looping over all
245     # (params[i], grads[i]) pairs.
246     updates = [
247         (param_i, param_i - learning_rate * grad_i)
248         for param_i, grad_i in zip(params, grads)
249     ]
250 
251     train_model = theano.function(
252         [index],
253         cost,
254         updates=updates,
255         givens={
256             x: train_set_x[index * batch_size: (index + 1) * batch_size],
257             y: train_set_y[index * batch_size: (index + 1) * batch_size]
258         }
259     )
260     # end-snippet-1
261 
262     ###############
263     # TRAIN MODEL #
264     ###############
265     print '... training'
266     # early-stopping parameters
267     patience = 10000  # look as this many examples regardless
268     patience_increase = 2  # wait this much longer when a new best is
269                            # found
270     improvement_threshold = 0.995  # a relative improvement of this much is
271                                    # considered significant
272     validation_frequency = min(n_train_batches, patience / 2)
273                                   # go through this many
274                                   # minibatche before checking the network
275                                   # on the validation set; in this case we
276                                   # check every epoch
277 
278     best_validation_loss = numpy.inf
279     best_iter = 0
280     test_score = 0.
281     start_time = timeit.default_timer()
282 
283     epoch = 0
284     done_looping = False
285 
286     while (epoch < n_epochs) and (not done_looping):
287         epoch = epoch + 1
288         for minibatch_index in xrange(n_train_batches):
289 
290             iter = (epoch - 1) * n_train_batches + minibatch_index
291 
292             if iter % 100 == 0:
293                 print 'training @ iter = ', iter
294             cost_ij = train_model(minibatch_index)
295 
296             if (iter + 1) % validation_frequency == 0:
297 
298                 # compute zero-one loss on validation set
299                 validation_losses = [validate_model(i) for i
300                                      in xrange(n_valid_batches)]
301                 this_validation_loss = numpy.mean(validation_losses)
302                 print('epoch %i, minibatch %i/%i, validation error %f %%' %
303                       (epoch, minibatch_index + 1, n_train_batches,
304                        this_validation_loss * 100.))
305 
306                 # if we got the best validation score until now
307                 if this_validation_loss < best_validation_loss:
308 
309                     #improve patience if loss improvement is good enough
310                     if this_validation_loss < best_validation_loss *  \
311                        improvement_threshold:
312                         patience = max(patience, iter * patience_increase)
313 
314                     # save best validation score and iteration number
315                     best_validation_loss = this_validation_loss
316                     best_iter = iter
317 
318                     # test it on the test set
319                     test_losses = [
320                         test_model(i)
321                         for i in xrange(n_test_batches)
322                     ]
323                     test_score = numpy.mean(test_losses)
324                     print(('     epoch %i, minibatch %i/%i, test error of '
325                            'best model %f %%') %
326                           (epoch, minibatch_index + 1, n_train_batches,
327                            test_score * 100.))
328 
329             if patience <= iter:
330                 done_looping = True
331                 break
332 
333     end_time = timeit.default_timer()
334     print('Optimization complete.')
335     print('Best validation score of %f %% obtained at iteration %i, '
336           'with test performance %f %%' %
337           (best_validation_loss * 100., best_iter + 1, test_score * 100.))
338     print >> sys.stderr, ('The code for file ' +
339                           os.path.split(__file__)[1] +
340                           ' ran for %.2fm' % ((end_time - start_time) / 60.))
341 
342 if __name__ == '__main__':
343     evaluate_lenet5()
344 
345 
346 def experiment(state, channel):
347     evaluate_lenet5(state.learning_rate, dataset=state.dataset)

View Code