用UFLDL的方法改写Denoising Autoencoder
最近在学Autoencoder的相关知识，已经学习完DFLDL的Sparse_Autoencoder相关知识，想学学Denoising Autoencoder和contractive Autoencoder。
本篇先说说Denoising Autoencoder的学习。
参考大神tornadomeet的博客（http://www.cnblogs.com/tornadomeet/p/3261247.html），我准备自己尝试第一个实验。下载完DeepLearnToolbox并且自己把代码test_example_SAE跑了一遍后发现，没有Denoising时候，识别结果error为0.10，然后加上50%的Denoising后，就出现“Too big error”的警告，原来是识别结果error超过0.16，而代码的最后一句为
assert(er < 0.16, 'Too big error')。
但是tornadomeet大神不是说加上50%的Denoising后test的结果会更好吗？仔细对比了一下发现，toolbox里面的代码网络结构是[784 100 10]的，也就是只有一层hidden layer，代码如下：
function test_example_SAE
load ../data/mnist_uint8;
train_x = double(train_x)/255;
test_x = double(test_x)/255;
train_y = double(train_y);
test_y = double(test_y);
addpath('../SAE')
addpath('../NN')
%% ex1 train a 100 hidden unit SDAE and use it to initialize a FFNN
% Setup and train a stacked denoising autoencoder (SDAE)
rand('state',0)
sae = saesetup([784 100]);
sae.ae{1}.activation_function = 'sigm';
sae.ae{1}.learningRate = 1;
sae.ae{1}.inputZeroMaskedFraction = 0.5;
opts.numepochs = 1;
opts.batchsize = 100;
sae = saetrain(sae, train_x, opts);
visualize(sae.ae{1}.W{1}(:,2:end)')
% Use the SDAE to initialize a FFNN
nn = nnsetup([784 100 10]);
nn.activation_function = 'sigm';
nn.learningRate = 1;
nn.W{1} = sae.ae{1}.W{1};
% Train the FFNN
opts.numepochs = 1;
opts.batchsize = 100;
nn = nntrain(nn, train_x, train_y, opts);
[er, bad] = nntest(nn, test_x, test_y);
assert(er < 0.16, 'Too big error');
而tornadomeet大神的代码是两层hidden layer，网络结构是[784 100 100 10]的，存在着这个差别，所以两层hidden layer的结果是：
接下来我自己尝试去写这整个代码，但是toolbox的代码由于是要适合多个test的，所以每个被调用的函数里面都列举了各种的可能性，
switch nn.activation_function
case 'sigm'
% Calculate the unit's outputs (including the bias term)
nn.a{i} = sigm(nn.a{i - 1} * nn.W{i - 1}');
case 'tanh_opt'
nn.a{i} = tanh_opt(nn.a{i - 1} * nn.W{i - 1}');
end
%dropout
if(nn.dropoutFraction > 0)
if(nn.testing)
nn.a{i} = nn.a{i}.*(1 - nn.dropoutFraction);
else
nn.dropOutMask{i} = (rand(size(nn.a{i}))>nn.dropoutFraction);
nn.a{i} = nn.a{i}.*nn.dropOutMask{i};
end
end
%calculate running exponential activations for use with sparsity
if(nn.nonSparsityPenalty>0)
nn.p{i} = 0.99 * nn.p{i} + 0.01 * mean(nn.a{i}, 1);
end
%Add the bias term
nn.a{i} = [ones(m,1) nn.a{i}];
end
switch nn.output
case 'sigm'
nn.a{n} = sigm(nn.a{n - 1} * nn.W{n - 1}');
case 'linear'
nn.a{n} = nn.a{n - 1} * nn.W{n - 1}';
case 'softmax'
nn.a{n} = nn.a{n - 1} * nn.W{n - 1}';
nn.a{n} = exp(bsxfun(@minus, nn.a{n}, max(nn.a{n},[],2)));
nn.a{n} = bsxfun(@rdivide, nn.a{n}, sum(nn.a{n}, 2));
end
代码太冗余，不适合初学者，所以我自己尝试用UFLDL里面的那种代码风格，自己从头到尾写了一遍代码，结果发现，即使是用一层隐含层，网络结构是[784 100 10]，用了denoising后，效果也会比没有denoising好。
下面是自己写的代码和结果（本人是初学者，代码不够规范，希望大家见谅并批评指出）：
the error rate is 9.3%
加上denoising后，the error rate is 7.3%
%% Code for Denoising Autoencoder
% the network is [784 100 10]
% By xubinlxb
close all
clear
clc
%% load data
load './data/mnist_uint8.mat'
train_x = double(train_x'./255);
train_y = double(train_y');
test_x = double(test_x'./255);
test_y = double(test_y');
%% parameter initialized
nn = struct;
nn.visibleSize = size(train_x,1);
nn.hiddenSize = 100;
nn.outputSize = 10;
nn.lambda = 0.0001;
nn.batchSize=100;
nn.LearnRate = 3;
nn.Method = 'unsupervise';
nn.DenoisingPara = 0.5;
%% build Layer1 network
% configuration the cost funtion
rng('default');
nn = theta_initialization(nn);
disp('unsupervise train...')
[L1, nn] = batch_build(nn, train_x ,train_x);
[cost1,~] = build_one_layer_addWeight(nn,train_x,train_x);
% Visualization
visualize(nn.W1');
% display_network(nn.W1', 12);
%% train the whole network
nn.DenoisingPara = 0;
nn.Method = 'supervise';
W1 = nn.W1;
b1 = nn.b1;
nn = theta_initialization(nn);
nn.W1 = W1;
nn.b1 = b1;
clear W1 b1
disp('supervise train...')
[L2, nn] = batch_build(nn, train_x, train_y);
[cost2,~] = build_one_layer_addWeight(nn,train_x,train_y);
%% test the whole network
label = predict(nn,test_x);
[~,expected] = max(test_y,[],1);
badNum = numel(find(label~=expected));
er = badNum/size(test_y,2);
assert(er<0.16,'Too big error')
fprintf('the errror rate is %f\n',er)
function [ L, nn ] = batch_build( nn, train_x ,train_y)
m = size(train_x,2);
batchSize = nn.batchSize;
batchNum = m/ batchSize;
L = [];
for i = 1:batchNum
if (mod(i,100) == 0)
fprintf('batch %d,\n',i);
end
kk = randperm(m);
batch_x = train_x( :, kk(((i-1)*batchSize+1): i*batchSize) );
batch_y = train_y( :, kk(((i-1)*batchSize+1): i*batchSize) );
[cost, nn] = build_one_layer_addWeight(nn, batch_x ,batch_y);
L = [L,cost];
end
end
function [ cost, nn ] = build_one_layer_addWeight( nn, x, y )
% initialize the parameter
lambda = nn.lambda;
LearnRate = nn.LearnRate;
W1 = nn.W1;
b1 = nn.b1;
W2 = nn.W2;
b2 = nn.b2;
W1grad = zeros(size(W1));
b1grad = zeros(size(b1));
W2grad = zeros(size(W2));
b2grad = zeros(size(b2));
%% BP algorithm
m = size(x,2);
% Denoising Autoencoder
x = x.*(rand(size(x)) > nn.DenoisingPara);
% feedforward
z2 = W1* x+ repmat(b1,1,m);
a2 = sigmoid(z2);
z3 = W2* a2+ repmat(b2,1,m);
a3 = sigmoid(z3);
cost = 0.5/m.* sum(sum((y - a3).^2))+ 0.5*lambda.* (sum(W1(:).^2)+sum(W2(:).^2));
% Back forword calculator the grad
d3 = -(y - a3).* a3 .*(1-a3);
d2 = W2'* d3 .* a2 .*(1-a2);
W2grad = 1.0/m.* (d3 *a2')+ lambda.* W2;
b2grad = 1.0/m.* sum(d3,2);
W1grad = 1.0/m.* (d2 *x')+ lambda.*W1;
b1grad = 1.0/m.* sum(d2,2);
W2 = W2 - LearnRate*W2grad;
b2 = b2 - LearnRate*b2grad;
W1 = W1 - LearnRate*W1grad;
b1 = b1 - LearnRate*b1grad;
%grad = [ W1grad(:);b1grad(:);W2grad(:);b2grad(:) ];
nn.W1 = W1;
nn.b1 = b1;
nn.W2 = W2;
nn.b2 = b2;
end
function sigm = sigmoid(x)
sigm = 1./(1+exp(-x));
end
function [ nn ] = theta_initialization( nn )
visibleSize = nn.visibleSize;
hiddenSize = nn.hiddenSize;
switch nn.Method
case 'unsupervise'
outputSize = nn.visibleSize;
case 'supervise'
outputSize = nn.outputSize;
otherwise
error('nn.Method error');
end
r = sqrt(6) / sqrt(hiddenSize+visibleSize+1); % weights uniformly from the interval [-r, r]
W1 = rand(hiddenSize, visibleSize) * 2 * r - r;
r = sqrt(6) / sqrt(hiddenSize+outputSize+1);
W2 = rand(outputSize, hiddenSize) * 2 * r - r;
b1 = zeros(hiddenSize, 1);
b2 = zeros(outputSize, 1);
nn.W1 = W1;
nn.b1 = b1;
nn.W2 = W2;
nn.b2 = b2;
end
function label = predict( nn, x)
%UNTITLED3 Summary of this function goes here
% Detailed explanation goes here
W1 = nn.W1;
b1 = nn.b1;
W2 = nn.W2;
b2 = nn.b2;
m = size(x,2);
label = zeros(1,m);
z2 = W1*x +repmat(b1,1,m);
a2 = sigmoid(z2);
z3 = W2*a2 +repmat(b2,1,m);
a3 = sigmoid(z3);
[~,label] = max(a3,[],1);
end
function sigm = sigmoid(x)
sigm = 1./(1+exp(-x));
end
function r=visualize(X, mm, s1, s2)
%FROM RBMLIB http://code.google.com/p/matrbm/
%Visualize weights X. If the function is called as a void method,
%it does the plotting. But if the function is assigned to a variable
%outside of this code, the formed image is returned instead.
if ~exist('mm','var')
mm = [min(X(:)) max(X(:))];
end
if ~exist('s1','var')
s1 = 0;
end
if ~exist('s2','var')
s2 = 0;
end
[D,N]= size(X);
s=sqrt(D);
if s==floor(s) || (s1 ~=0 && s2 ~=0)
if (s1 ==0 || s2 ==0)
s1 = s; s2 = s;
end
%its a square, so data is probably an image
num=ceil(sqrt(N));
a=mm(2)*ones(num*s2+num-1,num*s1+num-1);
x=0;
y=0;
for i=1:N
im = reshape(X(:,i),s1,s2)';
a(x*s2+1+x : x*s2+s2+x, y*s1+1+y : y*s1+s1+y)=im;
x=x+1;
if(x>=num)
x=0;
y=y+1;
end
end
d=true;
else
%there is not much we can do
a=X;
end
%return the image, or plot the image
if nargout==1
r=a;
else
imagesc(a, [mm(1) mm(2)]);
axis equal
colormap gray
end