【deep learning学习笔记】注释yusugomori的RBM代码 --- cpp文件 -- 模型训练
关键是 CD-k（contrastive_divergence）算法的实现。
// the CD-k algorithm
void RBM::contrastive_divergence (
int *input, // the input visiable sample
double lr, // the learning rate
int k // the k value in CD-k
)
{
// allocate the memory for <v,h>|data, <v,h>|reconstructed
// the probability expectiion (mean) of the hidden nodes, p(h0|v)
double *ph_mean = new double[n_hidden];
// the 0-1 state of the hidden nodes
int *ph_sample = new int[n_hidden];
// the probability expectiion (mean) of the visiable nodes, p(v|h)
double *nv_means = new double[n_visible];
// the pointer to the 0-1 state of the visiable nodes
int *nv_samples = new int[n_visible];
// the probability expectiion (mean) of the hidden nodes, p(h1|v)
double *nh_means = new double[n_hidden];
// the 0-1 state of the hidden nodes
int *nh_samples = new int[n_hidden];
/* CD-k */
// step 1: get the probability and 0-1 state in the hidden nodes according to
// the input from the visiable nodes
sample_h_given_v(input, ph_mean, ph_sample);
// step 2: gibbs sample
// data in visiable node --> p(h0|v) in hidden node --> reconstruct the visiable node p(v|h) --> p(h1|v) in hidden node ...
for(int step=0; step<k; step++)
{
if(step == 0)
{
gibbs_hvh (ph_sample, // one input sample from hidden node, h0 -- input
nv_means, // the output probability of visiable nodes -- output
nv_samples, // the calculated 0-1 state of visiable node -- output
nh_means, // the output probability of reconstructed hidden node h1 -- output
nh_samples // the calculated 0-1 state of reconstructed hidden node h1 -- output
);
}
else
{
gibbs_hvh (nh_samples, // one input sample from hidden node, h0 -- input
nv_means, // the output probability of visiable nodes -- output
nv_samples, // the calculated 0-1 state of visiable node -- output
nh_means, // the output probability of reconstructed hidden node h1 -- output
nh_samples // the calculated 0-1 state of reconstructed hidden node h1 -- output
);
}
}
// update the value of W
for(int i=0; i<n_hidden; i++)
{
for(int j=0; j<n_visible; j++)
{
// w[hidden][visible] += learningRate * DeltaW, while DeltaW = <v,h>|data - <v,h>|reconstruct
// <v,h>|data is the expectation of v_i and h_j given the input data
// <v,h>|data = ( 0-1 of h0 ) * ( 0-1 in input sample) / the total number of sample
// <v,h>|reconstruct is the expectation v_i and h_j by the reconstructed visible nodes
// <v,h>|data = ( 0-1 of h1 ) * ( 0-1 in reconstructed node) / the total number of sample
W[i][j] += lr * (ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
}
// hiddenBias += learningRate * ( <h>|data - <h>|model )
// = learningRate * ( h0 - h1 ) / N
hbias[i] += lr * (ph_sample[i] - nh_means[i]) / N;
}
for(int i=0; i<n_visible; i++)
{
// visibleBias += learningRate * ( <v>|data - <v>|model )
// = learningRate * ( input - reconstructedNode ) / N
vbias[i] += lr * (input[i] - nv_samples[i]) / N;
}
// release the memory for <v,h>|data, <v,h>|reconstructed
delete[] ph_mean;
delete[] ph_sample;
delete[] nv_means;
delete[] nv_samples;
delete[] nh_means;
delete[] nh_samples;
} // contrastive_divergence
void RBM::sample_h_given_v (
int *v0_sample, // one input sample from visiable nodes -- input
double *mean, // the output probability of hidden nodes -- output
int *sample // the calculated 0-1 state of hidden node -- output
)
{
// iterate all the hidden node
for(int i=0; i<n_hidden; i++)
{
// calculate the probablity of hidden node given the input sample and
// the RBM weight from bottem to top
mean[i] = propup(v0_sample, W[i], hbias[i]);
// binomial test to decide the 0-1 state of each hidden node
sample[i] = binomial(1, mean[i]);
}
}
void RBM::sample_v_given_h (
int *h0_sample, // one input sample from hidden nodes -- input
double *mean, // the output probability of visiable nodes -- output
int *sample // the calculated 0-1 state of visiable node -- output
)
{
// iterate all the visiable node
for(int i=0; i<n_visible; i++)
{
// calculate the probablity of visible node given the hidden node sample and
// the RBM weight from top to bottem
mean[i] = propdown(h0_sample, i, vbias[i]);
// binomial test to decide the 0-1 state of each visiable node
// this step reconstruct the visiable nodes
sample[i] = binomial(1, mean[i]);
}
}
// the returned probability is : p (hi|v) = sigmod ( sum_j(vj * wij) + bi)
double RBM::propup (
int *v, // one input sample from visiable node -- input
double *w, // the weight W connecting one hidden node to all visible node -- input
double b // the bias for this hidden node -- input
)
{
// calculated sum_j(vj * wij) + bi )
double pre_sigmoid_activation = 0.0;
for(int j=0; j<n_visible; j++)
{
pre_sigmoid_activation += w[j] * v[j];
}
pre_sigmoid_activation += b;
// sigmod (pre_sigmoid_activation)
return sigmoid(pre_sigmoid_activation);
}
// the returned probability is : p (vi|h) = sigmod ( sum_j(hj * wij) + ci)
double RBM::propdown(
int *h, // one input sample from hidden node -- input
int i, // the index of visiable node in the W matrix -- input
double b // the bias for this visible node -- input
)
{
// calcualte sum_j(hj * wij) + ci
double pre_sigmoid_activation = 0.0;
for(int j=0; j<n_hidden; j++)
{
pre_sigmoid_activation += W[j][i] * h[j];
}
pre_sigmoid_activation += b;
// sigmod (pre_sigmoid_activation)
return sigmoid(pre_sigmoid_activation);
}
void RBM::gibbs_hvh (
int *h0_sample, // one input sample from hidden node, h0 -- input
double *nv_means, // the output probability of visiable nodes -- output
int *nv_samples, // the calculated 0-1 state of visiable node -- output
double *nh_means, // the output probability of reconstructed hidden node h1 -- output
int *nh_samples // the calculated 0-1 state of reconstructed hidden node h1 -- output
)
{
// calculate p(v|h)
sample_v_given_h(h0_sample, nv_means, nv_samples);
// calculate p(h|v)
sample_h_given_v(nv_samples, nh_means, nh_samples);
}
void RBM::reconstruct(int *v, double *reconstructed_v)
{
// h[i] = p(h_i|v)
double *h = new double[n_hidden];
double pre_sigmoid_activation;
// calculate p(h|v) given v
for(int i=0; i<n_hidden; i++)
{
h[i] = propup(v, W[i], hbias[i]);
}
// calculate p(v_reconstruct|h)
for(int i=0; i<n_visible; i++)
{
pre_sigmoid_activation = 0.0;
for(int j=0; j<n_hidden; j++)
{
pre_sigmoid_activation += W[j][i] * h[j];
}
pre_sigmoid_activation += vbias[i];
reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
}
delete[] h;
}