加正则前后计算量变化

N(0,1)

正太分布

高维样本的稀疏性问题

import tensorflow as tf
import numpy as np

W = tf.Variable([[1],[1]],dtype=tf.float32)
W

x = tf.placeholder(tf.float32,shape=[None,2])
x

y = tf.placeholder(tf.float32,shape=[None,1])
y

loss = tf.losses.absolute_difference(y,tf.matmul(x,W))
optimizer = tf.train.GradientDescentOptimizer(0.01)
train_step = optimizer.minimize(loss)

一条训练数据 （[1,0],0.5）

train_x = np.array([[1,0]],dtype=np.float32)
train_y = np.array([[.5]],dtype=np.float32)

init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)
    for i in range(5):
        sess.run(train_step,feed_dict={x:train_x,y:train_y})
        W_res = sess.run(W)
        print(W_res)
        print('==================')

[[0.99]
 [1.  ]]
==================
[[0.98]
 [1.  ]]
==================
[[0.97]
 [1.  ]]
==================
[[0.96000004]
 [1.        ]]
==================
[[0.95000005]
 [1.        ]]
==================

加正则后

loss += tf.nn.l2_loss(W)
loss

optimizer = tf.train.GradientDescentOptimizer(0.01)
train_step = optimizer.minimize(loss)

init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)
    for i in range(5):
        sess.run(train_step,feed_dict={x:train_x,y:train_y})
        W_res = sess.run(W)
        print(W_res)
        print('==================')

[[0.98]
 [0.99]]
==================
[[0.9602    ]
 [0.98010004]]
==================
[[0.940598  ]
 [0.97029907]]
==================
[[0.92119205]
 [0.9605961 ]]
==================
[[0.9019801 ]
 [0.95099014]]
==================