Tensorflow（二十七） —— 过拟合与欠拟合
Tensorflow（二十七） —— 过拟合与欠拟合
- 1. 过拟合检测
- 2. k折交叉验证
- 3. 过拟合的解决方案
-
- 3.1 正则化
- 3.2 动量与学习率
- 3.3 early stopping / Dropout / stochastic gradient descent
1. 过拟合检测
# ************************* 1、how to detect overfitting
# 1.1 splitting
(x,y),(x_test,y_test) = keras.datasets.mnist.load_data()
x_train,x_val = tf.split(x,num_or_size_splits=[50000,10000])
y_train,y_val = tf.split(y,num_or_size_splits = [50000,10000])
def preprocess(x,y):
# x是一张照片
x = tf.reshape(tf.cast(x,dtype=tf.float32)/255.,[28*28])
y = tf.one_hot(tf.cast(y,dtype=tf.int32),depth=10)
return x,y
db_train = tf.data.Dataset.from_tensor_slices((x_train,y_train))
db_val = tf.data.Dataset.from_tensor_slices((x_val,y_val))
db_train = db_train.map(preprocess).shuffle(10000).batch(128)
db_val = db_val.map(preprocess).shuffle(10000).batch(128)
"""
案例
"""
network = keras.Sequential([
keras.layers.Dense(28*28,activation = tf.nn.relu),
keras.layers.Dense(256,activation = tf.nn.relu),
keras.layers.Dense(10)
])
network.build(input_shape = [None,28*28])
network.compile(optimizer = keras.optimizers.Adam(lr = 1e-3),\
loss = tf.losses.CategoricalCrossentropy(from_logits = True),\
metrics = ["accuracy"])
network.fit(db_train,epochs = 10,validation_data = db_val,validation_freq = 2)
network.evaluate(db_val)
2. k折交叉验证
# 1.3 k-fold cross-validation
"""
交叉验证
1、merge train/val sets
2、random sample 1/k as val set
"""
# 自行实现
for epoch in range(500):
index = tf.range(60000)
index = tf.random.shuffle(index)
x_train,y_train = tf.gather(x,index[:5000]),tf.gather(y,index[:5000])
x_val,y_val = tf.gather(x,index[-1000:]),tf,gather(y,index[-1000:])
# 简捷
network.fit(db_train,\
validation_split = 0.1,\
validation_freq = 2,\
epochs = 10)
3. 过拟合的解决方案
3.1 正则化
# 2.1 regularization
"""
weight decay
"""
# 用法一
L2_model = keras.Sequential([
keras.layers.Dense(256,kernel_regularizer = keras.regularizers.l2(0.001),\
bias_regularizer = keras.regularizers.l2(0.001),# 自由设定 惩罚度不同
activation = tf.nn.relu),
keras.layers.Dense(128,kernel_regularizer = keras.regularizers.l2(0.001),\
bias_regularizer = keras.regularizers.l2(0.001),
activation = tf.nn.relu),
keras.layers.Dense(64,kernel_regularizer = keras.regularizers.l2(0.001),\
bias_regularizer = keras.regularizers.l2(0.001),
activation = tf.nn.relu),
keras.layers.Dense(10,activation = tf.nn.sigmoid)
])
# 用法二
for step,(x,y) in enumerate(db_train):
with tf.GradientTape() as tape:
loss = tf.losses.categorical_crossentropy(y_onehot,out,from_logits=True)
loss_regularizer = []
for p in network.trainable_variables:
loss_regularizer.append(tf.nn.l2_loss(p))
loss_regularizer = tf.reduce_sum(tf.stack(loss_regularizer))
loss = loss + 0.0001*loss_regularizer
grads = tape.gradient(loss,network.trainable_variables)
optimizer.apply_gradients(zip(grads,network.trainable_variables))
3.2 动量与学习率
# 2.2 动量与学习率
"""
momentum 动量
learning rate decay
"""
# momentum
optimizer = keras.optimizers.SGD(learning_rate = 0.001,momentum=0.9) # 动量因子一般设定为0.9
optimizer = keras.optimizers.RMSprop(learning_rate=0.001,momentum=0.9) # 此处也是动量因子，指数加权平均系数已默认设置为0.9
optimizer = keras.optimizers.Adam(learning_rate=0.001,beta_1 = 0.9,beta_2 = 0.999) # beta_1为动量因子，beta_2为指数加权平均系数
# learning rate decay
optimizer = keras.optimizers.SGD(learning_rate=0.2)
for epoch in range(100):
optimizer.learning_rate = 0.2 * (100-epoch)/100
3.3 early stopping / Dropout / stochastic gradient descent
# Dropout
network = keras.models.Sequential([
keras.layers.Dense(256,activation = tf.nn.relu),\
keras.layers.Dropout(0.5),\
keras.layers.Dense(128,activation = tf.nn.relu),\
keras.layers.Dropout(0.5),\
keras.layers.Dense(64,activation = tf.nn.relu),\
keras.layers.Dense(32,activation = tf.nn.relu),\
keras.layers.Dense(10)
])
# 针对于训练集
for step,(x,y) in enumerate(db):
out = network(x,training = True)
# 针对于验证集
out = network(x,training = False)
# 针对于测试集
out = network(x,training = False)
本文为参考龙龙老师的“深度学习与TensorFlow 2入门实战“课程书写的学习笔记
by CyrusMay 2022 04 18