TensorFlow2.x 学习笔记(五)神经网络与全连接层

数据加载

keras.datasets

boston housing

• Boston housing price regression dataset

mnist/fashion mnist

• MNIST/Fashion-MNIST dataset

MNIST

(x, y), (x_test, y_test) = keras.datasets.mnist.load_data()
x.shape # 60000,28,28
y.shape # 60000,
x_test.shape, y_test.shape
# (10000, 28, 28), (10000, )
y_onehot = tf.one_hot(y, depth=10)

cifar10/100

• small images classification dataset
• 10/100区别在于大类下的label划分
• [32, 32, 3]

(x, y), (x_test, y_test) = keras.datasets.cifar10.load_data()
x.shape, y.shape, x_test.shape, y_test.shape
# ((50000, 32, 32, 3), (50000, 1), (10000, 32, 32, 3), (10000, 1))

imdb

• sentiment classification dataset

tf.data.Dataset

• from_tensor_slices()

(x, y), (x_test, y_test) = keras.datasets.cifar10.load_data()
db = tf.data.Dataset.from_tensor_slices(x_test)
next(iter(db)).shape
# [32, 32, 3]
db = tf.data.Dataset.from_tensor_slices((x_test, y_test))
next(iter(db))[0].shape
# [32, 32, 3]

• .shuffle

db = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db = fb.shuffle(10000)

• .map

def preprocess(x, y):
	x = tf.cast(x, dtype=tf.float32)/255.
	y = tf.cast(y, dtype=tf.int32)
	y = tf.one_hot(y, depth=10)
	return x, y
db2 = db.map(preprocess)
res = next(iter(db2))
res[0].shape, res[1].shape
# [32, 32, 3], [1, 10]
res[1][:2]
# shape:(1, 10) dtype = float32

• batch

db3 = db2.batch(32)
res = next(iter(db3))
res[0].shape, res[1].shape
# [32, 32, 32, 3], [32, 1, 10]

• StopIteration

db_iter = iter(db3)
while True:
	x,y = next(db_iter)
##########OutOfRangeError#############

• .repeat()

db4 = db3.repeat()
db4 = db3.repeat(4)

Example

def pre_mnist(x, y):
	x = tf.cast(x, tf.float32) / 255.0
	y = tf.cast(y, tf.int64)
	return x, y
def mnist_dataset():
	(x, y), (x_val, y_val) = datasets.fashion_mnist.load_data()
	y = tf.one_hot(y, depth=10)
	y_val = tf.one_hot(y_val, depth=10)
	
	ds = tf.data.Dataset.from_tensor_slices((x, y))
	ds = ds.map(pre_mnist)
	ds = ds.shuffle(60000).batch(100)
	ds_val = tf.data.Dataset.from_tensor_slices((x_val, y_val))
	ds_val = ds_val.map(pre_mnist)
	ds_val = ds_val.shuffle(10000).batch(100)
	return ds, ds_val

全连接层

Dense

x = tf.random.normal([4,784])
net = tf.keras.layers.Dense(512)
out = net(x)
out.shape
TensorShape([4, 512])
net.kernel.shape, net.bias.shape
# [784, 512], [512]

net = tf.keras.layers.Dense(10)
net.bias

net.get_weights() # []
net.weights # []

net.build(input_shape=(None, 4))
net.kernel.shape, net.bias.shape
# [4, 10], [10]

net,build(input_shape=(None, 20))
net.kernel.shape, net.bias.shape
# [20, 10], [10]

Sequential

x = tf.random.normal([2, 3])
model = keras.Sequential([
		keras.layers.Dense(2, activation='relu'),
		keras.layers.Dense(2, activation='relu'),
		keras.layers.Dense(2)
	])
model.build(input_shape=[None, 3])
model.summary()

for p inmodel.trainable_variables:
	print(p.name, p.shape)

Output

$y\in R^d$

• linear regression
• naive classification with MSE
• other general prediction
• out = relu(X@W + b)
  • logits

$y_i\in [0,1]$

• binary classification
  • $y>0.5\to 1$
  • $y>0.5\to 1$
• Image Generation
  • rgb

tf.sigmoid

$\sigma (x)=\frac{1}{1+e^{-x}}$

a = tf.linspace(-6., 6, 10)
tf.sigmoid(a)
x = tf.random.normal([1, 28, 28])*5
tf.reduce_min(x), tf.reduce_max(x)

x = tf.sigmoid(x)
tf.reduce_min(x), tf.reduce_max(x)

$y_i\in [0,1],\sum y_i=1$

softmax

a = tf.linspace(-2., 2, 5)
tf.nn.softmax(a)

Classification

logits = tf.random.uniform([1, 10], minval=-2, maxval=2)
prob = tf.nn.softmax(logits, axis=1)
tf.reduce_sum(prob, axis=1)

$y_i\in [-1,1]$

tanh

a = tf.linspace(-2., 2, 5)
tf.tanh(a)

损失函数

MSE

• $MSE=\frac{1}{N}\sum (y-out{)}^2$
• $L_{2-norm}=\sqrt{\sum (y-out{)}^2}$

y = tf.constant([1, 2, 3, 0, 2])
y = tf.one_hot(y, depth=4)
y = tf.cast(y, dtype=tf.float32)

out = tf.random.normal([5, 4])

loss1 = tf.reduce_mean(tf.square(y - out))
loss2 = tf.square(tf.norm(y - out))/(5 * 4)
loss3 = tf.reduce_mean(tf.losses.MSE(y, out))#这个mean求的是batch的平均，MSE输出的shape为(5,)
#loss1 = loss2 = loss3

Cross Entropy Loss

Entropy

• $H(P)=-\sum _{i}p(i)logp(i)=-E_{x\sim p}[logp(x)]$

Cross Entropy

• $H(p,q)=-\sum p(x)logq(x)=-E_{x\sim p}[logq(x)]$
• $H(p,q)=H(p)+D_{KL}(p|q)$
• KL散度可以衡量两个分布的差异
• $D_{KL}(p||q)=E_{x\sim p}[log\frac{p(x)}{q(x)}]$

Classification

Single output(binary)

$loss=-(ylog\hat{y}+(1-y)log(1-\hat{y}))$

Multi output

onehot编码时
$loss=-log{q}_{i}wherepi=1$

Categorical Cross Entropy

tf.losses.categorical_crossentropy([0, 1, 0, 0], [0.25, 0.25, 0.25, 0.25])
#1.3862
tf.losses.categorical_crossentropy([0, 1, 0, 0], [0.01, 0.97, 0.01, 0.01])
#0.0304
tf.losses.BinaryCrossentropy()([1],[0.1])#class
tf.losses.binary_crossentropy([1],[0.1])#func

Numerical Stability

x = tf.random.normal([1, 784])
w = tf.random.normal([784, 2])
b = tf.zeros([2])
logits = x @ w + b
prob = tf.math.softmax(logits, axis=1)
tf.losses.categorical_crossentropy([0, 1], logits, from_logits=True)
tf.losses.categorical_crossentropy([0, 1], prob)#same as above but not recommanded

随着tf2.1的更新，这些被整合到tf.keras.losses中了