深度学习笔记（1）:损失函数和激活函数的选择关系

参考：链接







代码：

from abc import ABC, abstractmethod
import numpy as np
import time
import re
from collections import OrderedDict


class ActivationBase(ABC):
	def __init__(self, **kwargs):
		super().__init__()
	def __call__(self, z):
		if z.ndim == 1:
			z = z.reshape(1, -1)
		return self.forward(z)
		
	@abstractmethod
	def forward(self, z):
		"""
	函数作用：前向传播，通过激活函数获得 a
		"""
		raise NotImplementedError
		
	@abstractmethod
	def grad(self, x, **kwargs):
	"""
	函数作用：反向传播，获得梯度
	"""
		raise NotImplementedError


class ReLU(ActivationBase):
"""
整流线性单元。
"""
	def __init__(self):
		super().__init__()
	def __str__(self):
		return "ReLU"
	def forward(self, z):
		return np.clip(z, 0, np.inf)
	def grad(self, x):
		return (x > 0).astype(int)


class Sigmoid(ActivationBase):
"""
sigmoid 激活函数。更多激活函数见 ../method 文件夹。
"""
	def __init__(self):
		super().__init__()
	def __str__(self):
		return "Sigmoid"
	def forward(self, z):
		return 1 / (1 + np.exp(-z))
	def grad(self, x):
		return self.forward(x) * (1 - self.forward(x))

class Tanh(ActivationBase):
"""
双曲正切函数。
"""
	def __init__(self):
		super().__init__()
	def __str__(self):
		return "Tanh"
	def forward(self, z):
		return np.tanh(z)
	def grad(self, x):
		return 1 - np.tanh(x) ** 2

class Affine(ActivationBase):
"""
affine 激活函数， 即仿射变换。 输出 slope*z + intercept。当 slope=1 且 intercept=0 表示不做变换。
"""
	def __init__(self, slope=1, intercept=0):
		self.slope = slope
		self.intercept = intercept
		super().__init__()
	def __str__(self):
		return "Affine(slope={}, intercept={})".format(self.slope, self.intercept)
	def forward(self, z):
		return self.slope * z + self.intercept
	def grad(self, x):
		return self.slope * np.ones_like(x)


class ActivationInitializer(object):
	def __init__(self, acti_name='sigmoid'):
		self.acti_name = acti_name
	def __call__(self):
		if self.acti_name.lower() == 'sigmoid':
			acti_fn = Sigmoid()
		elif self.acti_name.lower() == 'relu':

			acti_fn = ReLU()
		elif "affine" in self.acti_name.lower():
			r = r"affine\(slope=(.*), intercept=(.*)\)"
			slope, intercept = re.match(r, 	self.acti_name.lower()).groups()
			acti_fn = Affine(float(slope), float(intercept))
		return acti_fn