import numpy as np
[docs]
class Activation:
[docs]
def forward(self, x):
raise NotImplementedError
[docs]
def backward(self, grad):
raise NotImplementedError
def __call__(self, x):
return self.forward(x)
[docs]
class Linear(Activation):
[docs]
def forward(self, x):
self._input = x
return x
[docs]
def backward(self, grad):
return grad
[docs]
class ReLU(Activation):
[docs]
def forward(self, x):
self._mask = x > 0
return np.where(self._mask, x, 0.0)
[docs]
def backward(self, grad):
return grad * self._mask
[docs]
class LeakyReLU(Activation):
def __init__(self, alpha=0.01):
self.alpha = alpha
[docs]
def forward(self, x):
self._input = x
return np.where(x > 0, x, self.alpha * x)
[docs]
def backward(self, grad):
dx = np.where(self._input > 0, 1.0, self.alpha)
return grad * dx
[docs]
class ELU(Activation):
def __init__(self, alpha=1.0):
self.alpha = alpha
[docs]
def forward(self, x):
self._input = x
return np.where(x > 0, x, self.alpha * (np.exp(np.clip(x, -500, 0)) - 1.0))
[docs]
def backward(self, grad):
out = np.where(self._input > 0, 1.0,
self.alpha * np.exp(np.clip(self._input, -500, 0)))
return grad * out
[docs]
class SELU(Activation):
_ALPHA = 1.6732632423543772
_SCALE = 1.0507009873554805
[docs]
def forward(self, x):
self._input = x
return self._SCALE * np.where(x > 0, x, self._ALPHA * (np.exp(np.clip(x, -500, 0)) - 1.0))
[docs]
def backward(self, grad):
dx = self._SCALE * np.where(self._input > 0, 1.0,
self._ALPHA * np.exp(np.clip(self._input, -500, 0)))
return grad * dx
[docs]
class Sigmoid(Activation):
[docs]
def forward(self, x):
self._out = 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))
return self._out
[docs]
def backward(self, grad):
return grad * self._out * (1.0 - self._out)
[docs]
class Tanh(Activation):
[docs]
def forward(self, x):
self._out = np.tanh(x)
return self._out
[docs]
def backward(self, grad):
return grad * (1.0 - self._out ** 2)
[docs]
class Softmax(Activation):
[docs]
def forward(self, x):
shifted = x - np.max(x, axis=-1, keepdims=True)
exp_x = np.exp(shifted)
self._out = exp_x / np.sum(exp_x, axis=-1, keepdims=True)
return self._out
[docs]
def backward(self, grad):
batch_size = self._out.shape[0]
dx = np.empty_like(grad)
for i in range(batch_size):
s = self._out[i].reshape(-1, 1)
jacobian = np.diagflat(s) - s @ s.T
dx[i] = jacobian @ grad[i]
return dx
[docs]
class Softplus(Activation):
[docs]
def forward(self, x):
self._input = x
return np.log1p(np.exp(-np.abs(x))) + np.maximum(x, 0)
[docs]
def backward(self, grad):
return grad / (1.0 + np.exp(-self._input))
[docs]
class Swish(Activation):
[docs]
def forward(self, x):
self._input = x
self._sig = 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))
return x * self._sig
[docs]
def backward(self, grad):
swish = self._input * self._sig
return grad * (swish + self._sig * (1.0 - swish))
[docs]
class Mish(Activation):
[docs]
def forward(self, x):
self._input = x
sp = np.log1p(np.exp(np.clip(x, -500, 500)))
self._tanh_sp = np.tanh(sp)
return x * self._tanh_sp
[docs]
def backward(self, grad):
sp = np.log1p(np.exp(np.clip(self._input, -500, 500)))
sig = 1.0 / (1.0 + np.exp(-np.clip(self._input, -500, 500)))
tanh_sp = self._tanh_sp
dtanh = 1.0 - tanh_sp ** 2
dsp = sig
return grad * (tanh_sp + self._input * dtanh * dsp)
_REGISTRY = {
"linear": Linear,
"relu": ReLU,
"leaky_relu": LeakyReLU,
"elu": ELU,
"selu": SELU,
"sigmoid": Sigmoid,
"tanh": Tanh,
"softmax": Softmax,
"softplus": Softplus,
"swish": Swish,
"mish": Mish,
}
[docs]
def get(identifier):
if identifier is None:
return Linear()
if isinstance(identifier, Activation):
return identifier
if isinstance(identifier, str):
key = identifier.lower()
if key in _REGISTRY:
return _REGISTRY[key]()
raise ValueError(f"Unknown activation: '{identifier}'. Available: {list(_REGISTRY)}")
raise TypeError(f"Could not interpret activation: {identifier}")