Numpy实现激活函数

￡神魔★判官ぃ 2022-09-11 11:08 206阅读 0赞

import numpy as np
    
    
    # Collection of activation functions
    # Reference: https://en.wikipedia.org/wiki/Activation_function
    
    class Sigmoid():
        def __call__(self, x):
            return 1 / (1 + np.exp(-x))
    
        def gradient(self, x):
            return self.__call__(x) * (1 - self.__call__(x))
    
    
    class Softmax():
        def __call__(self, x):
            e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
            return e_x / np.sum(e_x, axis=-1, keepdims=True)
    
        def gradient(self, x):
            p = self.__call__(x)
            return p * (1 - p)
    
    
    class TanH():
        def __call__(self, x):
            return 2 / (1 + np.exp(-2 * x)) - 1
    
        def gradient(self, x):
            return 1 - np.power(self.__call__(x), 2)
    
    
    class ReLU():
        def __call__(self, x):
            return np.where(x >= 0, x, 0)
    
        def gradient(self, x):
            return np.where(x >= 0, 1, 0)
    
    
    class LeakyReLU():
        def __init__(self, alpha=0.2):
            self.alpha = alpha
    
        def __call__(self, x):
            return np.where(x >= 0, x, self.alpha * x)
    
        def gradient(self, x):
            return np.where(x >= 0, 1, self.alpha)
    
    
    class ELU(object):
        def __init__(self, alpha=0.1):
            self.alpha = alpha
    
        def __call__(self, x):
            return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))
    
        def gradient(self, x):
            return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)
    
    
    class SELU():
        # Reference : https://arxiv.org/abs/1706.02515,
        # https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
        def __init__(self):
            self.alpha = 1.6732632423543772848170429916717
            self.scale = 1.0507009873554804934193349852946
    
        def __call__(self, x):
            return self.scale * np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))
    
        def gradient(self, x):
            return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))
    
    
    class SoftPlus(object):
        def __call__(self, x):
            return np.log(1 + np.exp(x))
    
        def gradient(self, x):
            return 1 / (1 + np.exp(-x))
    
    
    class Swish(object):
        def __init__(self, b):
            self.b = b
    
        def __call__(self, x):
            return x * (np.exp(self.b * x) / (np.exp(self.b * x) + 1))
    
        def gradient(self, x):
            return self.b * x / (1 + np.exp(-self.b * x)) + (1 / (1 + np.exp(-self.b * x)))(
                1 - self.b * (x / (1 + np.exp(-self.b * x))))
    
    
    def sech(x):
        """sech函数"""
        return 2 / (np.exp(x) + np.exp(-x))
    
    
    def sigmoid(x):
        """sigmoid函数"""
        return 1 / (1 + np.exp(-x))
    
    
    def soft_plus(x):
        """softplus函数"""
        return np.log(1 + np.exp(x))
    
    
    def tan_h(x):
        """tanh函数"""
        return (np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x))
    
    
    class Mish:
    
        def __call__(self, x):
            return x * tan_h(soft_plus(x))
    
        def gradient(self, x):
            return sech(soft_plus(x)) * sech(soft_plus(x)) * x * sigmoid(x) + tan_h(soft_plus(x))
    
    class SILU(object):
    
        def __call__(self, x):
            return x * sigmoid(x)
    
        def gradient(self, x):
            return self.__call__(x) + sigmoid(x) * (1 - self.__call__(x))