2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license a fast c implementation of the tanh approximation of the gelu activation function see https arxiv orgabs1606 08415 this implementation is equivalent to newgelu and fastgelu but much faster however it is not an exact numerical match due to rounding errors implementation of the gelu activation function currently in google bert repo identical to openai gpt also see the gaussian error linear units paper https arxiv orgabs1606 08415 original implementation of the gelu activation function in google bert repo when initially created for information openai gpt s gelu is slightly different and gives slightly different results 0 5 x 1 torch tanhmath sqrt2 math pi x 0 044715 torch powx 3 this is now written in c in nn functional also see the gaussian error linear units paper https arxiv orgabs1606 08415 applies gelu approximation that is slower than quickgelu but more accurate see https github comhendrycksgelus applies gelu approximation that is fast but somewhat inaccurate see https github comhendrycksgelus clip the range of possible gelu outputs between min max this is especially useful for quantization purpose as it allows mapping negatives values in the gelu spectrum for more information on this trick please refer to https arxiv orgabs2004 09602 gaussian error linear unit original implementation of the gelu activation function in google bert repo when initially created for information openai gpt s gelu is slightly different and gives slightly different results 0 5 x 1 torch tanhmath sqrt2 math pi x 0 044715 torch powx 3 see https arxiv orgabs1606 08415 applies gelu approximation that is faster than default and more accurate than quickgelu see https github comhendrycksgelus implemented along with mega moving average equipped gated attention see mish a selfregularized nonmonotonic activation function misra https arxiv orgabs1908 08681 also visit the official repository for the paper https github comdigantamisra98mish applies the linear activation function i e forwarding input directly to output applies elementwise activation based on laplace function introduced in mega as an attention activation see https arxiv orgabs2209 10655 inspired by squared relu but with bounded range and gradient for better stability applies the relu2 activation introduced in https arxiv orgabs2109 08668v2 for backwards compatibility with from activations import gelupython 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license a fast c implementation of the tanh approximation of the gelu activation function see https arxiv org abs 1606 08415 this implementation is equivalent to newgelu and fastgelu but much faster however it is not an exact numerical match due to rounding errors implementation of the gelu activation function currently in google bert repo identical to openai gpt also see the gaussian error linear units paper https arxiv org abs 1606 08415 original implementation of the gelu activation function in google bert repo when initially created for information openai gpt s gelu is slightly different and gives slightly different results 0 5 x 1 torch tanh math sqrt 2 math pi x 0 044715 torch pow x 3 this is now written in c in nn functional also see the gaussian error linear units paper https arxiv org abs 1606 08415 applies gelu approximation that is slower than quickgelu but more accurate see https github com hendrycks gelus applies gelu approximation that is fast but somewhat inaccurate see https github com hendrycks gelus clip the range of possible gelu outputs between min max this is especially useful for quantization purpose as it allows mapping negatives values in the gelu spectrum for more information on this trick please refer to https arxiv org abs 2004 09602 gaussian error linear unit original implementation of the gelu activation function in google bert repo when initially created for information openai gpt s gelu is slightly different and gives slightly different results 0 5 x 1 torch tanh math sqrt 2 math pi x 0 044715 torch pow x 3 see https arxiv org abs 1606 08415 applies gelu approximation that is faster than default and more accurate than quickgelu see https github com hendrycks gelus implemented along with mega moving average equipped gated attention see mish a self regularized non monotonic activation function misra https arxiv org abs 1908 08681 also visit the official repository for the paper https github com digantamisra98 mish applies the linear activation function i e forwarding input directly to output applies elementwise activation based on laplace function introduced in mega as an attention activation see https arxiv org abs 2209 10655 inspired by squared relu but with bounded range and gradient for better stability applies the relu 2 activation introduced in https arxiv org abs 2109 08668v2 for backwards compatibility with from activations import gelu_python
import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


logger = logging.get_logger(__name__)


class PytorchGELUTanh(nn.Module):
    

    def __init__(self):
        super().__init__()
        if version.parse(torch.__version__) < version.parse("1.12.0"):
            raise ImportError(
                f"You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use "
                "PytorchGELUTanh. Please upgrade torch."
            )

    def forward(self, input: Tensor) -> Tensor:
        return nn.functional.gelu(input, approximate="tanh")


class NewGELUActivation(nn.Module):
    

    def forward(self, input: Tensor) -> Tensor:
        return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.044715 * torch.pow(input, 3.0))))


class GELUActivation(nn.Module):
    

    def __init__(self, use_gelu_python: bool = False):
        super().__init__()
        if use_gelu_python:
            self.act = self._gelu_python
        else:
            self.act = nn.functional.gelu

    def _gelu_python(self, input: Tensor) -> Tensor:
        return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0)))

    def forward(self, input: Tensor) -> Tensor:
        return self.act(input)


class FastGELUActivation(nn.Module):
    

    def forward(self, input: Tensor) -> Tensor:
        return 0.5 * input * (1.0 + torch.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input)))


class QuickGELUActivation(nn.Module):
    

    def forward(self, input: Tensor) -> Tensor:
        return input * torch.sigmoid(1.702 * input)


class ClippedGELUActivation(nn.Module):
    

    def __init__(self, min: float, max: float):
        if min > max:
            raise ValueError(f"min should be < max (got min: {min}, max: {max})")

        super().__init__()
        self.min = min
        self.max = max

    def forward(self, x: Tensor) -> Tensor:
        return torch.clip(gelu(x), self.min, self.max)


class AccurateGELUActivation(nn.Module):
    

    def __init__(self):
        super().__init__()
        self.precomputed_constant = math.sqrt(2 / math.pi)

    def forward(self, input: Tensor) -> Tensor:
        return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044715 * torch.pow(input, 3))))


class MishActivation(nn.Module):
    

    def __init__(self):
        super().__init__()
        if version.parse(torch.__version__) < version.parse("1.9.0"):
            self.act = self._mish_python
        else:
            self.act = nn.functional.mish

    def _mish_python(self, input: Tensor) -> Tensor:
        return input * torch.tanh(nn.functional.softplus(input))

    def forward(self, input: Tensor) -> Tensor:
        return self.act(input)


class LinearActivation(nn.Module):
    

    def forward(self, input: Tensor) -> Tensor:
        return input


class LaplaceActivation(nn.Module):
    

    def forward(self, input, mu=0.707107, sigma=0.282095):
        input = (input - mu).div(sigma * math.sqrt(2.0))
        return 0.5 * (1.0 + torch.erf(input))


class ReLUSquaredActivation(nn.Module):
    

    def forward(self, input):
        relu_applied = nn.functional.relu(input)
        squared = torch.square(relu_applied)
        return squared


class ClassInstantier(OrderedDict):
    def __getitem__(self, key):
        content = super().__getitem__(key)
        cls, kwargs = content if isinstance(content, tuple) else (content, {})
        return cls(**kwargs)


ACT2CLS = {
    "gelu": GELUActivation,
    "gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}),
    "gelu_fast": FastGELUActivation,
    "gelu_new": NewGELUActivation,
    "gelu_python": (GELUActivation, {"use_gelu_python": True}),
    "gelu_pytorch_tanh": PytorchGELUTanh,
    "gelu_accurate": AccurateGELUActivation,
    "laplace": LaplaceActivation,
    "leaky_relu": nn.LeakyReLU,
    "linear": LinearActivation,
    "mish": MishActivation,
    "quick_gelu": QuickGELUActivation,
    "relu": nn.ReLU,
    "relu2": ReLUSquaredActivation,
    "relu6": nn.ReLU6,
    "sigmoid": nn.Sigmoid,
    "silu": nn.SiLU,
    "swish": nn.SiLU,
    "tanh": nn.Tanh,
}
ACT2FN = ClassInstantier(ACT2CLS)


def get_activation(activation_string):
    if activation_string in ACT2FN:
        return ACT2FN[activation_string]
    else:
        raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}")



gelu_python = get_activation("gelu_python")
gelu_new = get_activation("gelu_new")
gelu = get_activation("gelu")
gelu_fast = get_activation("gelu_fast")
quick_gelu = get_activation("quick_gelu")
silu = get_activation("silu")
mish = get_activation("mish")
linear_act = get_activation("linear")