2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license deal with dynamic shape in tensorflow cleanly args tensor tf tensor or np ndarray the tensor we want the shape of returns listint the shape of the tensor as a list stable wrapper that returns the same output as tf nn softmax but that works reliably with xla on cpu it is meant as a workaround for the following issuehttps github comtensorflowtensorflowissues55682 and will be removed after it gets fixed the arguments and outputs are the same as tf nn softmax and relies on the fact that softmaxx softmaxx c see https ogunlao github io20200426youdontreallyknowsoftmax html args logits tf tensor must be one of the following types half float32 float64 axis int optional the dimension softmax would be performed on the default is 1 which indicates the last dimension name str optional a name for the operation returns tf tensor a tensor has the same type and shape as logits todo when the issue linked above gets sorted add a check on tf version here and use the original function if it has the fix after we drop the support for unfixed versions remove this function this is a very simplified functional layernorm designed to duplicate the functionality of pytorch nn functional layernorm when this is needed to port models in transformers get mean and variance on the axis to be normalized reshape scale and weight to have the same rank as inputs but with 1 dimensions on every dimension except axis compute layer normalization using the batchnormalization function replicates the behavior of torch flatten in tf if enddim or startdim is negative count them from the end invert an attention mask e g switches 0 and 1 args encoderattentionmask torch tensor an attention mask returns tf tensor the inverted attention mask t5 has a mask that can compare sequence ids we can simulate this here with this transposition cf https github comtensorflowmeshblob8d2465e9bc93129b913b5ccc6a59aa97abd96ec6meshtensorflow transformertransformerlayers pyl270 encoderextendedattentionmask encoderextendedattentionmask encoderextendedattentionmask transpose1 2 tf gather on which tf embedding layers are based won t check positive out of bound indices on gpu returning zeros instead this function adds a check against that dangerous silent behavior args tensor tf tensor the tensor of indices to check embeddim int the embedding dimension tensorname str optional the name of the tensor to use in the error message saves attributes data of the specified name into the hdf5 group this method deals with an inherent problem of hdf5 file which is not able to store data larger than hdf5objectheaderlimit bytes args group a pointer to a hdf5 group name a name of the attributes to save data attributes data to store raises runtimeerror if any single attribute is too large to be saved copied from keras to transformers to avoid versioning issues check that no item in data is larger than hdf5objectheaderlimit because in that case even chunking the array would not make the saving possible expecting this to never be true this will never loop forever thanks to the test above loads attributes of the specified name from the hdf5 group this method deals with an inherent problem of hdf5 file which is not able to store data larger than hdf5objectheaderlimit bytes args group a pointer to a hdf5 group name a name of the attributes to load returns data attributes data copied from keras to transformers to avoid versioning issues expands 1dimensional tensors into 2dimensional tensors copied from keras to here to avoid versioning issues def expandsingle1dtensort if isinstancet tf tensor and t shape rank 1 return tf expanddimst axis1 return t return tf nest mapstructureexpandsingle1dtensor data 2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license deal with dynamic shape in tensorflow cleanly args tensor tf tensor or np ndarray the tensor we want the shape of returns list int the shape of the tensor as a list stable wrapper that returns the same output as tf nn softmax but that works reliably with xla on cpu it is meant as a workaround for the following issue https github com tensorflow tensorflow issues 55682 and will be removed after it gets fixed the arguments and outputs are the same as tf nn softmax and relies on the fact that softmax x softmax x c see https ogunlao github io 2020 04 26 you_dont_really_know_softmax html args logits tf tensor must be one of the following types half float32 float64 axis int optional the dimension softmax would be performed on the default is 1 which indicates the last dimension name str optional a name for the operation returns tf tensor a tensor has the same type and shape as logits todo when the issue linked above gets sorted add a check on tf version here and use the original function if it has the fix after we drop the support for unfixed versions remove this function this is a very simplified functional layernorm designed to duplicate the functionality of pytorch nn functional layer_norm when this is needed to port models in transformers get mean and variance on the axis to be normalized reshape scale and weight to have the same rank as inputs but with 1 dimensions on every dimension except axis compute layer normalization using the batch_normalization function replicates the behavior of torch flatten in tf if end_dim or start_dim is negative count them from the end invert an attention mask e g switches 0 and 1 args encoder_attention_mask torch tensor an attention mask returns tf tensor the inverted attention mask catches stray numpy inputs t5 has a mask that can compare sequence ids we can simulate this here with this transposition cf https github com tensorflow mesh blob 8d2465e9bc93129b913b5ccc6a59aa97abd96ec6 mesh_tensorflow transformer transformer_layers py l270 encoder_extended_attention_mask encoder_extended_attention_mask encoder_extended_attention_mask transpose 1 2 tf gather on which tf embedding layers are based won t check positive out of bound indices on gpu returning zeros instead this function adds a check against that dangerous silent behavior args tensor tf tensor the tensor of indices to check embed_dim int the embedding dimension tensor_name str optional the name of the tensor to use in the error message saves attributes data of the specified name into the hdf5 group this method deals with an inherent problem of hdf5 file which is not able to store data larger than hdf5_object_header_limit bytes args group a pointer to a hdf5 group name a name of the attributes to save data attributes data to store raises runtimeerror if any single attribute is too large to be saved copied from keras to transformers to avoid versioning issues check that no item in data is larger than hdf5_object_header_limit because in that case even chunking the array would not make the saving possible expecting this to never be true this will never loop forever thanks to the test above loads attributes of the specified name from the hdf5 group this method deals with an inherent problem of hdf5 file which is not able to store data larger than hdf5_object_header_limit bytes args group a pointer to a hdf5 group name a name of the attributes to load returns data attributes data copied from keras to transformers to avoid versioning issues expands 1 dimensional tensor s into 2 dimensional tensor s copied from keras to here to avoid versioning issues
from typing import List, Optional, Union

import numpy as np
import tensorflow as tf

from .utils import logging


logger = logging.get_logger(__name__)


def shape_list(tensor: Union[tf.Tensor, np.ndarray]) -> List[int]:
    
    if isinstance(tensor, np.ndarray):
        return list(tensor.shape)

    dynamic = tf.shape(tensor)

    if tensor.shape == tf.TensorShape(None):
        return dynamic

    static = tensor.shape.as_list()

    return [dynamic[i] if s is None else s for i, s in enumerate(static)]


def stable_softmax(logits: tf.Tensor, axis: Optional[int] = None, name: Optional[str] = None) -> tf.Tensor:
    
    
    
    return tf.nn.softmax(logits=logits + 1e-9, axis=axis, name=name)


def functional_layernorm(inputs, weight, bias, epsilon=1e-5, axis=-1):
    
    
    

    if weight.shape.rank != 1 or bias.shape.rank != 1 or not isinstance(axis, int):
        raise NotImplementedError("Only 1D weight and bias tensors are supported for now, with only a single axis.")

    
    mean, variance = tf.nn.moments(inputs, axes=[axis], keepdims=True)

    if axis != -1:
        
        
        shape = [1] * inputs.shape.rank
        shape[axis] = shape_list(inputs)[axis]
        weight = tf.reshape(weight, shape)
        bias = tf.reshape(bias, shape)

    
    
    outputs = tf.nn.batch_normalization(
        inputs,
        mean,
        variance,
        offset=bias,
        scale=weight,
        variance_epsilon=epsilon,
    )
    return outputs


def flatten(input, start_dim=0, end_dim=-1):
    

    
    if end_dim < 0:
        end_dim += input.shape.rank
    if start_dim < 0:
        start_dim += input.shape.rank

    if start_dim == end_dim:
        return input

    in_shape = tf.shape(input)
    flattened_dim = tf.math.reduce_prod(in_shape[start_dim : end_dim + 1])
    out_shape = tf.concat([in_shape[:start_dim], [flattened_dim], in_shape[end_dim + 1 :]], axis=0)
    return tf.reshape(input, out_shape)


def invert_attention_mask(encoder_attention_mask: tf.Tensor) -> tf.Tensor:
    
    if not isinstance(encoder_attention_mask, tf.Tensor):
        encoder_attention_mask = tf.convert_to_tensor(encoder_attention_mask)  
    if encoder_attention_mask.shape.rank == 3:
        encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :]
    if encoder_attention_mask.shape.rank == 2:
        encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :]
    
    
    
    
    
    encoder_extended_attention_mask = (
        tf.cast(1, encoder_attention_mask.dtype) - encoder_extended_attention_mask
    ) * encoder_extended_attention_mask.dtype.min

    return encoder_extended_attention_mask


def check_embeddings_within_bounds(tensor: tf.Tensor, embed_dim: int, tensor_name: str = "input_ids") -> None:
    
    tf.debugging.assert_less(
        tensor,
        tf.cast(embed_dim, dtype=tensor.dtype),
        message=(
            f"The maximum value of {tensor_name} ({tf.math.reduce_max(tensor)}) must be smaller than the embedding "
            f"layer's input dimension ({embed_dim}). The likely cause is some problem at tokenization time."
        ),
    )


def save_attributes_to_hdf5_group(group, name, data):
    
    HDF5_OBJECT_HEADER_LIMIT = 64512
    
    
    
    bad_attributes = [x for x in data if len(x) > HDF5_OBJECT_HEADER_LIMIT]

    
    if bad_attributes:
        raise RuntimeError(
            "The following attributes cannot be saved to HDF5 file because "
            f"they are larger than {HDF5_OBJECT_HEADER_LIMIT} "
            f"bytes: {bad_attributes}"
        )

    data_npy = np.asarray(data)

    num_chunks = 1
    chunked_data = np.array_split(data_npy, num_chunks)

    
    while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
        num_chunks += 1
        chunked_data = np.array_split(data_npy, num_chunks)

    if num_chunks > 1:
        for chunk_id, chunk_data in enumerate(chunked_data):
            group.attrs["%s%d" % (name, chunk_id)] = chunk_data
    else:
        group.attrs[name] = data


def load_attributes_from_hdf5_group(group, name):
    
    if name in group.attrs:
        data = [n.decode("utf8") if hasattr(n, "decode") else n for n in group.attrs[name]]
    else:
        data = []
        chunk_id = 0
        while "%s%d" % (name, chunk_id) in group.attrs:
            data.extend(
                [n.decode("utf8") if hasattr(n, "decode") else n for n in group.attrs["%s%d" % (name, chunk_id)]]
            )
            chunk_id += 1
    return data


def expand_1d(data):
    

    def _expand_single_1d_tensor(t):
        if isinstance(t, tf.Tensor) and t.shape.rank == 1:
            return tf.expand_dims(t, axis=-1)
        return t

    return tf.nest.map_structure(_expand_single_1d_tensor, data)