<commit_before>// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import TensorFlow

// Contains operators needed for Transformer; these will likely be upstreamed into swift-apis

/// Computes the Gaussian error linear unit (GELU) nonlinear activation function
@differentiable
func gelu<Scalar: TensorFlowFloatingPoint>(_ x: Tensor<Scalar>) -> Tensor<Scalar> {
    let xCubed = x * x * x
    let polynomial = 0.79788456 * (x + 0.044715 * xCubed)
    return 0.5 * x * (1.0 + tanh(polynomial))
}

/// Performs batched matrix multiplication of two tensors. The last two axes of each tensor
/// are treated as the matrix axes; all others are treated as batch axes.
@differentiable(wrt: (left, right) where Scalar : Differentiable & TensorFlowFloatingPoint)
func batchedMatmul<Scalar : Numeric>(
    _ left: Tensor<Scalar>,
    _ right: Tensor<Scalar>,
    adjointLeft: Bool = false,
    adjointRight: Bool = false<commit_msg>[AutoDiff] Fix Transformer CMake build linker error. (#329)<commit_after>// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import TensorFlow

// Contains operators needed for Transformer; these will likely be upstreamed into swift-apis

/// Computes the Gaussian error linear unit (GELU) nonlinear activation function
@differentiable
func gelu<Scalar: TensorFlowFloatingPoint>(_ x: Tensor<Scalar>) -> Tensor<Scalar> {
    let xCubed = x * x * x
    let polynomial = 0.79788456 * (x + 0.044715 * xCubed)
    return 0.5 * x * (1.0 + tanh(polynomial))
}

/// Performs batched matrix multiplication of two tensors. The last two axes of each tensor
/// are treated as the matrix axes; all others are treated as batch axes.
@usableFromInline
@differentiable(wrt: (left, right) where Scalar : Differentiable & TensorFlowFloatingPoint)
func batchedMatmul<Scalar : Numeric>(
    _ left: Tensor<Scalar>,
    _ right: Tensor<Scalar>,
    adjointLeft: Bool = false,
    adjointRight: Bool = false
{'fraction_non_alphanumeric': 0.09022309711286089, 'fraction_numerical': 0.01935695538057743, 'mean_word_length': 3.824367088607595, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5294927', 'n_tokens_mistral': 880, 'n_tokens_neox': 813, 'n_words': 369}