<commit_before>/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

#include <arith.hpp>
#include <backend.hpp>
#include <cast.hpp>
#include <common/err_common.hpp>
#include <common/half.hpp>
#include <handle.hpp>
#include <math.hpp>
#include <mean.hpp>
#include <reduce.hpp>
#include <tile.hpp>
#include <af/defines.h>
#include <af/dim4.hpp>
#include <af/statistics.h>

#include "stats.h"

#include <tuple>

using namespace detail;

using common::half;
using std::ignore;
using std::make_tuple;
using std::tie;
using std::tuple;

template<typename inType, typename outType>
static outType varAll(const af_array& in, const bool isbiased) {
    using weightType          = typename baseOutType<outType>::type;
    const Array<inType> inArr = getArray<inType>(in);
    Array<outType> input      = cast<outType>(inArr);

    Array<outType> meanCnst = createValueArray<outType>(
        input.dims(), mean<inType, weightType, outType>(inArr));

    Array<outType> diff =
        arithOp<outType, af_sub_t>(input, meanCnst, input.dims());

    Array<outType> diffSq = arithOp<outType, af_mul_t>(diff, diff, diff.dims());

    outType result =
        division(reduce_all<af_add_t, outType, outType>(diffSq),
                 isbiased ? input.elements() : input.elements() - 1);

    return result;
}

template<typename inType, typename outType>
static outType varAll(const af_array& in, const af_array weights) {
    using bType = typename baseOutType<outType>::type;

    Array<outType> input = cast<outType>(getArray<inType>(in));
    Array<outType> wts   = cast<outType>(getArray<bType>(weights));

    bType wtsSum = reduce_all<af_add_t, bType, bType>(getArray<bType>(weights));
    auto wtdMean = mean<outType, bType>(input, getArray<bType>(weights));

    Array<outType> meanArr = createValueArray<outType>(input.dims(), wtdMean);
    Array<outType> diff =
        arithOp<outType, af_sub_t>(input, meanArr, input.dims());
    Array<outType> diffSq = arithOp<outType, af_mul_t>(diff, diff, diff.dims());

    Array<outType> accDiffSq =
        arithOp<outType, af_mul_t>(diffSq, wts, diffSq.dims());

    outType result =
        division(reduce_all<af_add_t, outType, outType>(accDiffSq), wtsSum);

    return result;
}

template<typename inType, typename outType>
static tuple<Array<outType>, Array<outType>> meanvar(
    const Array<inType>& in,
    const Array<typename baseOutType<outType>::type>& weights,
    const af_var_bias bias, const dim_t dim) {
    using weightType     = typename baseOutType<outType>::type;
    Array<outType> input = cast<outType>(in);
    dim4 iDims           = input.dims();

    Array<outType> meanArr = createEmptyArray<outType>({0});
    Array<outType> normArr = createEmptyArray<outType>({0});
    if (weights.isEmpty()) {
        meanArr  = mean<outType, weightType, outType>(input, dim);
        auto val = 1.0 / (bias == AF_VARIANCE_POPULATION ? iDims[dim]
                                                         : iDims[dim] - 1);
        normArr =
            createValueArray<outType>(meanArr.dims(), scalar<outType>(val));
    } else {
        meanArr               = mean<outType, weightType>(input, weights, dim);
        Array<outType> wtsSum = cast<outType>(
            reduce<af_add_t, weightType, weightType>(weights, dim));
        Array<outType> ones =
            createValueArray<outType>(wtsSum.dims(), scalar<outType>(1));
        if (bias == AF_VARIANCE_SAMPLE) {
            wtsSum = arithOp<outType, af_sub_t>(wtsSum, ones, ones.dims());
        }
        normArr = arithOp<outType, af_div_t>(ones, wtsSum, meanArr.dims());
    }

    /* now tile meanArr along dim and use it for variance computation */
    dim4 tileDims(1);
    tileDims[dim]           = iDims[dim];
    Array<outType> tMeanArr = tile<outType>(meanArr, tileDims);
    /* now mean array is ready */

    Array<outType> diff =
        arithOp<outType, af_sub_t>(input, tMeanArr, tMeanArr.dims());
    Array<outType> diffSq = arithOp<outType, af_mul_t>(diff, diff, diff.dims());
    Array<outType> redDiff = reduce<af_add_t, outType, outType>(diffSq, dim);

    Array<outType> variance =
        arithOp<outType, af_mul_t>(normArr, redDiff, redDiff.dims());

    return make_tuple(meanArr, variance);
}

template<typename inType, typename outType>
static tuple<af_array, af_array> meanvar(const af_array& in,
                                         const af_array& weights,
                                         const af_var_bias bias,
                                         const dim_t dim) {
    using weightType    = typename baseOutType<outType>::type;
    Array<outType> mean = createEmptyArray<outType>({0}),
                   var  = createEmptyArray<outType>({0});

    Array<weightType> w = createEmptyArray<weightType>({0});
    if (weights != 0) { w = getArray<weightType>(weights); }
    tie(mean, var) =
        meanvar<inType, outType>(getArray<inType>(in), w, bias, dim);
    return make_tuple(getHandle(mean), getHandle(var));
}<commit_msg>Remove unnecessary tile from var. Use arith output parameter instead<commit_after>/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

#include <arith.hpp>
#include <backend.hpp>
#include <cast.hpp>
#include <common/err_common.hpp>
#include <common/half.hpp>
#include <handle.hpp>
#include <math.hpp>
#include <mean.hpp>
#include <reduce.hpp>
#include <af/defines.h>
#include <af/dim4.hpp>
#include <af/statistics.h>

#include "stats.h"

#include <tuple>

using namespace detail;

using common::half;
using std::ignore;
using std::make_tuple;
using std::tie;
using std::tuple;

template<typename inType, typename outType>
static outType varAll(const af_array& in, const bool isbiased) {
    using weightType          = typename baseOutType<outType>::type;
    const Array<inType> inArr = getArray<inType>(in);
    Array<outType> input      = cast<outType>(inArr);

    Array<outType> meanCnst = createValueArray<outType>(
        input.dims(), mean<inType, weightType, outType>(inArr));

    Array<outType> diff =
        arithOp<outType, af_sub_t>(input, meanCnst, input.dims());

    Array<outType> diffSq = arithOp<outType, af_mul_t>(diff, diff, diff.dims());

    outType result =
        division(reduce_all<af_add_t, outType, outType>(diffSq),
                 isbiased ? input.elements() : input.elements() - 1);

    return result;
}

template<typename inType, typename outType>
static outType varAll(const af_array& in, const af_array weights) {
    using bType = typename baseOutType<outType>::type;

    Array<outType> input = cast<outType>(getArray<inType>(in));
    Array<outType> wts   = cast<outType>(getArray<bType>(weights));

    bType wtsSum = reduce_all<af_add_t, bType, bType>(getArray<bType>(weights));
    auto wtdMean = mean<outType, bType>(input, getArray<bType>(weights));

    Array<outType> meanArr = createValueArray<outType>(input.dims(), wtdMean);
    Array<outType> diff =
        arithOp<outType, af_sub_t>(input, meanArr, input.dims());
    Array<outType> diffSq = arithOp<outType, af_mul_t>(diff, diff, diff.dims());

    Array<outType> accDiffSq =
        arithOp<outType, af_mul_t>(diffSq, wts, diffSq.dims());

    outType result =
        division(reduce_all<af_add_t, outType, outType>(accDiffSq), wtsSum);

    return result;
}

template<typename inType, typename outType>
static tuple<Array<outType>, Array<outType>> meanvar(
    const Array<inType>& in,
    const Array<typename baseOutType<outType>::type>& weights,
    const af_var_bias bias, const dim_t dim) {
    using weightType     = typename baseOutType<outType>::type;
    Array<outType> input = cast<outType>(in);
    dim4 iDims           = input.dims();

    Array<outType> meanArr = createEmptyArray<outType>({0});
    Array<outType> normArr = createEmptyArray<outType>({0});
    if (weights.isEmpty()) {
        meanArr  = mean<outType, weightType, outType>(input, dim);
        auto val = 1.0 / (bias == AF_VARIANCE_POPULATION ? iDims[dim]
                                                         : iDims[dim] - 1);
        normArr =
            createValueArray<outType>(meanArr.dims(), scalar<outType>(val));
    } else {
        meanArr               = mean<outType, weightType>(input, weights, dim);
        Array<outType> wtsSum = cast<outType>(
            reduce<af_add_t, weightType, weightType>(weights, dim));
        Array<outType> ones =
            createValueArray<outType>(wtsSum.dims(), scalar<outType>(1));
        if (bias == AF_VARIANCE_SAMPLE) {
            wtsSum = arithOp<outType, af_sub_t>(wtsSum, ones, ones.dims());
        }
        normArr = arithOp<outType, af_div_t>(ones, wtsSum, meanArr.dims());
    }

    Array<outType> diff =
        arithOp<outType, af_sub_t>(input, meanArr, input.dims());
    Array<outType> diffSq = arithOp<outType, af_mul_t>(diff, diff, diff.dims());
    Array<outType> redDiff = reduce<af_add_t, outType, outType>(diffSq, dim);

    Array<outType> variance =
        arithOp<outType, af_mul_t>(normArr, redDiff, redDiff.dims());

    return make_tuple(meanArr, variance);
}

template<typename inType, typename outType>
static tuple<af_array, af_array> meanvar(const af_array& in,
                                         const af_array& weights,
                                         const af_var_bias bias,
                                         const dim_t dim) {
    using weightType    = typename baseOutType<outType>::type;
    Array<outType> mean = createEmptyArray<outType>({0}),
                   var  = createEmptyArray<outType>({0});

    Array<weightType> w = createEmptyArray<weightType>({0});
    if (weights != 0) { w = getArray<weightType>(weights); }
    tie(mean, var) =
        meanvar<inType, outType>(getArray<inType>(in), w, bias, dim);
    return make_tuple(getHandle(mean), getHandle(var));
}
{'fraction_non_alphanumeric': 0.15120441133791235, 'fraction_numerical': 0.0038695946599593693, 'mean_word_length': 3.473388143660753, 'pattern_counts': {'":': 0, '<': 192, '<?xml version=': 0, '>': 192, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6111077', 'n_tokens_mistral': 3359, 'n_tokens_neox': 3301, 'n_words': 713}