<commit_before>/*
 * Copyright 2014 <NAME>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * This code is part of the reference implementation of http://arxiv.org/abs/1409.6075
 * 
 * This is provided as an example to help in the understanding of the ITEM model system.
 */
package edu.columbia.tjw.item.algo;

import edu.columbia.tjw.item.algo.QuantApprox.QuantileNode;
import edu.columbia.tjw.item.data.InterpolatedCurve;
import java.io.Serializable;

/**
 *
 * @author tyler
 */
public final class QuantileDistribution implements Serializable
{
    private static final long serialVersionUID = 1116222802982789849L;

    private final double[] _eX;
    private final double[] _devX;
    private final double[] _eY;
    private final double[] _devY;
    private final long[] _count;
    private final long _totalCount;

    //Some stats on the global distribution (across all buckets). 
    private final double _meanX;
    private final double _meanY;
    private final double _meanDevX;
    private final double _meanDevY;

    public QuantileDistribution(final double[] eX_, final double[] eY_, final double[] devX_, final double[] devY_, final long[] count_, final boolean doCopy_)
    {
        final int size = eX_.length;

        if (size != eY_.length || size != devX_.length || size != devY_.length || size != count_.length)
        {
            throw new IllegalArgumentException("Length mismatch.");
        }

        double sumX = 0.0;
        double sumY = 0.0;
        double sumX2 = 0.0;
        double sumY2 = 0.0;

        long count = 0;

        if (doCopy_)
        {
            _eX = eX_.clone();
            _eY = eY_.clone();
            _devX = devX_.clone();
            _devY = devY_.clone();
            _count = count_.clone();
        }
        else
        {
            _eX = eX_;
            _eY = eY_;
            _devX = devX_;
            _devY = devY_;
            _count = count_;
        }

        for (int i = 0; i < size; i++)
        {
            final long bucketCount = _count[i];
            final double eXTerm = _eX[i];
            final double eYTerm = _eY[i];

            final double termX = eXTerm * bucketCount;
            final double termY = eYTerm * bucketCount;
            final double termX2 = bucketCount * eXTerm * eXTerm;
            final double termY2 = bucketCount * eYTerm * eYTerm;
            final double bucketVarX = _devX[i] * _devX[i] * bucketCount;

            sumX += termX;
            sumX2 += termX2;
            sumY += termY;
            sumY2 += termY2;

            count += bucketCount;
        }

        _meanX = sumX / count;
        _meanY = sumY / count;

        _meanDevX = Math.sqrt(DistMath.computeMeanVariance(sumX, sumX2, count));
        _meanDevY = Math.sqrt(DistMath.computeMeanVariance(sumY, sumY2, count));

        _totalCount = count;
    }

    public QuantileDistribution(final QuantApprox approx_)
    {
        final int approxSize = approx_.size();

        _eX = new double[approxSize];
        _devX = new double[approxSize];
        _eY = new double[approxSize];
        _devY = new double[approxSize];

        _count = new long[approxSize];

        long totalCount = 0;

        int pointer = 0;

        for (final QuantileNode next : approx_)
        {
            _eX[pointer] = next.getMeanX();
            _devX[pointer] = next.getStdDevX();
            _eY[pointer] = next.getMeanY();
            _devY[pointer] = next.getStdDevY();

            final long count = next.getCount();
            _count[pointer] = count;
            totalCount += count;
            pointer++;
        }

        _totalCount = totalCount;
        _meanX = approx_.getMeanX();
        _meanY = approx_.getMeanY();
        _meanDevX = approx_.getStdDevX();
        _meanDevY = approx_.getStdDevY();
    }

    public QuantileDistribution alphaTrim(final double alpha_)
    {
        if (alpha_ == 0)
        {
            return this;
        }

        if (alpha_ < 0 || alpha_ >= 0.5)
        {
            throw new IllegalArgumentException("Alpha (for trimming) must be in [0, 0.5]: " + alpha_);
        }
<commit_msg>Minor fix for zero mass nodes. <commit_after>/*
 * Copyright 2014 <NAME>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * This code is part of the reference implementation of http://arxiv.org/abs/1409.6075
 * 
 * This is provided as an example to help in the understanding of the ITEM model system.
 */
package edu.columbia.tjw.item.algo;

import edu.columbia.tjw.item.algo.QuantApprox.QuantileNode;
import edu.columbia.tjw.item.data.InterpolatedCurve;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

/**
 *
 * @author tyler
 */
public final class QuantileDistribution implements Serializable
{
    private static final long serialVersionUID = 1116222802982789849L;

    private final double[] _eX;
    private final double[] _devX;
    private final double[] _eY;
    private final double[] _devY;
    private final long[] _count;
    private final long _totalCount;

    //Some stats on the global distribution (across all buckets). 
    private final double _meanX;
    private final double _meanY;
    private final double _meanDevX;
    private final double _meanDevY;

    public QuantileDistribution(final double[] eX_, final double[] eY_, final double[] devX_, final double[] devY_, final long[] count_, final boolean doCopy_)
    {
        final int size = eX_.length;

        if (size != eY_.length || size != devX_.length || size != devY_.length || size != count_.length)
        {
            throw new IllegalArgumentException("Length mismatch.");
        }

        double sumX = 0.0;
        double sumY = 0.0;
        double sumX2 = 0.0;
        double sumY2 = 0.0;

        long count = 0;

        if (doCopy_)
        {
            _eX = eX_.clone();
            _eY = eY_.clone();
            _devX = devX_.clone();
            _devY = devY_.clone();
            _count = count_.clone();
        }
        else
        {
            _eX = eX_;
            _eY = eY_;
            _devX = devX_;
            _devY = devY_;
            _count = count_;
        }

        for (int i = 0; i < size; i++)
        {
            final long bucketCount = _count[i];
            final double eXTerm = _eX[i];
            final double eYTerm = _eY[i];

            final double termX = eXTerm * bucketCount;
            final double termY = eYTerm * bucketCount;
            final double termX2 = bucketCount * eXTerm * eXTerm;
            final double termY2 = bucketCount * eYTerm * eYTerm;
            final double bucketVarX = _devX[i] * _devX[i] * bucketCount;

            sumX += termX;
            sumX2 += termX2;
            sumY += termY;
            sumY2 += termY2;

            count += bucketCount;
        }

        _meanX = sumX / count;
        _meanY = sumY / count;

        _meanDevX = Math.sqrt(DistMath.computeMeanVariance(sumX, sumX2, count));
        _meanDevY = Math.sqrt(DistMath.computeMeanVariance(sumY, sumY2, count));

        _totalCount = count;
    }

    public QuantileDistribution(final QuantApprox approx_)
    {
        final List<QuantileNode> nodes = new ArrayList<>(approx_.size());

        for (final QuantileNode next : approx_)
        {
            if (next.getCount() > 0)
            {
                nodes.add(next);
            }
        }

        final int approxSize = nodes.size();

        _eX = new double[approxSize];
        _devX = new double[approxSize];
        _eY = new double[approxSize];
        _devY = new double[approxSize];

        _count = new long[approxSize];

        long totalCount = 0;

        int pointer = 0;

        for (final QuantileNode next : nodes)
        {
            _eX[pointer] = next.getMeanX();
            _devX[pointer] = next.getStdDevX();
            _eY[pointer] = next.getMeanY();
            _devY[pointer] = next.getStdDevY();

            final long count = next.getCount();
            _count[pointer] = count;
            totalCount += count;
            pointer++;
        }

        _totalCount = totalCount;
        _meanX = approx_.getMeanX();
        _meanY = approx_.getMeanY();
        _meanDevX = approx_.getStdDevX();
        _meanDevY = approx_.getStdDevY();
    }

    public QuantileDistribution alphaTrim(final double alpha_)
    {
        if (alpha_ == 0)
        {
            return this;
        }

        if (alpha_ < 0 || alpha_ >= 0.5)
        {
            throw new IllegalArgumentException("Alpha (for trimming) must be in [0, 0.5]: " + alpha_);
        }

{'fraction_non_alphanumeric': 0.09619593538301198, 'fraction_numerical': 0.013444502344971338, 'mean_word_length': 2.03863204559848, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1792699', 'n_tokens_mistral': 2932, 'n_tokens_neox': 2793, 'n_words': 850}