<commit_before>/*
 * Copyright (C) 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.granite.math;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Ordering;
import com.google.common.math.DoubleMath;

import java.io.Serializable;
import java.util.List;
import java.util.TreeSet;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;

public final class PercentileTools implements Serializable {

    public static double[] findQuantiles(final List<? extends Number> sortedValues, final int quantileCount) {
        checkNotNull(sortedValues, "sortedValues");

        if (sortedValues.isEmpty()) return new double[]{};

        checkArgument(quantileCount > 1 && sortedValues.size() >= quantileCount, "quantileCount must be greater than 1 and smaller than the number of elements");

        final double[] result = new double[quantileCount - 1];

        // 2 quantiles = .5
        // 3 quantiles = .33
        // 4 quantiles = .25
        double indexIncrement = 1.0 / (double) quantileCount;

        for (int currentQuantile = 1; currentQuantile < quantileCount; currentQuantile++) {

            final double indexCoefficient = indexIncrement * (double) currentQuantile;

            final double index = indexCoefficient * (sortedValues.size() + 1);

            final int indexInteger = (int) index;

            final double indexDecimal = index - indexInteger;

            final int zeroBasedIndex = indexInteger - 1;

            if (indexDecimal == 0.0) { // it's a whole number, so just return the value at the zero-based index
                result[currentQuantile - 1] = sortedValues.get(zeroBasedIndex).doubleValue();
            } else {
                result[currentQuantile - 1] = DoubleMath.mean(sortedValues.get(zeroBasedIndex).doubleValue(), sortedValues.get(zeroBasedIndex + 1).doubleValue());

            }

        }

        return result;
    }

    public static double findPercentile(final double value, final double[] quantiles) {
        checkNotNull(quantiles, "quantiles");
        checkArgument(quantiles.length > 0, "quantiles must contain at least one value");

        final double rank = 1.0 / (quantiles.length + 1.0);

        double percentile = rank;

        for (int index = 0; index < quantiles.length; index++) {

            if (value < quantiles[index]) {
                return percentile;<commit_msg>Simplify quantile math<commit_after>/*
 * Copyright (C) 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.granite.math;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Ordering;
import com.google.common.math.DoubleMath;

import java.io.Serializable;
import java.util.List;
import java.util.TreeSet;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;

public final class PercentileTools implements Serializable {

    public static double[] findQuantiles(final List<? extends Number> sortedValues, final int quantileCount) {
        checkNotNull(sortedValues, "sortedValues");

        if (sortedValues.isEmpty()) return new double[]{};

        checkArgument(quantileCount > 1 && sortedValues.size() >= quantileCount, "quantileCount must be greater than 1 and smaller than the number of elements");

        final double[] result = new double[quantileCount - 1];

        // 2 quantiles = .5
        // 3 quantiles = .33
        // 4 quantiles = .25
        // ...etc
        double phi = 1.0 / (double) quantileCount;

        for (int currentQuantile = 1; currentQuantile < quantileCount; currentQuantile++) {

            final double index = phi * (double) currentQuantile * (sortedValues.size() + 1);

            final int indexInteger = (int) index;

            if (index - indexInteger == 0.0) { // it's a whole number, so just return the value at the zero-based index
                result[currentQuantile - 1] = sortedValues.get(indexInteger - 1).doubleValue();
            } else {
                result[currentQuantile - 1] = DoubleMath.mean(sortedValues.get(indexInteger - 1).doubleValue(), sortedValues.get(indexInteger).doubleValue());

            }

        }

        return result;
    }

    public static double findPercentile(final double value, final double[] quantiles) {
        checkNotNull(quantiles, "quantiles");
        checkArgument(quantiles.length > 0, "quantiles must contain at least one value");

        final double rank = 1.0 / (quantiles.length + 1.0);

        double percentile = rank;

        for (int index = 0; index < quantiles.length; index++) {

            if (value < quantiles[index]) {
                return percentile;
{'fraction_non_alphanumeric': 0.09712910434932095, 'fraction_numerical': 0.012033694344163659, 'mean_word_length': 2.9986254295532646, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '828172', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1493, 'n_words': 522}