<commit_before>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
 * 2.0 and the Server Side Public License, v 1; you may not use this file except
 * in compliance with, at your election, the Elastic License 2.0 or the Server
 * Side Public License, v 1.
 */
package org.elasticsearch.search.aggregations.support;

import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.OrdinalMap;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Rounding;
import org.elasticsearch.common.Rounding.Prepared;
import org.elasticsearch.common.lucene.ScorerAware;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues;
import org.elasticsearch.index.fielddata.DocValueBits;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.fielddata.SortingBinaryDocValues;
import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
import org.elasticsearch.index.mapper.RangeType;
import org.elasticsearch.script.AggregationScript;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.AggregationExecutionException;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
import org.elasticsearch.search.aggregations.support.values.ScriptBytesValues;
import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues;
import org.elasticsearch.search.aggregations.support.values.ScriptLongValues;

import java.io.IOException;
import java.util.function.Function;
import java.util.function.LongUnaryOperator;

/**
 * A unified interface to different ways of getting input data for
 * {@link Aggregator}s like DocValues from Lucene or script output. The
 * top level sub-classes define type-specific behavior, such as
 * {@link ValuesSource.Numeric#isFloatingPoint()}. Second level subclasses are
 * then specialized based on where they read values from, e.g. script or field
 * cases. There are also adapter classes like {@link CellIdSource} which do
 * run-time conversion from one type to another, often dependent on a user
 * specified parameter (precision in that case).
 */
public abstract class ValuesSource {

    /**
     * Get a byte array like view into the values. This is the "native" way
     * to access {@link Bytes}-style values.
     */
    public abstract SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException;

    /**
     * Get a "has any values" view into the values. It'll try to pick the
     * "most native" way to check if there are any values, but it builds its
     * own view into the values so if you need any of the actual values its
     * best to use something like {@link #bytesValues} or
     * {@link Numeric#doubleValues} but if you <strong>just</strong>
     * need to know if there are any values then use this.
     */
    public abstract DocValueBits docsWithValue(LeafReaderContext context) throws IOException;

    /** Whether this values source needs scores. */
    public boolean needsScores() {
        return false;
    }

    /**
     * Build a function to prepare {@link Rounding}s.
     * <p>
     * This returns a {@linkplain Function} because auto date histogram will
     * need to call it many times over the course of running the aggregation.
     * Other aggregations should feel free to call it once.
     */
    protected abstract Function<Rounding, Rounding.Prepared> roundingPreparer() throws IOException;

    /**
     * Check if this values source supports using global and segment ordinals.
     * <p>
     * If this returns {@code true} then it is safe to cast it to {@link ValuesSource.Bytes.WithOrdinals}.
     */
    public boolean hasOrdinals() {
        return false;
    }

    /**
     * {@linkplain ValuesSource} for fields who's values are best thought of
     * as byte arrays without any other meaning like {@code keyword} or
     * {@code ip}. Aggregations that operate on these values presume only
     * that {@link DocValueFormat#format(BytesRef)} will correctly convert
     * the resulting {@link BytesRef} into something human readable.
     */
    public abstract static class Bytes extends ValuesSource {

        @Override
        public DocValueBits docsWithValue(LeafReaderContext context) throws IOException {
            final SortedBinaryDocValues bytes = bytesValues(context);
            return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes);
        }

        @Override
        public final Function<Rounding, Rounding.Prepared> roundingPreparer() throws IOException {
            throw new AggregationExecutionException("can't round a [BYTES]");
        }

        /**
         * Specialization of {@linkplain Bytes} who's underlying storage
         * de-duplicates its bytes by storing them in a per-leaf sorted
         * lookup table. Aggregations that are aware of these lookup tables
         * can operate directly on the value's position in the table, know as
         * the "ordinal". They can then later translate the ordinal into
         * the {@link BytesRef} value.
         */
        public abstract static class WithOrdinals extends Bytes {

            public static final WithOrdinals EMPTY = new WithOrdinals() {

                @Override
                public SortedSetDocValues ordinalsValues(LeafReaderContext context) {
                    return DocValues.emptySortedSet();
                }

                @Override
                public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) {
                    return DocValues.emptySortedSet();
                }

                @Override
                public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException {
                    return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary();
                }

                @Override
                public LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) throws IOException {
                    return LongUnaryOperator.identity();
                }

            };

            @Override
            public DocValueBits docsWithValue(LeafReaderContext context) throws IOException {
                final SortedSetDocValues ordinals = ordinalsValues(context);
                return org.elasticsearch.index.fielddata.FieldData.docsWithValue(ordinals);
            }

            /**
             * Get a view into the leaf's ordinals and their {@link BytesRef} values.
             * <p>
             * Use {@link SortedSetDocValues#advanceExact}, {@link SortedSetDocValues#getValueCount},
             * and {@link SortedSetDocValues#nextOrd} to fetch the ordinals. Use
             * {@link SortedSetDocValues#lookupOrd} to convert form the ordinal
             * number into the {@link BytesRef} value. Make sure to
             * {@link BytesRef#deepCopyOf(BytesRef) copy} the result if you need
             * to keep it.
             * <p>
             * Each leaf may have a different ordinal for the same byte array.
             * Imagine, for example, an index where one leaf has the values
             * {@code "a", "b", "d"} and another leaf has the values
             * {@code "b", "c", "d"}. {@code "a"} has the ordinal {@code 0} in
             * the first leaf and doesn't exist in the second leaf.
             * {@code "b"} has the ordinal {@code 1} in the first leaf
             * and {@code 0} in the second leaf. {@code "c"} doesn't exist in
             * the first leaf and has the ordinal {@code 1} in the second leaf.
             * And {@code "d"} gets the ordinal {@code 2} in both leaves.
             * <p>
             * This can be a huge pain if you have to compare the ordinals
             * of values from different segments.
             */<commit_msg>Update javadoc<commit_after>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
 * 2.0 and the Server Side Public License, v 1; you may not use this file except
 * in compliance with, at your election, the Elastic License 2.0 or the Server
 * Side Public License, v 1.
 */
package org.elasticsearch.search.aggregations.support;

import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.OrdinalMap;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Rounding;
import org.elasticsearch.common.Rounding.Prepared;
import org.elasticsearch.common.lucene.ScorerAware;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues;
import org.elasticsearch.index.fielddata.DocValueBits;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.fielddata.SortingBinaryDocValues;
import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
import org.elasticsearch.index.mapper.RangeType;
import org.elasticsearch.script.AggregationScript;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.AggregationExecutionException;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
import org.elasticsearch.search.aggregations.support.values.ScriptBytesValues;
import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues;
import org.elasticsearch.search.aggregations.support.values.ScriptLongValues;

import java.io.IOException;
import java.util.function.Function;
import java.util.function.LongUnaryOperator;

/**
 * A unified interface to different ways of getting input data for
 * {@link Aggregator}s like DocValues from Lucene or script output. The
 * top level sub-classes define type-specific behavior, such as
 * {@link ValuesSource.Numeric#isFloatingPoint()}. Second level subclasses are
 * then specialized based on where they read values from, e.g. script or field
 * cases. There are also adapter classes like {@link CellIdSource} which do
 * run-time conversion from one type to another, often dependent on a user
 * specified parameter (precision in that case).
 */
public abstract class ValuesSource {

    /**
     * Get a byte array like view into the values. This is the "native" way
     * to access {@link Bytes}-style values.
     */
    public abstract SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException;

    /**
     * Get a "has any values" view into the values. It'll try to pick the
     * "most native" way to check if there are any values, but it builds its
     * own view into the values so if you need any of the actual values its
     * best to use something like {@link #bytesValues} or
     * {@link Numeric#doubleValues} but if you <strong>just</strong>
     * need to know if there are any values then use this.
     */
    public abstract DocValueBits docsWithValue(LeafReaderContext context) throws IOException;

    /** Whether this values source needs scores. */
    public boolean needsScores() {
        return false;
    }

    /**
     * Build a function to prepare {@link Rounding}s.
     * <p>
     * This returns a {@linkplain Function} because auto date histogram will
     * need to call it many times over the course of running the aggregation.
     * Other aggregations should feel free to call it once.
     */
    protected abstract Function<Rounding, Rounding.Prepared> roundingPreparer() throws IOException;

    /**
     * Check if this values source supports using global and segment ordinals.
     * <p>
     * If this returns {@code true} then it is safe to cast it to {@link ValuesSource.Bytes.WithOrdinals}.
     */
    public boolean hasOrdinals() {
        return false;
    }

    /**
     * {@linkplain ValuesSource} for fields who's values are best thought of
     * as byte arrays without any other meaning like {@code keyword} or
     * {@code ip}. Aggregations that operate on these values presume only
     * that {@link DocValueFormat#format(BytesRef)} will correctly convert
     * the resulting {@link BytesRef} into something human readable.
     */
    public abstract static class Bytes extends ValuesSource {

        @Override
        public DocValueBits docsWithValue(LeafReaderContext context) throws IOException {
            final SortedBinaryDocValues bytes = bytesValues(context);
            return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes);
        }

        @Override
        public final Function<Rounding, Rounding.Prepared> roundingPreparer() throws IOException {
            throw new AggregationExecutionException("can't round a [BYTES]");
        }

        /**
         * Specialization of {@linkplain Bytes} who's underlying storage
         * de-duplicates its bytes by storing them in a per-leaf sorted
         * lookup table. Aggregations that are aware of these lookup tables
         * can operate directly on the value's position in the table, know as
         * the "ordinal". They can then later translate the ordinal into
         * the {@link BytesRef} value.
         */
        public abstract static class WithOrdinals extends Bytes {

            public static final WithOrdinals EMPTY = new WithOrdinals() {

                @Override
                public SortedSetDocValues ordinalsValues(LeafReaderContext context) {
                    return DocValues.emptySortedSet();
                }

                @Override
                public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) {
                    return DocValues.emptySortedSet();
                }

                @Override
                public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException {
                    return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary();
                }

                @Override
                public LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) throws IOException {
                    return LongUnaryOperator.identity();
                }

            };

            @Override
            public DocValueBits docsWithValue(LeafReaderContext context) throws IOException {
                final SortedSetDocValues ordinals = ordinalsValues(context);
                return org.elasticsearch.index.fielddata.FieldData.docsWithValue(ordinals);
            }

            /**
             * Get a view into the leaf's ordinals and their {@link BytesRef} values.
             * <p>
             * Use {@link SortedSetDocValues#advanceExact}, {@link SortedSetDocValues#getValueCount},
             * and {@link SortedSetDocValues#nextOrd} to fetch the ordinals. Use
             * {@link SortedSetDocValues#lookupOrd} to convert form the ordinal
             * number into the {@link BytesRef} value. Make sure to
             * {@link BytesRef#deepCopyOf(BytesRef) copy} the result if you need
             * to keep it.
             * <p>
             * Each leaf may have a different ordinal for the same byte array.
             * Imagine, for example, an index where one leaf has the values
             * {@code "a", "b", "d"} and another leaf has the values
             * {@code "b", "c", "d"}. {@code "a"} has the ordinal {@code 0} in
             * the first leaf and doesn't exist in the second leaf.
             * {@code "b"} has the ordinal {@code 1} in the first leaf
             * and {@code 0} in the second leaf. {@code "c"} doesn't exist in
             * the first leaf and has the ordinal {@code 1} in the second leaf.
             * And {@code "d"} gets the ordinal {@code 2} in both leaves.
             * <p>
             * If you have to compare the ordinals of values from different
             * segments then you'd need to somehow merge them. {@link #globalOrdinalsValues}
             * provides such a merging at the cost of longer startup times when
             * the index has been modified.
             */
{'fraction_non_alphanumeric': 0.07639926922438134, 'fraction_numerical': 0.0012179593644466588, 'mean_word_length': 3.054769921436588, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6038449', 'n_tokens_mistral': 4669, 'n_tokens_neox': 4540, 'n_words': 1571}