<commit_before>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License;
 * you may not use this file except in compliance with the Elastic License.
 */
package org.elasticsearch.xpack.core.ml.job.config;

import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;

/**
 * Autodetect analysis configuration options describes which fields are
 * analysed and the functions to use.
 * <p>
 * The configuration can contain multiple detectors, a new anomaly detector will
 * be created for each detector configuration. The fields
 * <code>bucketSpan, summaryCountFieldName and categorizationFieldName</code>
 * apply to all detectors.
 * <p>
 * If a value has not been set it will be <code>null</code>
 * Object wrappers are used around integral types &amp; booleans so they can take
 * <code>null</code> values.
 */
public class AnalysisConfig implements ToXContentObject, Writeable {
    /**
     * Serialisation names
     */
    public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config");
    private static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
    private static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name");
    static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters");
    private static final ParseField CATEGORIZATION_ANALYZER = CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER;
    private static final ParseField LATENCY = new ParseField("latency");
    private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
    private static final ParseField DETECTORS = new ParseField("detectors");
    private static final ParseField INFLUENCERS = new ParseField("influencers");
    private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");

    public static final String ML_CATEGORY_FIELD = "mlcategory";
    public static final Set<String> AUTO_CREATED_FIELDS = new HashSet<>(Collections.singletonList(ML_CATEGORY_FIELD));

    // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> LENIENT_PARSER = createParser(true);
    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> STRICT_PARSER = createParser(false);

    @SuppressWarnings("unchecked")
    private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createParser(boolean ignoreUnknownFields) {
        ConstructingObjectParser<AnalysisConfig.Builder, Void> parser = new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(),
            ignoreUnknownFields, a -> new AnalysisConfig.Builder((List<Detector>) a[0]));

        parser.declareObjectArray(ConstructingObjectParser.constructorArg(),
            (p, c) -> (ignoreUnknownFields ? Detector.LENIENT_PARSER : Detector.STRICT_PARSER).apply(p, c).build(), DETECTORS);
        parser.declareString((builder, val) ->
            builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN);
        parser.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME);
        parser.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS);
        // This one is nasty - the syntax for analyzers takes either names or objects at many levels, hence it's not
        // possible to simply declare whether the field is a string or object and a completely custom parser is required
        parser.declareField(Builder::setCategorizationAnalyzerConfig,
            (p, c) -> CategorizationAnalyzerConfig.buildFromXContentFragment(p, ignoreUnknownFields),
            CATEGORIZATION_ANALYZER, ObjectParser.ValueType.OBJECT_OR_STRING);
        parser.declareString((builder, val) ->
            builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
        parser.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
        parser.declareStringArray(Builder::setInfluencers, INFLUENCERS);
        parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);

        return parser;
    }

    /**
     * These values apply to all detectors
     */
    private final TimeValue bucketSpan;
    private final String categorizationFieldName;
    private final List<String> categorizationFilters;
    private final CategorizationAnalyzerConfig categorizationAnalyzerConfig;
    private final TimeValue latency;
    private final String summaryCountFieldName;
    private final List<Detector> detectors;
    private final List<String> influencers;
    private final Boolean multivariateByFields;

    private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
                           CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
                           List<Detector> detectors, List<String> influencers, Boolean multivariateByFields) {
        this.detectors = detectors;
        this.bucketSpan = bucketSpan;
        this.latency = latency;
        this.categorizationFieldName = categorizationFieldName;
        this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
        this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
        this.summaryCountFieldName = summaryCountFieldName;
        this.influencers = Collections.unmodifiableList(influencers);
        this.multivariateByFields = multivariateByFields;
    }

    public AnalysisConfig(StreamInput in) throws IOException {
        bucketSpan = in.readTimeValue();
        categorizationFieldName = in.readOptionalString();
        categorizationFilters = in.readBoolean() ? Collections.unmodifiableList(in.readList(StreamInput::readString)) : null;
        if (in.getVersion().onOrAfter(Version.V_6_2_0)) {
            categorizationAnalyzerConfig = in.readOptionalWriteable(CategorizationAnalyzerConfig::new);
        } else {
            categorizationAnalyzerConfig = null;
        }
        latency = in.readOptionalTimeValue();
        summaryCountFieldName = in.readOptionalString();
        detectors = Collections.unmodifiableList(in.readList(Detector::new));
        influencers = Collections.unmodifiableList(in.readList(StreamInput::readString));

        // BWC for result_finalization_window and overlapping_buckets
        // TODO Remove in 7.0.0
        if (in.getVersion().before(Version.CURRENT)) {//setting to current before backport
            in.readOptionalBoolean();
            in.readOptionalLong();
        }
        multivariateByFields = in.readOptionalBoolean();

        // BWC for removed multiple_bucket_spans
        // TODO Remove in 7.0.0
        if (in.getVersion().before(Version.V_6_5_0)) {
            if (in.readBoolean()) {
                final int arraySize = in.readVInt();
                for (int i = 0; i < arraySize; i++) {
                    in.readTimeValue();
                }
            }
        }

        // BWC for removed per-partition normalization
        // TODO Remove in 7.0.0
        if (in.getVersion().before(Version.V_6_5_0)) {
            in.readBoolean();
        }
    }

    @Override
    public void writeTo(StreamOutput out) throws IOException {
        out.writeTimeValue(bucketSpan);
        out.writeOptionalString(categorizationFieldName);
        if (categorizationFilters != null) {
            out.writeBoolean(true);
            out.writeStringList(categorizationFilters);
        } else {
            out.writeBoolean(false);
        }
        if (out.getVersion().onOrAfter(Version.V_6_2_0)) {
            out.writeOptionalWriteable(categorizationAnalyzerConfig);
        }
        out.writeOptionalTimeValue(latency);
        out.writeOptionalString(summaryCountFieldName);
        out.writeList(detectors);
        out.writeStringList(influencers);

        // BWC for result_finalization_window and overlapping_buckets
        // TODO Remove in 7.0.0
        if (out.getVersion().before(Version.CURRENT)) { //setting to current before backport
            out.writeOptionalBoolean(null);
            out.writeOptionalLong(null);
        }
        out.writeOptionalBoolean(multivariateByFields);

        // BWC for removed multiple_bucket_spans
        // TODO Remove in 7.0.0
        if (out.getVersion().before(Version.V_6_5_0)) {
            out.writeBoolean(false);
        }

        // BWC for removed per-partition normalization
        // TODO Remove in 7.0.0
        if (out.getVersion().before(Version.V_6_5_0)) {
            out.writeBoolean(false);
        }
    }

    /**
     * The analysis bucket span
     *
     * @return The bucketspan or <code>null</code> if not set
     */
    public TimeValue getBucketSpan() {
        return bucketSpan;<commit_msg>[ML] adjust BWC for PR (#35645)<commit_after>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License;
 * you may not use this file except in compliance with the Elastic License.
 */
package org.elasticsearch.xpack.core.ml.job.config;

import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;

/**
 * Autodetect analysis configuration options describes which fields are
 * analysed and the functions to use.
 * <p>
 * The configuration can contain multiple detectors, a new anomaly detector will
 * be created for each detector configuration. The fields
 * <code>bucketSpan, summaryCountFieldName and categorizationFieldName</code>
 * apply to all detectors.
 * <p>
 * If a value has not been set it will be <code>null</code>
 * Object wrappers are used around integral types &amp; booleans so they can take
 * <code>null</code> values.
 */
public class AnalysisConfig implements ToXContentObject, Writeable {
    /**
     * Serialisation names
     */
    public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config");
    private static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
    private static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name");
    static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters");
    private static final ParseField CATEGORIZATION_ANALYZER = CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER;
    private static final ParseField LATENCY = new ParseField("latency");
    private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
    private static final ParseField DETECTORS = new ParseField("detectors");
    private static final ParseField INFLUENCERS = new ParseField("influencers");
    private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");

    public static final String ML_CATEGORY_FIELD = "mlcategory";
    public static final Set<String> AUTO_CREATED_FIELDS = new HashSet<>(Collections.singletonList(ML_CATEGORY_FIELD));

    // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> LENIENT_PARSER = createParser(true);
    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> STRICT_PARSER = createParser(false);

    @SuppressWarnings("unchecked")
    private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createParser(boolean ignoreUnknownFields) {
        ConstructingObjectParser<AnalysisConfig.Builder, Void> parser = new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(),
            ignoreUnknownFields, a -> new AnalysisConfig.Builder((List<Detector>) a[0]));

        parser.declareObjectArray(ConstructingObjectParser.constructorArg(),
            (p, c) -> (ignoreUnknownFields ? Detector.LENIENT_PARSER : Detector.STRICT_PARSER).apply(p, c).build(), DETECTORS);
        parser.declareString((builder, val) ->
            builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN);
        parser.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME);
        parser.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS);
        // This one is nasty - the syntax for analyzers takes either names or objects at many levels, hence it's not
        // possible to simply declare whether the field is a string or object and a completely custom parser is required
        parser.declareField(Builder::setCategorizationAnalyzerConfig,
            (p, c) -> CategorizationAnalyzerConfig.buildFromXContentFragment(p, ignoreUnknownFields),
            CATEGORIZATION_ANALYZER, ObjectParser.ValueType.OBJECT_OR_STRING);
        parser.declareString((builder, val) ->
            builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
        parser.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
        parser.declareStringArray(Builder::setInfluencers, INFLUENCERS);
        parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);

        return parser;
    }

    /**
     * These values apply to all detectors
     */
    private final TimeValue bucketSpan;
    private final String categorizationFieldName;
    private final List<String> categorizationFilters;
    private final CategorizationAnalyzerConfig categorizationAnalyzerConfig;
    private final TimeValue latency;
    private final String summaryCountFieldName;
    private final List<Detector> detectors;
    private final List<String> influencers;
    private final Boolean multivariateByFields;

    private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
                           CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
                           List<Detector> detectors, List<String> influencers, Boolean multivariateByFields) {
        this.detectors = detectors;
        this.bucketSpan = bucketSpan;
        this.latency = latency;
        this.categorizationFieldName = categorizationFieldName;
        this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
        this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
        this.summaryCountFieldName = summaryCountFieldName;
        this.influencers = Collections.unmodifiableList(influencers);
        this.multivariateByFields = multivariateByFields;
    }

    public AnalysisConfig(StreamInput in) throws IOException {
        bucketSpan = in.readTimeValue();
        categorizationFieldName = in.readOptionalString();
        categorizationFilters = in.readBoolean() ? Collections.unmodifiableList(in.readList(StreamInput::readString)) : null;
        if (in.getVersion().onOrAfter(Version.V_6_2_0)) {
            categorizationAnalyzerConfig = in.readOptionalWriteable(CategorizationAnalyzerConfig::new);
        } else {
            categorizationAnalyzerConfig = null;
        }
        latency = in.readOptionalTimeValue();
        summaryCountFieldName = in.readOptionalString();
        detectors = Collections.unmodifiableList(in.readList(Detector::new));
        influencers = Collections.unmodifiableList(in.readList(StreamInput::readString));

        // BWC for result_finalization_window and overlapping_buckets
        // TODO Remove in 7.0.0
        if (in.getVersion().before(Version.V_6_6_0)) {
            in.readOptionalBoolean();
            in.readOptionalLong();
        }
        multivariateByFields = in.readOptionalBoolean();

        // BWC for removed multiple_bucket_spans
        // TODO Remove in 7.0.0
        if (in.getVersion().before(Version.V_6_5_0)) {
            if (in.readBoolean()) {
                final int arraySize = in.readVInt();
                for (int i = 0; i < arraySize; i++) {
                    in.readTimeValue();
                }
            }
        }

        // BWC for removed per-partition normalization
        // TODO Remove in 7.0.0
        if (in.getVersion().before(Version.V_6_5_0)) {
            in.readBoolean();
        }
    }

    @Override
    public void writeTo(StreamOutput out) throws IOException {
        out.writeTimeValue(bucketSpan);
        out.writeOptionalString(categorizationFieldName);
        if (categorizationFilters != null) {
            out.writeBoolean(true);
            out.writeStringList(categorizationFilters);
        } else {
            out.writeBoolean(false);
        }
        if (out.getVersion().onOrAfter(Version.V_6_2_0)) {
            out.writeOptionalWriteable(categorizationAnalyzerConfig);
        }
        out.writeOptionalTimeValue(latency);
        out.writeOptionalString(summaryCountFieldName);
        out.writeList(detectors);
        out.writeStringList(influencers);

        // BWC for result_finalization_window and overlapping_buckets
        // TODO Remove in 7.0.0
        if (out.getVersion().before(Version.V_6_6_0)) {
            out.writeOptionalBoolean(null);
            out.writeOptionalLong(null);
        }
        out.writeOptionalBoolean(multivariateByFields);

        // BWC for removed multiple_bucket_spans
        // TODO Remove in 7.0.0
        if (out.getVersion().before(Version.V_6_5_0)) {
            out.writeBoolean(false);
        }

        // BWC for removed per-partition normalization
        // TODO Remove in 7.0.0
        if (out.getVersion().before(Version.V_6_5_0)) {
            out.writeBoolean(false);
        }
    }

    /**
     * The analysis bucket span
     *
     * @return The bucketspan or <code>null</code> if not set
     */
    public TimeValue getBucketSpan() {
        return bucketSpan;
{'fraction_non_alphanumeric': 0.0855434089906368, 'fraction_numerical': 0.004264914946811118, 'mean_word_length': 3.658597853391185, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4760507', 'n_tokens_mistral': 5580, 'n_tokens_neox': 5506, 'n_words': 1240}