<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.processing;

import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Schema;
import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor;
import com.yahoo.vespa.model.container.search.QueryProfiles;

import java.util.Arrays;
import java.util.Collection;
import java.util.Set;

/**
 * Executor of processors. This defines the right order of processor execution.
 *
 * @author bratseth
 * @author bjorncs
 */
public class Processing {

    private Collection<ProcessorFactory> processors() {
        return Arrays.asList(
                SearchMustHaveDocument::new,
                UrlFieldValidator::new,
                BuiltInFieldSets::new,
                ReservedDocumentNames::new,
                IndexFieldNames::new,
                IntegerIndex2Attribute::new,
                MakeAliases::new,
                UriHack::new,
                LiteralBoost::new,
                TagType::new,
                ValidateFieldTypesDocumentsOnly::new,
                IndexingInputs::new,
                OptimizeIlscript::new,
                ValidateFieldWithIndexSettingsCreatesIndex::new,
                AttributesImplicitWord::new,
                MutableAttributes::new,
                CreatePositionZCurve::new,
                DictionaryProcessor::new,
                WordMatch::new,
                ImportedFieldsResolver::new,
                ImplicitSummaries::new,
                ImplicitSummaryFields::new,
                AdjustPositionSummaryFields::new,
                SummaryConsistency::new,
                SummaryNamesFieldCollisions::new,
                SummaryFieldsMustHaveValidSource::new,
                MatchedElementsOnlyResolver::new,
                AddAttributeTransformToSummaryOfImportedFields::new,
                MakeDefaultSummaryTheSuperSet::new,
                Bolding::new,
                AttributeProperties::new,
                SetRankTypeEmptyOnFilters::new,
                SummaryDynamicStructsArrays::new,
                StringSettingsOnNonStringFields::new,
                IndexingOutputs::new,
                ExactMatch::new,
                NGramMatch::new,
                TextMatch::new,
                MultifieldIndexHarmonizer::new,
                FilterFieldNames::new,
                MatchConsistency::new,
                ValidateFieldTypes::new,
                SummaryDiskAccessValidator::new,
                DisallowComplexMapAndWsetKeyTypes::new,
                SortingSettings::new,
                FieldSetSettings::new,
                AddExtraFieldsToDocument::new,
                PredicateProcessor::new,
                MatchPhaseSettingsValidator::new,
                DiversitySettingsValidator::new,
                TensorFieldProcessor::new,
                RankProfileTypeSettingsProcessor::new,
                ReferenceFieldsProcessor::new,
                FastAccessValidator::new,
                ReservedFunctionNames::new,
                OnnxModelConfigGenerator::new,
                OnnxModelTypeResolver::new,
                RankingExpressionTypeResolver::new,
                ValidateStructTypeInheritance::new,
                BoolAttributeValidator::new,
                PagedAttributeValidator::new,
                // These should be last:
                IndexingValidation::new,
                IndexingValues::new);
    }

    /** Processors of rank profiles only (those who tolerate and do something useful when the search field is null) */
    private Collection<ProcessorFactory> rankProfileProcessors() {
        return Arrays.asList(
                RankProfileTypeSettingsProcessor::new,
                ReservedFunctionNames::new,
                RankingExpressionTypeResolver::new);
    }

    /**
     * Runs all search processors on the given {@link Schema} object. These will modify the search object, <b>possibly
     * exchanging it with another</b>, as well as its document types.
     *
     * @param schema the search to process
     * @param deployLogger the log to log messages and warnings for application deployment to
     * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
     * @param queryProfiles the query profiles contained in the application this search is part of
     * @param processorsToSkip a set of processor classes we should not invoke in this. Useful for testing.
     */
    public void process(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
                        QueryProfiles queryProfiles, boolean validate, boolean documentsOnly,
                        Set<Class<? extends Processor>> processorsToSkip) {
        Collection<ProcessorFactory> factories = processors();
        factories.stream()
                .map(factory -> factory.create(schema, deployLogger, rankProfileRegistry, queryProfiles))
                .filter(processor -> ! processorsToSkip.contains(processor.getClass()))
                .forEach(processor -> processor.process(validate, documentsOnly));
    }

    /**
     * Runs rank profiles processors only.
     *
     * @param deployLogger the log to log messages and warnings for application deployment to
     * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
     * @param queryProfiles the query profiles contained in the application this search is part of
     */
    public void processRankProfiles(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
                                    QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) {
        Collection<ProcessorFactory> factories = rankProfileProcessors();
        factories.stream()
                 .map(factory -> factory.create(null, deployLogger, rankProfileRegistry, queryProfiles))
                 .forEach(processor -> processor.process(validate, documentsOnly));
    }

    @FunctionalInterface
    public interface ProcessorFactory {
        Processor create(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
                         QueryProfiles queryProfiles);
    }

}
<commit_msg>put tpye validations together<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.processing;

import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Schema;
import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor;
import com.yahoo.vespa.model.container.search.QueryProfiles;

import java.util.Arrays;
import java.util.Collection;
import java.util.Set;

/**
 * Executor of processors. This defines the right order of processor execution.
 *
 * @author bratseth
 * @author bjorncs
 */
public class Processing {

    private Collection<ProcessorFactory> processors() {
        return Arrays.asList(
                SearchMustHaveDocument::new,
                UrlFieldValidator::new,
                BuiltInFieldSets::new,
                ReservedDocumentNames::new,
                IndexFieldNames::new,
                IntegerIndex2Attribute::new,
                MakeAliases::new,
                UriHack::new,
                LiteralBoost::new,
                TagType::new,
                ValidateFieldTypesDocumentsOnly::new,
                IndexingInputs::new,
                OptimizeIlscript::new,
                ValidateFieldWithIndexSettingsCreatesIndex::new,
                AttributesImplicitWord::new,
                MutableAttributes::new,
                CreatePositionZCurve::new,
                DictionaryProcessor::new,
                WordMatch::new,
                ImportedFieldsResolver::new,
                ImplicitSummaries::new,
                ImplicitSummaryFields::new,
                AdjustPositionSummaryFields::new,
                SummaryConsistency::new,
                SummaryNamesFieldCollisions::new,
                SummaryFieldsMustHaveValidSource::new,
                MatchedElementsOnlyResolver::new,
                AddAttributeTransformToSummaryOfImportedFields::new,
                MakeDefaultSummaryTheSuperSet::new,
                Bolding::new,
                AttributeProperties::new,
                SetRankTypeEmptyOnFilters::new,
                SummaryDynamicStructsArrays::new,
                StringSettingsOnNonStringFields::new,
                IndexingOutputs::new,
                ExactMatch::new,
                NGramMatch::new,
                TextMatch::new,
                MultifieldIndexHarmonizer::new,
                FilterFieldNames::new,
                MatchConsistency::new,
                ValidateStructTypeInheritance::new,
                ValidateFieldTypes::new,
                SummaryDiskAccessValidator::new,
                DisallowComplexMapAndWsetKeyTypes::new,
                SortingSettings::new,
                FieldSetSettings::new,
                AddExtraFieldsToDocument::new,
                PredicateProcessor::new,
                MatchPhaseSettingsValidator::new,
                DiversitySettingsValidator::new,
                TensorFieldProcessor::new,
                RankProfileTypeSettingsProcessor::new,
                ReferenceFieldsProcessor::new,
                FastAccessValidator::new,
                ReservedFunctionNames::new,
                OnnxModelConfigGenerator::new,
                OnnxModelTypeResolver::new,
                RankingExpressionTypeResolver::new,
                BoolAttributeValidator::new,
                PagedAttributeValidator::new,
                // These should be last:
                IndexingValidation::new,
                IndexingValues::new);
    }

    /** Processors of rank profiles only (those who tolerate and do something useful when the search field is null) */
    private Collection<ProcessorFactory> rankProfileProcessors() {
        return Arrays.asList(
                RankProfileTypeSettingsProcessor::new,
                ReservedFunctionNames::new,
                RankingExpressionTypeResolver::new);
    }

    /**
     * Runs all search processors on the given {@link Schema} object. These will modify the search object, <b>possibly
     * exchanging it with another</b>, as well as its document types.
     *
     * @param schema the search to process
     * @param deployLogger the log to log messages and warnings for application deployment to
     * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
     * @param queryProfiles the query profiles contained in the application this search is part of
     * @param processorsToSkip a set of processor classes we should not invoke in this. Useful for testing.
     */
    public void process(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
                        QueryProfiles queryProfiles, boolean validate, boolean documentsOnly,
                        Set<Class<? extends Processor>> processorsToSkip) {
        Collection<ProcessorFactory> factories = processors();
        factories.stream()
                .map(factory -> factory.create(schema, deployLogger, rankProfileRegistry, queryProfiles))
                .filter(processor -> ! processorsToSkip.contains(processor.getClass()))
                .forEach(processor -> processor.process(validate, documentsOnly));
    }

    /**
     * Runs rank profiles processors only.
     *
     * @param deployLogger the log to log messages and warnings for application deployment to
     * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
     * @param queryProfiles the query profiles contained in the application this search is part of
     */
    public void processRankProfiles(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
                                    QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) {
        Collection<ProcessorFactory> factories = rankProfileProcessors();
        factories.stream()
                 .map(factory -> factory.create(null, deployLogger, rankProfileRegistry, queryProfiles))
                 .forEach(processor -> processor.process(validate, documentsOnly));
    }

    @FunctionalInterface
    public interface ProcessorFactory {
        Processor create(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
                         QueryProfiles queryProfiles);
    }

}

{'fraction_non_alphanumeric': 0.0706139665677238, 'fraction_numerical': 0.00046867676925480396, 'mean_word_length': 2.3134057971014492, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6964166', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2808, 'n_words': 681}