<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;

import com.yahoo.document.*;
import com.yahoo.searchdefinition.Index;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.document.*;
import com.yahoo.searchdefinition.processing.ExactMatch;
import com.yahoo.searchdefinition.processing.NGramMatch;
import com.yahoo.vespa.documentmodel.SummaryField;
import com.yahoo.search.config.IndexInfoConfig;

import java.util.*;

/**
 * Per-index commands which should be applied to queries prior to searching
 *
 * @author bratseth
 */
public class IndexInfo extends Derived implements IndexInfoConfig.Producer {

    private static final String CMD_ATTRIBUTE = "attribute";
    private static final String CMD_DEFAULT_POSITION = "default-position";
    private static final String CMD_DYNTEASER = "dynteaser";
    private static final String CMD_FULLURL = "fullurl";
    private static final String CMD_HIGHLIGHT = "highlight";
    private static final String CMD_INDEX = "index";
    private static final String CMD_LOWERCASE = "lowercase";
    private static final String CMD_MATCH_GROUP = "match-group ";
    private static final String CMD_NORMALIZE = "normalize";
    private static final String CMD_STEM = "stem";
    private static final String CMD_URLHOST = "urlhost";
    private static final String CMD_WORD = "word";
    private static final String CMD_PLAIN_TOKENS = "plain-tokens";
    private static final String CMD_MULTIVALUE = "multivalue";
    private static final String CMD_FAST_SEARCH = "fast-search";
    private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds";
    private static final String CMD_NUMERICAL = "numerical";
    private Set<IndexCommand> commands = new java.util.LinkedHashSet<>();
    private Map<String, String> aliases = new java.util.LinkedHashMap<>();
    private Map<String, FieldSet> fieldSets;
    private Search search;

    public IndexInfo(Search search) {
        this.fieldSets = search.fieldSets().userFieldSets();
        addIndexCommand("sddocname", CMD_INDEX);
        addIndexCommand("sddocname", CMD_WORD);
        derive(search);
    }

    @Override
    protected void derive(Search search) {
        super.derive(search); // Derive per field
        this.search = search;
        // Populate fieldsets with actual field objects, bit late to do that here but
        for (FieldSet fs : fieldSets.values()) {
            for (String fieldName : fs.getFieldNames()) {
                fs.fields().add(search.getField(fieldName));
            }
        }
        // Must follow, because index settings overrides field settings
        for (Index index : search.getExplicitIndices()) {
            derive(index, search);
        }

        // Commands for summary fields
        // TODO: Move to fieldinfo and implement differently. This is not right
        for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) {
            if (summaryField.getTransform().isTeaser()) {
                addIndexCommand(summaryField.getName(), CMD_DYNTEASER);
            }
            if (summaryField.getTransform().isBolded()) {
                addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT);
            }
        }
    }

    @Override
    protected void derive(Index index, Search search) {
        if (index.getMatchGroup().size() > 0) {
            addIndexCommand(index.getName(), CMD_MATCH_GROUP + toSpaceSeparated(index.getMatchGroup()));
        }
    }

    private String toSpaceSeparated(Collection c) {
        StringBuffer b = new StringBuffer();
        for (Iterator i = c.iterator(); i.hasNext();) {
            b.append(i.next());
            if (i.hasNext()) {
                b.append(" ");
            }
        }
        return b.toString();
    }

    @Override
    protected void derive(ImmutableSDField field, Search search) {
        if (field.getDataType().equals(DataType.PREDICATE)) {
            Index index = field.getIndex(field.getName());
            if (index != null) {
                BooleanIndexDefinition options = index.getBooleanIndexDefiniton();
                if (options.hasLowerBound() || options.hasUpperBound()) {
                    addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" +
                            (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." +
                            (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]");
                }
            }
        }

        // Field level aliases
        for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
            String alias = e.getKey();
            String name = e.getValue();
            addIndexAlias(alias, name);
        }
        if (field.usesStructOrMap()) {
            for (ImmutableSDField structField : field.getStructFields()) {
                derive(structField, search); // Recursion
            }
        }

        if (field.getDataType().equals(PositionDataType.INSTANCE) ||
            field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
        {
            addIndexCommand(field.getName(), CMD_DEFAULT_POSITION);
        }

        addIndexCommand(field, CMD_INDEX); // List the indices

        if (field.doesIndexing() || field.doesLowerCasing()) {
            addIndexCommand(field, CMD_LOWERCASE);
        }

        if (field.getDataType().isMultivalue()) {
            addIndexCommand(field, CMD_MULTIVALUE);
        }

        if (field.doesAttributing() && !field.doesIndexing()) {
            addIndexCommand(field.getName(), CMD_ATTRIBUTE);
            Attribute attribute = field.getAttributes().get(field.getName());<commit_msg>Generate index-info config for struct attributes.<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;

import com.yahoo.document.*;
import com.yahoo.searchdefinition.Index;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.document.*;
import com.yahoo.searchdefinition.processing.ExactMatch;
import com.yahoo.searchdefinition.processing.NGramMatch;
import com.yahoo.vespa.documentmodel.SummaryField;
import com.yahoo.search.config.IndexInfoConfig;

import java.util.*;

/**
 * Per-index commands which should be applied to queries prior to searching
 *
 * @author bratseth
 */
public class IndexInfo extends Derived implements IndexInfoConfig.Producer {

    private static final String CMD_ATTRIBUTE = "attribute";
    private static final String CMD_DEFAULT_POSITION = "default-position";
    private static final String CMD_DYNTEASER = "dynteaser";
    private static final String CMD_FULLURL = "fullurl";
    private static final String CMD_HIGHLIGHT = "highlight";
    private static final String CMD_INDEX = "index";
    private static final String CMD_LOWERCASE = "lowercase";
    private static final String CMD_MATCH_GROUP = "match-group ";
    private static final String CMD_NORMALIZE = "normalize";
    private static final String CMD_STEM = "stem";
    private static final String CMD_URLHOST = "urlhost";
    private static final String CMD_WORD = "word";
    private static final String CMD_PLAIN_TOKENS = "plain-tokens";
    private static final String CMD_MULTIVALUE = "multivalue";
    private static final String CMD_FAST_SEARCH = "fast-search";
    private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds";
    private static final String CMD_NUMERICAL = "numerical";
    private Set<IndexCommand> commands = new java.util.LinkedHashSet<>();
    private Map<String, String> aliases = new java.util.LinkedHashMap<>();
    private Map<String, FieldSet> fieldSets;
    private Search search;

    public IndexInfo(Search search) {
        this.fieldSets = search.fieldSets().userFieldSets();
        addIndexCommand("sddocname", CMD_INDEX);
        addIndexCommand("sddocname", CMD_WORD);
        derive(search);
    }

    @Override
    protected void derive(Search search) {
        super.derive(search); // Derive per field
        this.search = search;
        // Populate fieldsets with actual field objects, bit late to do that here but
        for (FieldSet fs : fieldSets.values()) {
            for (String fieldName : fs.getFieldNames()) {
                fs.fields().add(search.getField(fieldName));
            }
        }
        // Must follow, because index settings overrides field settings
        for (Index index : search.getExplicitIndices()) {
            derive(index, search);
        }

        // Commands for summary fields
        // TODO: Move to fieldinfo and implement differently. This is not right
        for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) {
            if (summaryField.getTransform().isTeaser()) {
                addIndexCommand(summaryField.getName(), CMD_DYNTEASER);
            }
            if (summaryField.getTransform().isBolded()) {
                addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT);
            }
        }
    }

    @Override
    protected void derive(Index index, Search search) {
        if (index.getMatchGroup().size() > 0) {
            addIndexCommand(index.getName(), CMD_MATCH_GROUP + toSpaceSeparated(index.getMatchGroup()));
        }
    }

    private String toSpaceSeparated(Collection c) {
        StringBuffer b = new StringBuffer();
        for (Iterator i = c.iterator(); i.hasNext();) {
            b.append(i.next());
            if (i.hasNext()) {
                b.append(" ");
            }
        }
        return b.toString();
    }

    @Override
    protected void derive(ImmutableSDField field, Search search) {
        derive(field, search, false);
    }

    protected void derive(ImmutableSDField field, Search search, boolean inPosition) {
        if (field.getDataType().equals(DataType.PREDICATE)) {
            Index index = field.getIndex(field.getName());
            if (index != null) {
                BooleanIndexDefinition options = index.getBooleanIndexDefiniton();
                if (options.hasLowerBound() || options.hasUpperBound()) {
                    addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" +
                            (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." +
                            (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]");
                }
            }
        }

        // Field level aliases
        for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
            String alias = e.getKey();
            String name = e.getValue();
            addIndexAlias(alias, name);
        }
        boolean isPosition = field.getDataType().equals(PositionDataType.INSTANCE) ||
                field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE));
        if (field.usesStructOrMap()) {
            for (ImmutableSDField structField : field.getStructFields()) {
                derive(structField, search, isPosition); // Recursion
            }
        }

        if (isPosition) {
            addIndexCommand(field.getName(), CMD_DEFAULT_POSITION);
        }

        addIndexCommand(field, CMD_INDEX); // List the indices

        if (field.doesIndexing() || field.doesLowerCasing()) {
            addIndexCommand(field, CMD_LOWERCASE);
        }

        if (field.getDataType().isMultivalue()) {
            addIndexCommand(field, CMD_MULTIVALUE);
        }

        Attribute attribute = field.getAttributes().get(field.getName());
        if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) {
            addIndexCommand(field.getName(), CMD_ATTRIBUTE);
{'fraction_non_alphanumeric': 0.10725366876310273, 'fraction_numerical': 0.0011740041928721173, 'mean_word_length': 2.684275563793636, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4232787', 'n_tokens_mistral': 3231, 'n_tokens_neox': 3117, 'n_words': 772}