<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.search;

import com.yahoo.config.FileReference;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.producer.UserConfigRepo;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
import com.yahoo.search.config.IndexInfoConfig;
import com.yahoo.searchdefinition.RankingConstant;
import com.yahoo.vespa.config.search.AttributesConfig;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
import com.yahoo.vespa.model.utils.FileSender;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

/**
 * Superclass for search clusters.
 *
 * @author <NAME>
 */
public abstract class AbstractSearchCluster extends AbstractConfigProducer
    implements
        DocumentdbInfoConfig.Producer,
        IndexInfoConfig.Producer,
        IlscriptsConfig.Producer {

    private Double queryTimeout;
    protected String clusterName;
    protected int index;
    private Double visibilityDelay = 0.0;
    private List<String> documentNames = new ArrayList<>();

    protected List<SearchDefinitionSpec> localSDS = new LinkedList<>();

    public void prepareToDistributeFiles(List<SearchNode> backends) {
        for (SearchDefinitionSpec sds : localSDS) {
            for (RankingConstant constant : sds.getSearchDefinition().getSearch().getRankingConstants().values()) {
                FileReference reference = (constant.getPathType() == RankingConstant.PathType.FILE)
                        ? FileSender.sendFileToServices(constant.getFileName(), backends)
                        : FileSender.sendUriToServices(constant.getUri(), backends);;
                constant.setFileReference(reference.value());
            }
        }
    }

    public static final class IndexingMode {

        public static final IndexingMode REALTIME  = new IndexingMode("REALTIME");
        public static final IndexingMode STREAMING = new IndexingMode("STREAMING");

        private final String name;

        private IndexingMode(String name) {
            this.name = name;
        }
<commit_msg>Do not distribute null<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.search;

import com.yahoo.config.FileReference;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.producer.UserConfigRepo;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
import com.yahoo.search.config.IndexInfoConfig;
import com.yahoo.searchdefinition.RankingConstant;
import com.yahoo.vespa.config.search.AttributesConfig;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
import com.yahoo.vespa.model.utils.FileSender;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

/**
 * Superclass for search clusters.
 *
 * @author <NAME>
 */
public abstract class AbstractSearchCluster extends AbstractConfigProducer
    implements
        DocumentdbInfoConfig.Producer,
        IndexInfoConfig.Producer,
        IlscriptsConfig.Producer {

    private Double queryTimeout;
    protected String clusterName;
    protected int index;
    private Double visibilityDelay = 0.0;
    private List<String> documentNames = new ArrayList<>();

    protected List<SearchDefinitionSpec> localSDS = new LinkedList<>();

    public void prepareToDistributeFiles(List<SearchNode> backends) {
        for (SearchDefinitionSpec sds : localSDS) {
            for (RankingConstant constant : sds.getSearchDefinition().getSearch().getRankingConstants().values()) {
                FileReference reference = (constant.getPathType() == RankingConstant.PathType.FILE)
                        ? FileSender.sendFileToServices(constant.getFileName(), backends)
                        : FileSender.sendUriToServices(constant.getUri(), backends);
                if (reference != null) {
                    constant.setFileReference(reference.value());
                }
            }
        }
    }

    public static final class IndexingMode {

        public static final IndexingMode REALTIME  = new IndexingMode("REALTIME");
        public static final IndexingMode STREAMING = new IndexingMode("STREAMING");

        private final String name;

        private IndexingMode(String name) {
            this.name = name;
        }

{'fraction_non_alphanumeric': 0.08787155565198525, 'fraction_numerical': 0.003471468865263615, 'mean_word_length': 3.458413926499033, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5939709', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1180, 'n_words': 278}