<commit_before>package org.codelibs.elasticsearch.dynarank.ranker;

import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;

import org.codelibs.elasticsearch.dynarank.script.DiversitySortScriptEngineService;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchResponseSections;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.InternalAggregations;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.profile.SearchProfileShardResults;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.threadpool.ThreadPool;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

public class DynamicRanker extends AbstractLifecycleComponent {

    private static DynamicRanker instance = null;

    public static final String DEFAULT_SCRIPT_TYPE = "inline";

    public static final String DEFAULT_SCRIPT_LANG = "groovy";

    public static final Setting<String> SETTING_INDEX_DYNARANK_SCRIPT =
            Setting.simpleString("index.dynarank.script_sort.script", Property.IndexScope);

    public static final Setting<String> SETTING_INDEX_DYNARANK_LANG =
            Setting.simpleString("index.dynarank.script_sort.lang", Property.IndexScope);

    public static final Setting<String> SETTING_INDEX_DYNARANK_TYPE =
            new Setting<>("index.dynarank.script_sort.type", s -> DEFAULT_SCRIPT_TYPE, Function.identity(), Property.IndexScope);

    public static final Setting<Settings> SETTING_INDEX_DYNARANK_PARAMS =
            Setting.groupSetting("index.dynarank.script_sort.params.", Property.IndexScope);

    public static final Setting<Integer> SETTING_INDEX_DYNARANK_REORDER_SIZE =
            Setting.intSetting("index.dynarank.reorder_size", 100, Property.IndexScope);

    public static final Setting<TimeValue> SETTING_DYNARANK_CACHE_EXPIRE =
            Setting.timeSetting("dynarank.cache.expire", TimeValue.MINUS_ONE, Property.NodeScope);

    public static final Setting<TimeValue> SETTING_DYNARANK_CACHE_CLEAN_INTERVAL =
            Setting.timeSetting("dynarank.cache.clean_interval", TimeValue.timeValueSeconds(60), Property.NodeScope);

    private static final String DYNARANK_RERANK_ENABLE = "_rerank";

    private static final String DYNARANK_MIN_TOTAL_HITS = "_minTotalHits";

    private final ClusterService clusterService;

    private final ScriptService scriptService;

    private final Cache<String, ScriptInfo> scriptInfoCache;

    private final ThreadPool threadPool;

    private final TimeValue cleanInterval;<commit_msg>add Dynamic<commit_after>package org.codelibs.elasticsearch.dynarank.ranker;

import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;

import org.codelibs.elasticsearch.dynarank.script.DiversitySortScriptEngineService;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchResponseSections;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.InternalAggregations;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.profile.SearchProfileShardResults;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.threadpool.ThreadPool;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

public class DynamicRanker extends AbstractLifecycleComponent {

    private static DynamicRanker instance = null;

    public static final String DEFAULT_SCRIPT_TYPE = "inline";

    public static final String DEFAULT_SCRIPT_LANG = "groovy";

    public static final Setting<String> SETTING_INDEX_DYNARANK_SCRIPT =
            Setting.simpleString("index.dynarank.script_sort.script", Property.IndexScope, Property.Dynamic);

    public static final Setting<String> SETTING_INDEX_DYNARANK_LANG =
            Setting.simpleString("index.dynarank.script_sort.lang", Property.IndexScope, Property.Dynamic);

    public static final Setting<String> SETTING_INDEX_DYNARANK_TYPE = new Setting<>("index.dynarank.script_sort.type",
            s -> DEFAULT_SCRIPT_TYPE, Function.identity(), Property.IndexScope, Property.Dynamic);

    public static final Setting<Settings> SETTING_INDEX_DYNARANK_PARAMS =
            Setting.groupSetting("index.dynarank.script_sort.params.", Property.IndexScope, Property.Dynamic);

    public static final Setting<Integer> SETTING_INDEX_DYNARANK_REORDER_SIZE =
            Setting.intSetting("index.dynarank.reorder_size", 100, Property.IndexScope, Property.Dynamic);

    public static final Setting<TimeValue> SETTING_DYNARANK_CACHE_EXPIRE =
            Setting.timeSetting("dynarank.cache.expire", TimeValue.MINUS_ONE, Property.NodeScope);

    public static final Setting<TimeValue> SETTING_DYNARANK_CACHE_CLEAN_INTERVAL =
            Setting.timeSetting("dynarank.cache.clean_interval", TimeValue.timeValueSeconds(60), Property.NodeScope);

    private static final String DYNARANK_RERANK_ENABLE = "_rerank";

    private static final String DYNARANK_MIN_TOTAL_HITS = "_minTotalHits";

    private final ClusterService clusterService;

    private final ScriptService scriptService;

    private final Cache<String, ScriptInfo> scriptInfoCache;

    private final ThreadPool threadPool;

    private final TimeValue cleanInterval;
{'fraction_non_alphanumeric': 0.08388420332801459, 'fraction_numerical': 0.0011397310234784591, 'mean_word_length': 6.169117647058823, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4047177', 'n_tokens_mistral': 2355, 'n_tokens_neox': 2378, 'n_words': 425}