<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.searchchain;

import com.google.inject.Inject;
import com.yahoo.component.AbstractComponent;
import com.yahoo.component.chain.Chain;
import com.yahoo.component.chain.ChainsConfigurer;
import com.yahoo.component.chain.model.ChainsModel;
import com.yahoo.component.chain.model.ChainsModelBuilder;
import com.yahoo.component.provider.ComponentRegistry;
import com.yahoo.concurrent.ThreadFactoryFactory;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.container.core.ChainsConfig;
import com.yahoo.container.handler.threadpool.ContainerThreadPool;
import com.yahoo.language.Linguistics;
import com.yahoo.language.simple.SimpleLinguistics;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.IndexModel;
import com.yahoo.language.process.SpecialTokenRegistry;
import com.yahoo.processing.rendering.Renderer;
import com.yahoo.search.Searcher;
import com.yahoo.search.config.IndexInfoConfig;
import com.yahoo.search.rendering.RendererRegistry;
import com.yahoo.vespa.configdefinition.SpecialtokensConfig;

import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * Provides creation of fully configured query Execution instances.
 * Have an instance of this injected if you need to execute queries which are not initiated from
 * an external request.
 *
 * @author bratseth
 */
public class ExecutionFactory extends AbstractComponent {

    private final SearchChainRegistry searchChainRegistry;
    private final IndexFacts indexFacts;
    private final SpecialTokenRegistry specialTokens;
    private final Linguistics linguistics;
    private final RendererRegistry rendererRegistry;
    private final Executor executor;

    private static ThreadPoolExecutor createRenderingExecutor() {
        int threadCount = Runtime.getRuntime().availableProcessors();
        ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, 1L, TimeUnit.SECONDS,
                new LinkedBlockingQueue<>(),
                ThreadFactoryFactory.getThreadFactory("common-rendering"));
        executor.prestartAllCoreThreads();
        return executor;
    }

    @Inject
    public ExecutionFactory(ChainsConfig chainsConfig,
                            IndexInfoConfig indexInfo,
                            QrSearchersConfig clusters,
                            ComponentRegistry<Searcher> searchers,
                            SpecialtokensConfig specialTokens,
                            Linguistics linguistics,
                            ComponentRegistry<Renderer> renderers,
                            Executor executor) {
        this.searchChainRegistry = createSearchChainRegistry(searchers, chainsConfig);
        this.indexFacts = new IndexFacts(new IndexModel(indexInfo, clusters)).freeze();
        this.specialTokens = new SpecialTokenRegistry(specialTokens);
        this.linguistics = linguistics;
        this.rendererRegistry = new RendererRegistry(renderers.allComponents(), createRenderingExecutor());
        this.executor = executor != null ? executor : Executors.newSingleThreadExecutor();
    }

    /** @deprecated pass the container threadpool */
    @Deprecated // TODO: Remove on Vespa 8
    public ExecutionFactory(ChainsConfig chainsConfig,
                            IndexInfoConfig indexInfo,
                            QrSearchersConfig clusters,
                            ComponentRegistry<Searcher> searchers,
                            SpecialtokensConfig specialTokens,
                            Linguistics linguistics,
                            ComponentRegistry<Renderer> renderers) {
        this(chainsConfig, indexInfo, clusters, searchers, specialTokens, linguistics, renderers, (Executor)null);
    }

    private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, ChainsConfig chainsConfig) {
        SearchChainRegistry searchChainRegistry = new SearchChainRegistry(searchers);
        ChainsModel chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig);
        ChainsConfigurer.prepareChainRegistry(searchChainRegistry, chainsModel, searchers);
        searchChainRegistry.freeze();
        return searchChainRegistry;
    }

    /**
     * Creates a new execution starting at a search chain.
     * An execution instance should be used once to execute a (tree of) search chains.
     */
    public Execution newExecution(Chain<? extends Searcher> searchChain) {
        return new Execution(searchChain,
                             new Execution.Context(searchChainRegistry, indexFacts, specialTokens, rendererRegistry, linguistics, executor));
    }

    /**
     * Creates a new execution starting at a search chain.
     * An execution instance should be used once to execute a (tree of) search chains.
     */
    public Execution newExecution(String searchChainId) {
        return new Execution(searchChainRegistry().getChain(searchChainId),
                             new Execution.Context(searchChainRegistry, indexFacts, specialTokens, rendererRegistry, linguistics, executor));
    }

    /** Returns the search chain registry used by this */
    public SearchChainRegistry searchChainRegistry() { return searchChainRegistry; }

    /** Returns the renderers known to this */
    public RendererRegistry rendererRegistry() { return rendererRegistry; }

    @Override
    public void deconstruct() {
        rendererRegistry.deconstruct();
    }

    public static ExecutionFactory empty() {
        return new ExecutionFactory(new ChainsConfig.Builder().build(),
                                    new IndexInfoConfig.Builder().build(),
                                    new QrSearchersConfig.Builder().build(),
                                    new ComponentRegistry<>(),
                                    new SpecialtokensConfig.Builder().build(),
                                    new SimpleLinguistics(),
                                    new ComponentRegistry<>(),
                                    (Executor)null);
    }

}<commit_msg>Remember to shutdown common rendering threadpool.<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.searchchain;

import com.google.inject.Inject;
import com.yahoo.component.AbstractComponent;
import com.yahoo.component.chain.Chain;
import com.yahoo.component.chain.ChainsConfigurer;
import com.yahoo.component.chain.model.ChainsModel;
import com.yahoo.component.chain.model.ChainsModelBuilder;
import com.yahoo.component.provider.ComponentRegistry;
import com.yahoo.concurrent.ThreadFactoryFactory;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.container.core.ChainsConfig;
import com.yahoo.language.Linguistics;
import com.yahoo.language.simple.SimpleLinguistics;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.IndexModel;
import com.yahoo.language.process.SpecialTokenRegistry;
import com.yahoo.processing.rendering.Renderer;
import com.yahoo.search.Searcher;
import com.yahoo.search.config.IndexInfoConfig;
import com.yahoo.search.rendering.RendererRegistry;
import com.yahoo.vespa.configdefinition.SpecialtokensConfig;

import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * Provides creation of fully configured query Execution instances.
 * Have an instance of this injected if you need to execute queries which are not initiated from
 * an external request.
 *
 * @author bratseth
 */
public class ExecutionFactory extends AbstractComponent {

    private final SearchChainRegistry searchChainRegistry;
    private final IndexFacts indexFacts;
    private final SpecialTokenRegistry specialTokens;
    private final Linguistics linguistics;
    private final ThreadPoolExecutor renderingExecutor;
    private final RendererRegistry rendererRegistry;
    private final Executor executor;

    private static ThreadPoolExecutor createRenderingExecutor() {
        int threadCount = Runtime.getRuntime().availableProcessors();
        ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, 1L, TimeUnit.SECONDS,
                new LinkedBlockingQueue<>(),
                ThreadFactoryFactory.getThreadFactory("common-rendering"));
        executor.prestartAllCoreThreads();
        return executor;
    }

    @Inject
    public ExecutionFactory(ChainsConfig chainsConfig,
                            IndexInfoConfig indexInfo,
                            QrSearchersConfig clusters,
                            ComponentRegistry<Searcher> searchers,
                            SpecialtokensConfig specialTokens,
                            Linguistics linguistics,
                            ComponentRegistry<Renderer> renderers,
                            Executor executor) {
        this.searchChainRegistry = createSearchChainRegistry(searchers, chainsConfig);
        this.indexFacts = new IndexFacts(new IndexModel(indexInfo, clusters)).freeze();
        this.specialTokens = new SpecialTokenRegistry(specialTokens);
        this.linguistics = linguistics;
        this.renderingExecutor = createRenderingExecutor();
        this.rendererRegistry = new RendererRegistry(renderers.allComponents(), renderingExecutor);
        this.executor = executor != null ? executor : Executors.newSingleThreadExecutor();
    }

    /** @deprecated pass the container threadpool */
    @Deprecated // TODO: Remove on Vespa 8
    public ExecutionFactory(ChainsConfig chainsConfig,
                            IndexInfoConfig indexInfo,
                            QrSearchersConfig clusters,
                            ComponentRegistry<Searcher> searchers,
                            SpecialtokensConfig specialTokens,
                            Linguistics linguistics,
                            ComponentRegistry<Renderer> renderers) {
        this(chainsConfig, indexInfo, clusters, searchers, specialTokens, linguistics, renderers, null);
    }

    private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, ChainsConfig chainsConfig) {
        SearchChainRegistry searchChainRegistry = new SearchChainRegistry(searchers);
        ChainsModel chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig);
        ChainsConfigurer.prepareChainRegistry(searchChainRegistry, chainsModel, searchers);
        searchChainRegistry.freeze();
        return searchChainRegistry;
    }

    /**
     * Creates a new execution starting at a search chain.
     * An execution instance should be used once to execute a (tree of) search chains.
     */
    public Execution newExecution(Chain<? extends Searcher> searchChain) {
        return new Execution(searchChain,
                             new Execution.Context(searchChainRegistry, indexFacts, specialTokens, rendererRegistry, linguistics, executor));
    }

    /**
     * Creates a new execution starting at a search chain.
     * An execution instance should be used once to execute a (tree of) search chains.
     */
    public Execution newExecution(String searchChainId) {
        return new Execution(searchChainRegistry().getChain(searchChainId),
                             new Execution.Context(searchChainRegistry, indexFacts, specialTokens, rendererRegistry, linguistics, executor));
    }

    /** Returns the search chain registry used by this */
    public SearchChainRegistry searchChainRegistry() { return searchChainRegistry; }

    /** Returns the renderers known to this */
    public RendererRegistry rendererRegistry() { return rendererRegistry; }

    @Override
    public void deconstruct() {
        rendererRegistry.deconstruct();
        renderingExecutor.shutdown();
        try {
            if ( ! renderingExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
                renderingExecutor.shutdownNow();
            }
        } catch (InterruptedException e) {
            renderingExecutor.shutdownNow();
        }
    }

    public static ExecutionFactory empty() {
        return new ExecutionFactory(new ChainsConfig.Builder().build(),
                                    new IndexInfoConfig.Builder().build(),
                                    new QrSearchersConfig.Builder().build(),
                                    new ComponentRegistry<>(),
                                    new SpecialtokensConfig.Builder().build(),
                                    new SimpleLinguistics(),
                                    new ComponentRegistry<>(),
                                    null);
    }

}
{'fraction_non_alphanumeric': 0.0723105417502701, 'fraction_numerical': 0.0006945516283377065, 'mean_word_length': 2.839703703703704, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6081449', 'n_tokens_mistral': 3056, 'n_tokens_neox': 3013, 'n_words': 759}