<commit_before>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.xml;

import com.google.common.collect.ImmutableList;
import com.yahoo.component.Version;
import com.yahoo.config.application.Xml;
import com.yahoo.config.model.ConfigModelContext;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.application.provider.IncludeDirs;
import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
import com.yahoo.config.model.builder.xml.ConfigModelId;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.NodeType;
import com.yahoo.container.jdisc.config.MetricDefaultsConfig;
import com.yahoo.search.rendering.RendererRegistry;
import com.yahoo.text.XML;
import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.model.AbstractService;
import com.yahoo.vespa.model.HostResource;
import com.yahoo.vespa.model.builder.xml.dom.DomClientProviderBuilder;
import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder;
import com.yahoo.vespa.model.builder.xml.dom.DomFilterBuilder;
import com.yahoo.vespa.model.builder.xml.dom.DomHandlerBuilder;
import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
import com.yahoo.vespa.model.builder.xml.dom.ServletBuilder;
import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder;
import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessingBuilder;
import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearchChainsBuilder;
import com.yahoo.vespa.model.clients.ContainerDocumentApi;
import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.ContainerModel;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
import com.yahoo.vespa.model.container.docproc.DocprocChains;
import com.yahoo.vespa.model.container.http.Http;
import com.yahoo.vespa.model.container.http.xml.HttpBuilder;
import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder;
import com.yahoo.vespa.model.container.processing.ProcessingChains;
import com.yahoo.vespa.model.container.search.ContainerSearch;
import com.yahoo.vespa.model.container.search.PageTemplates;
import com.yahoo.vespa.model.container.search.QueryProfiles;
import com.yahoo.vespa.model.container.search.SemanticRules;
import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
import com.yahoo.vespa.model.container.xml.document.DocumentFactoryBuilder;

import com.yahoo.vespa.model.content.StorageGroup;
import org.w3c.dom.Element;
import org.w3c.dom.Node;

import java.util.*;
import java.util.function.Consumer;
import java.util.stream.Collectors;

/**
 * @author tonytv
 */
public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {

    /**
     * Default path to vip status file for container in Hosted Vespa.
     */
    static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().vespaHome() + "var/mediasearch/oor/status.html";
    /**
     * Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE
     */
    static final String HOSTED_VESPA_STATUS_FILE_YINST_SETTING = "cloudconfig_server__tenant_vip_status_file";

    public enum Networking { disable, enable }

    private ApplicationPackage app;
    private final boolean standaloneBuilder;
    private final Networking networking;
    protected DeployLogger log;

    public static final List<ConfigModelId> configModelIds =  
            ImmutableList.of(ConfigModelId.fromName("container"), ConfigModelId.fromName("jdisc"));

    private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName();
    private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName();

    public ContainerModelBuilder(boolean standaloneBuilder, Networking networking) {
        super(ContainerModel.class);
        this.standaloneBuilder = standaloneBuilder;
        this.networking = networking;
    }

    @Override
    public List<ConfigModelId> handlesElements() {
        return configModelIds;
    }

    @Override
    public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) {
        app = modelContext.getApplicationPackage();
        checkVersion(spec);

        this.log = modelContext.getDeployLogger();
        ContainerCluster cluster = createContainerCluster(spec, modelContext);
        addClusterContent(cluster, spec, modelContext);
        addBundlesForPlatformComponents(cluster);

        model.setCluster(cluster);
    }

    protected void addBundlesForPlatformComponents(ContainerCluster cluster) {
        for (Component<?, ?> component : cluster.getAllComponents()) {
            String componentClass = component.model.bundleInstantiationSpec.getClassName();
            BundleMapper.getBundlePath(componentClass).
                    ifPresent(cluster::addPlatformBundle);
        }
    }

    private ContainerCluster createContainerCluster(Element spec, final ConfigModelContext modelContext) {
        return new VespaDomBuilder.DomConfigProducerBuilder<ContainerCluster>() {
            @Override
            protected ContainerCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
                return new ContainerCluster(ancestor, modelContext.getProducerId(), modelContext.getProducerId());
            }
        }.build(modelContext.getParentProducer(), spec);
    }

    private void addClusterContent(ContainerCluster cluster, Element spec, ConfigModelContext context) {
        DocumentFactoryBuilder.buildDocumentFactories(cluster, spec);

        addConfiguredComponents(cluster, spec);
        addHandlers(cluster, spec);

        addHttp(spec, cluster);
        addRestApis(spec, cluster);
        addServlets(spec, cluster);
        addProcessing(spec, cluster);
        addSearch(spec, cluster, context.getDeployState().getQueryProfiles(), context.getDeployState().getSemanticRules());
        addDocproc(spec, cluster);
        addDocumentApi(spec, cluster);  // NOTE: Must be done after addSearch

        addAccessLogs(cluster, spec);
        addRoutingAliases(cluster, spec, context.getDeployState().zone().environment());
        addNodes(cluster, spec, context);

        addClientProviders(spec, cluster);
        addServerProviders(spec, cluster);
        addLegacyFilters(spec, cluster);

        addDefaultHandlers(cluster);
        addStatusHandlers(cluster, context);
        addDefaultComponents(cluster);
        setDefaultMetricConsumerFactory(cluster);

        //TODO: overview handler, see DomQrserverClusterBuilder
        //TODO: cache options.
    }

    private void addRoutingAliases(ContainerCluster cluster, Element spec, Environment environment) {
        if (environment != Environment.prod) return;

        Element aliases = XML.getChild(spec, "aliases");
        for (Element alias : XML.getChildren(aliases, "service-alias")) {
            cluster.serviceAliases().add(XML.getValue(alias));
        }
        for (Element alias : XML.getChildren(aliases, "endpoint-alias")) {
            cluster.endpointAliases().add(XML.getValue(alias));
        }
    }

    private void addConfiguredComponents(ContainerCluster cluster, Element spec) {
        for (Element components : XML.getChildren(spec, "components")) {
            addIncludes(components);
            addConfiguredComponents(cluster, components, "component");
        }
        addConfiguredComponents(cluster, spec, "component");
    }<commit_msg>Add TODO for removing legacy 'filter' under 'jdisc'<commit_after>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.xml;

import com.google.common.collect.ImmutableList;
import com.yahoo.component.Version;
import com.yahoo.config.application.Xml;
import com.yahoo.config.model.ConfigModelContext;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.application.provider.IncludeDirs;
import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
import com.yahoo.config.model.builder.xml.ConfigModelId;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.NodeType;
import com.yahoo.container.jdisc.config.MetricDefaultsConfig;
import com.yahoo.search.rendering.RendererRegistry;
import com.yahoo.text.XML;
import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.model.AbstractService;
import com.yahoo.vespa.model.HostResource;
import com.yahoo.vespa.model.builder.xml.dom.DomClientProviderBuilder;
import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder;
import com.yahoo.vespa.model.builder.xml.dom.DomFilterBuilder;
import com.yahoo.vespa.model.builder.xml.dom.DomHandlerBuilder;
import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
import com.yahoo.vespa.model.builder.xml.dom.ServletBuilder;
import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder;
import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessingBuilder;
import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearchChainsBuilder;
import com.yahoo.vespa.model.clients.ContainerDocumentApi;
import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.ContainerModel;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
import com.yahoo.vespa.model.container.docproc.DocprocChains;
import com.yahoo.vespa.model.container.http.Http;
import com.yahoo.vespa.model.container.http.xml.HttpBuilder;
import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder;
import com.yahoo.vespa.model.container.processing.ProcessingChains;
import com.yahoo.vespa.model.container.search.ContainerSearch;
import com.yahoo.vespa.model.container.search.PageTemplates;
import com.yahoo.vespa.model.container.search.QueryProfiles;
import com.yahoo.vespa.model.container.search.SemanticRules;
import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
import com.yahoo.vespa.model.container.xml.document.DocumentFactoryBuilder;

import com.yahoo.vespa.model.content.StorageGroup;
import org.w3c.dom.Element;
import org.w3c.dom.Node;

import java.util.*;
import java.util.function.Consumer;
import java.util.stream.Collectors;

/**
 * @author tonytv
 */
public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {

    /**
     * Default path to vip status file for container in Hosted Vespa.
     */
    static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().vespaHome() + "var/mediasearch/oor/status.html";
    /**
     * Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE
     */
    static final String HOSTED_VESPA_STATUS_FILE_YINST_SETTING = "cloudconfig_server__tenant_vip_status_file";

    public enum Networking { disable, enable }

    private ApplicationPackage app;
    private final boolean standaloneBuilder;
    private final Networking networking;
    protected DeployLogger log;

    public static final List<ConfigModelId> configModelIds =  
            ImmutableList.of(ConfigModelId.fromName("container"), ConfigModelId.fromName("jdisc"));

    private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName();
    private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName();

    public ContainerModelBuilder(boolean standaloneBuilder, Networking networking) {
        super(ContainerModel.class);
        this.standaloneBuilder = standaloneBuilder;
        this.networking = networking;
    }

    @Override
    public List<ConfigModelId> handlesElements() {
        return configModelIds;
    }

    @Override
    public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) {
        app = modelContext.getApplicationPackage();
        checkVersion(spec);

        this.log = modelContext.getDeployLogger();
        ContainerCluster cluster = createContainerCluster(spec, modelContext);
        addClusterContent(cluster, spec, modelContext);
        addBundlesForPlatformComponents(cluster);

        model.setCluster(cluster);
    }

    protected void addBundlesForPlatformComponents(ContainerCluster cluster) {
        for (Component<?, ?> component : cluster.getAllComponents()) {
            String componentClass = component.model.bundleInstantiationSpec.getClassName();
            BundleMapper.getBundlePath(componentClass).
                    ifPresent(cluster::addPlatformBundle);
        }
    }

    private ContainerCluster createContainerCluster(Element spec, final ConfigModelContext modelContext) {
        return new VespaDomBuilder.DomConfigProducerBuilder<ContainerCluster>() {
            @Override
            protected ContainerCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
                return new ContainerCluster(ancestor, modelContext.getProducerId(), modelContext.getProducerId());
            }
        }.build(modelContext.getParentProducer(), spec);
    }

    private void addClusterContent(ContainerCluster cluster, Element spec, ConfigModelContext context) {
        DocumentFactoryBuilder.buildDocumentFactories(cluster, spec);

        addConfiguredComponents(cluster, spec);
        addHandlers(cluster, spec);

        addHttp(spec, cluster);
        addRestApis(spec, cluster);
        addServlets(spec, cluster);
        addProcessing(spec, cluster);
        addSearch(spec, cluster, context.getDeployState().getQueryProfiles(), context.getDeployState().getSemanticRules());
        addDocproc(spec, cluster);
        addDocumentApi(spec, cluster);  // NOTE: Must be done after addSearch

        addAccessLogs(cluster, spec);
        addRoutingAliases(cluster, spec, context.getDeployState().zone().environment());
        addNodes(cluster, spec, context);

        addClientProviders(spec, cluster);
        addServerProviders(spec, cluster);
        addLegacyFilters(spec, cluster);  // TODO: Remove for Vespa 7

        addDefaultHandlers(cluster);
        addStatusHandlers(cluster, context);
        addDefaultComponents(cluster);
        setDefaultMetricConsumerFactory(cluster);

        //TODO: overview handler, see DomQrserverClusterBuilder
        //TODO: cache options.
    }

    private void addRoutingAliases(ContainerCluster cluster, Element spec, Environment environment) {
        if (environment != Environment.prod) return;

        Element aliases = XML.getChild(spec, "aliases");
        for (Element alias : XML.getChildren(aliases, "service-alias")) {
            cluster.serviceAliases().add(XML.getValue(alias));
        }
        for (Element alias : XML.getChildren(aliases, "endpoint-alias")) {
            cluster.endpointAliases().add(XML.getValue(alias));
        }
    }

    private void addConfiguredComponents(ContainerCluster cluster, Element spec) {
        for (Element components : XML.getChildren(spec, "components")) {
            addIncludes(components);
            addConfiguredComponents(cluster, components, "component");
        }
        addConfiguredComponents(cluster, spec, "component");
    }
{'fraction_non_alphanumeric': 0.09844684837130598, 'fraction_numerical': 0.001027376563727564, 'mean_word_length': 4.326037978757644, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 38}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '21961', 'n_tokens_mistral': 4649, 'n_tokens_neox': 4456, 'n_words': 822}