<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.application.pkg;

import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeploymentInstanceSpec;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.Endpoint;
import com.yahoo.config.application.api.Endpoint.Level;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.zone.ZoneApi;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;

import java.time.Instant;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * This contains validators for a {@link ApplicationPackage} that depend on a {@link Controller} to perform validation.
 *
 * @author mpolden
 */
public class ApplicationPackageValidator {

    private final Controller controller;

    public ApplicationPackageValidator(Controller controller) {
        this.controller = Objects.requireNonNull(controller, "controller must be non-null");
    }

    /**
     * Validate the given application package
     *
     * @throws IllegalArgumentException if any validations fail
     */
    public void validate(Application application, ApplicationPackage applicationPackage, Instant instant) {
        validateSteps(applicationPackage.deploymentSpec());
        validateEndpointRegions(applicationPackage.deploymentSpec());
        validateEndpointChange(application, applicationPackage, instant);
        validateCompactedEndpoint(applicationPackage);
        validateSecurityClientsPem(applicationPackage);
        validateDeprecatedElements(applicationPackage);
    }

    /** Verify that deployment spec does not use elements deprecated on a major version older than wanted major version */
    private void validateDeprecatedElements(ApplicationPackage applicationPackage) {
        int wantedMajor = applicationPackage.compileVersion().map(Version::getMajor)
                                            .or(() -> applicationPackage.deploymentSpec().majorVersion())
                                            .or(() -> controller.readVersionStatus().controllerVersion()
                                                                .map(VespaVersion::versionNumber)
                                                                .map(Version::getMajor))
                                            .orElseThrow(() -> new IllegalArgumentException("Could not determine wanted major version"));
        for (var deprecatedElement : applicationPackage.deploymentSpec().deprecatedElements()) {
            if (deprecatedElement.majorVersion() >= wantedMajor) continue;
            throw new IllegalArgumentException(deprecatedElement.humanReadableString());
        }
    }

    /** Verify that we have the security/clients.pem file for public systems */
    private void validateSecurityClientsPem(ApplicationPackage applicationPackage) {
        if (!controller.system().isPublic() || applicationPackage.deploymentSpec().steps().isEmpty()) return;
        if (applicationPackage.trustedCertificates().isEmpty())
            throw new IllegalArgumentException("Missing required file 'security/clients.pem'");
    }

    /** Verify that each of the production zones listed in the deployment spec exist in this system */
    private void validateSteps(DeploymentSpec deploymentSpec) {
        for (var spec : deploymentSpec.instances()) {
            for (var zone : spec.zones()) {
                Environment environment = zone.environment();
                if (zone.region().isEmpty()) continue;
                ZoneId zoneId = ZoneId.from(environment, zone.region().get());
                if (!controller.zoneRegistry().hasZone(zoneId)) {
                    throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!");
                }
            }
        }
    }

    /** Verify that:
     * <ul>
     *     <li>no single endpoint contains regions in different clouds</li>
     *     <li>application endpoints with different regions must be contained in CGP and AWS</li>
     * </ul>
     */
    private void validateEndpointRegions(DeploymentSpec deploymentSpec) {
        for (var instance : deploymentSpec.instances()) {
            validateEndpointRegions(instance.endpoints(), instance);
        }
        validateEndpointRegions(deploymentSpec.endpoints(), null);
    }

    private void validateEndpointRegions(List<Endpoint> endpoints, DeploymentInstanceSpec instance) {
        for (var endpoint : endpoints) {
            RegionName[] regions = new HashSet<>(endpoint.regions()).toArray(RegionName[]::new);
            Set<CloudName> clouds = controller.zoneRegistry().zones().all().in(Environment.prod)
                                              .in(regions)
                                              .zones().stream()
                                              .map(ZoneApi::getCloudName)
                                              .collect(Collectors.toSet());
            String endpointString = instance == null ? "Application endpoint '" + endpoint.endpointId() + "'"
                                                     : "Endpoint '" + endpoint.endpointId() + "' in " + instance;
            if (Set.of(CloudName.GCP, CloudName.AWS).containsAll(clouds)) { } // Everything is fine!
            else if (Set.of(CloudName.DEFAULT).containsAll(clouds)) {
                if (endpoint.level() == Level.application && regions.length != 1) {
                    throw new IllegalArgumentException(endpointString + " cannot contain different regions: " +
                                                       endpoint.regions().stream().sorted().toList());
                }
            }
            else {
                throw new IllegalArgumentException(endpointString + " cannot contain regions in different clouds: " +
                                                   endpoint.regions().stream().sorted().toList());
            }
        }
    }

    /** Verify endpoint configuration of given application package */
    private void validateEndpointChange(Application application, ApplicationPackage applicationPackage, Instant instant) {
        applicationPackage.deploymentSpec().instances().forEach(instance -> validateEndpointChange(application,
                                                                                                   instance.name(),
                                                                                                   applicationPackage,
                                                                                                   instant));
    }

    /** Verify that compactable endpoint parts (instance name and endpoint ID) do not clash */
    private void validateCompactedEndpoint(ApplicationPackage applicationPackage) {
        Map<List<String>, InstanceEndpoint> instanceEndpoints = new HashMap<>();<commit_msg>Use right cloud name in verification<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.application.pkg;

import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeploymentInstanceSpec;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.Endpoint;
import com.yahoo.config.application.api.Endpoint.Level;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.zone.ZoneApi;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;

import java.time.Instant;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * This contains validators for a {@link ApplicationPackage} that depend on a {@link Controller} to perform validation.
 *
 * @author mpolden
 */
public class ApplicationPackageValidator {

    private final Controller controller;

    public ApplicationPackageValidator(Controller controller) {
        this.controller = Objects.requireNonNull(controller, "controller must be non-null");
    }

    /**
     * Validate the given application package
     *
     * @throws IllegalArgumentException if any validations fail
     */
    public void validate(Application application, ApplicationPackage applicationPackage, Instant instant) {
        validateSteps(applicationPackage.deploymentSpec());
        validateEndpointRegions(applicationPackage.deploymentSpec());
        validateEndpointChange(application, applicationPackage, instant);
        validateCompactedEndpoint(applicationPackage);
        validateSecurityClientsPem(applicationPackage);
        validateDeprecatedElements(applicationPackage);
    }

    /** Verify that deployment spec does not use elements deprecated on a major version older than wanted major version */
    private void validateDeprecatedElements(ApplicationPackage applicationPackage) {
        int wantedMajor = applicationPackage.compileVersion().map(Version::getMajor)
                                            .or(() -> applicationPackage.deploymentSpec().majorVersion())
                                            .or(() -> controller.readVersionStatus().controllerVersion()
                                                                .map(VespaVersion::versionNumber)
                                                                .map(Version::getMajor))
                                            .orElseThrow(() -> new IllegalArgumentException("Could not determine wanted major version"));
        for (var deprecatedElement : applicationPackage.deploymentSpec().deprecatedElements()) {
            if (deprecatedElement.majorVersion() >= wantedMajor) continue;
            throw new IllegalArgumentException(deprecatedElement.humanReadableString());
        }
    }

    /** Verify that we have the security/clients.pem file for public systems */
    private void validateSecurityClientsPem(ApplicationPackage applicationPackage) {
        if (!controller.system().isPublic() || applicationPackage.deploymentSpec().steps().isEmpty()) return;
        if (applicationPackage.trustedCertificates().isEmpty())
            throw new IllegalArgumentException("Missing required file 'security/clients.pem'");
    }

    /** Verify that each of the production zones listed in the deployment spec exist in this system */
    private void validateSteps(DeploymentSpec deploymentSpec) {
        for (var spec : deploymentSpec.instances()) {
            for (var zone : spec.zones()) {
                Environment environment = zone.environment();
                if (zone.region().isEmpty()) continue;
                ZoneId zoneId = ZoneId.from(environment, zone.region().get());
                if (!controller.zoneRegistry().hasZone(zoneId)) {
                    throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!");
                }
            }
        }
    }

    /** Verify that:
     * <ul>
     *     <li>no single endpoint contains regions in different clouds</li>
     *     <li>application endpoints with different regions must be contained in CGP and AWS</li>
     * </ul>
     */
    private void validateEndpointRegions(DeploymentSpec deploymentSpec) {
        for (var instance : deploymentSpec.instances()) {
            validateEndpointRegions(instance.endpoints(), instance);
        }
        validateEndpointRegions(deploymentSpec.endpoints(), null);
    }

    private void validateEndpointRegions(List<Endpoint> endpoints, DeploymentInstanceSpec instance) {
        for (var endpoint : endpoints) {
            RegionName[] regions = new HashSet<>(endpoint.regions()).toArray(RegionName[]::new);
            Set<CloudName> clouds = controller.zoneRegistry().zones().all().in(Environment.prod)
                                              .in(regions)
                                              .zones().stream()
                                              .map(ZoneApi::getCloudName)
                                              .collect(Collectors.toSet());
            String endpointString = instance == null ? "Application endpoint '" + endpoint.endpointId() + "'"
                                                     : "Endpoint '" + endpoint.endpointId() + "' in " + instance;
            if (Set.of(CloudName.GCP, CloudName.AWS).containsAll(clouds)) { } // Everything is fine!
            else if (Set.of(CloudName.from("yahoo")).containsAll(clouds)) {
                if (endpoint.level() == Level.application && regions.length != 1) {
                    throw new IllegalArgumentException(endpointString + " cannot contain different regions: " +
                                                       endpoint.regions().stream().sorted().toList());
                }
            }
            else {
                throw new IllegalArgumentException(endpointString + " cannot contain regions in different clouds: " +
                                                   endpoint.regions().stream().sorted().toList());
            }
        }
    }

    /** Verify endpoint configuration of given application package */
    private void validateEndpointChange(Application application, ApplicationPackage applicationPackage, Instant instant) {
        applicationPackage.deploymentSpec().instances().forEach(instance -> validateEndpointChange(application,
                                                                                                   instance.name(),
                                                                                                   applicationPackage,
                                                                                                   instant));
    }

    /** Verify that compactable endpoint parts (instance name and endpoint ID) do not clash */
    private void validateCompactedEndpoint(ApplicationPackage applicationPackage) {
        Map<List<String>, InstanceEndpoint> instanceEndpoints = new HashMap<>();
{'fraction_non_alphanumeric': 0.09346766330841728, 'fraction_numerical': 0.0003899902502437439, 'mean_word_length': 2.3145196036191296, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1095765', 'n_tokens_mistral': 3358, 'n_tokens_neox': 3260, 'n_words': 826}