<commit_before>package io.syndesis.qe.templates;

import static org.assertj.core.api.Fail.fail;

import org.apache.commons.codec.binary.Base64;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;

import cz.xtf.openshift.OpenShiftBinaryClient;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.KubernetesList;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.ServiceAccount;
import io.fabric8.kubernetes.api.model.apiextensions.CustomResourceDefinition;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.utils.Serialization;
import io.fabric8.openshift.api.model.ImageStream;
import io.fabric8.openshift.api.model.ImageStreamList;
import io.fabric8.openshift.api.model.TagImportPolicy;
import io.fabric8.openshift.api.model.Template;
import io.syndesis.qe.Component;
import io.syndesis.qe.TestConfiguration;
import io.syndesis.qe.utils.HTTPResponse;
import io.syndesis.qe.utils.HttpUtils;
import io.syndesis.qe.utils.OpenShiftUtils;
import io.syndesis.qe.utils.TestUtils;
import io.syndesis.qe.utils.TodoUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class SyndesisTemplate {
    public static Template getTemplate() {
        try (InputStream is = new URL(TestConfiguration.syndesisTemplateUrl()).openStream()) {
            return OpenShiftUtils.client().templates().load(is).get();
        } catch (IOException ex) {
            throw new IllegalArgumentException("Unable to read template ", ex);
        }
    }

    public static ServiceAccount getSupportSA() {
        // Refresh the support SA URL as it can change during multiple tests executions
        try (InputStream is = new URL(TestConfiguration.syndesisTemplateSA()).openStream()) {
            return OpenShiftUtils.client().serviceAccounts().load(is).get();
        } catch (IOException ex) {
            throw new IllegalArgumentException("Unable to read SA ", ex);
        }
    }

    public static void deploy() {
        if (TestConfiguration.useOperator()) {
            deployUsingOperator();
        } else {
            deployUsingTemplate();
        }
    }

    public static void deployUsingTemplate() {
        log.info("Deploying using template");
        OpenShiftUtils.getInstance().cleanAndAssert();

        // get & create restricted SA
        OpenShiftUtils.getInstance().createServiceAccount(getSupportSA());
        // get token from SA `oc secrets get-token` && wait until created to prevent 404
        TestUtils.waitForEvent(Optional::isPresent,
                () -> OpenShiftUtils.getInstance().getSecrets().stream().filter(s -> s.getMetadata().getName().startsWith("syndesis-oauth-client-token")).findFirst(),
                TimeUnit.MINUTES,
                2,
                TimeUnit.SECONDS,
                5);

        Secret secret = OpenShiftUtils.getInstance().getSecrets().stream().filter(s -> s.getMetadata().getName().startsWith("syndesis-oauth-client-token")).findFirst().get();
        // token is Base64 encoded by default
        String oauthTokenEncoded = secret.getData().get("token");
        byte[] oauthTokenBytes = Base64.decodeBase64(oauthTokenEncoded);
        String oauthToken = new String(oauthTokenBytes);

        // get the template
        Template template = getTemplate();
        // set params
        Map<String, String> templateParams = new HashMap<>();
        templateParams.put("ROUTE_HOSTNAME", TestConfiguration.openShiftNamespace() + "." + TestConfiguration.openShiftRouteSuffix());
        templateParams.put("OPENSHIFT_MASTER", TestConfiguration.openShiftUrl());
        templateParams.put("OPENSHIFT_PROJECT", TestConfiguration.openShiftNamespace());
        templateParams.put("SAR_PROJECT", TestConfiguration.openShiftSARNamespace());
        templateParams.put("OPENSHIFT_OAUTH_CLIENT_SECRET", oauthToken);
        templateParams.put("TEST_SUPPORT_ENABLED", "true");
        templateParams.put("MAX_INTEGRATIONS_PER_USER", "5");
        if (TestUtils.isJenkins()) {
            templateParams.put("INTEGRATION_STATE_CHECK_INTERVAL", "150");
        }
        // process & create
        KubernetesList processedTemplate = OpenShiftUtils.getInstance().recreateAndProcessTemplate(template, templateParams);
        for (HasMetadata hasMetadata : processedTemplate.getItems()) {
            OpenShiftUtils.getInstance().createResources(hasMetadata);
        }

        //TODO: there's a bug in openshift-client, we need to initialize manually
        OpenShiftUtils.client().roleBindings().createOrReplaceWithNew()
                .withNewMetadata()
                    .withName("syndesis:editors")
                .endMetadata()
                .withNewRoleRef().withName("edit").endRoleRef()
                .addNewSubject().withKind("ServiceAccount").withName(Component.SERVER.getName()).withNamespace(TestConfiguration.openShiftNamespace()).endSubject()
                .addToUserNames(String.format("system:serviceaccount:%s:%s", TestConfiguration.openShiftNamespace(), Component.SERVER.getName()))
                .done();
        patchImageStreams();
        importProdImages();
    }

    private static void deployUsingOperator() {
        log.info("Deploying using Operator");
        if (!TestUtils.isUserAdmin()) {
            StringBuilder sb = new StringBuilder("\n");
            sb.append("****************************************************\n");
            sb.append("* Operator deployment needs user with admin rights *\n");
            sb.append("****************************************************\n");
            sb.append("If you are using minishift, you can use \"oc adm policy --as system:admin add-cluster-role-to-user cluster-admin developer\"\n");
            sb.append("Or use syndesis.config.template.use.operator=false\n");
            log.error(sb.toString());
            throw new RuntimeException(sb.toString());
        }

        OpenShiftUtils.getInstance().cleanAndAssert();
        deployCrd();
        // When testing upgrade using operator, the operator deploys last released version tag (and it is upgrading to latest daily), so this can be removed
        // when the version in syndesis-qe will be 1.7-SNAPSHOT
        // Use this only when doing upgrade
        if (System.getProperty("syndesis.upgrade.version") != null && Double.parseDouble(System.getProperty("syndesis.version").substring(0, 3)) <= 1.5) {
            deployOperatorPre73Way();
        } else {
            deployOperator();
        }
        importProdImages();
        deploySyndesisViaOperator();
        fixMavenRepos();
        patchImageStreams();
        // Prod template does have broker-amq deployment config defined for some reason, so delete it
        OpenShiftUtils.client().deploymentConfigs().withName("broker-amq").delete();
        TodoUtils.createDefaultRouteForTodo("todo2", "/");
    }

    private static void deployCrd() {
        log.info("Creating custom resource definition from " + TestConfiguration.syndesisOperatorCrdUrl());
        try (InputStream is = new URL(TestConfiguration.syndesisOperatorCrdUrl()).openStream()) {
            CustomResourceDefinition crd = OpenShiftUtils.client().customResourceDefinitions().load(is).get();
            OpenShiftUtils.client().customResourceDefinitions().create(crd);
        } catch (IOException ex) {
            throw new IllegalArgumentException("Unable to load CRD", ex);
        } catch (KubernetesClientException kce) {
            if (!kce.getMessage().contains("already exists")) {
                throw kce;
            }
        }
    }

    private static void deployOperator() {
        log.info("Deploying operator from " + TestConfiguration.syndesisOperatorUrl());
        final String output = OpenShiftBinaryClient.getInstance().executeCommandWithReturn(
                "Unable to process operator resource " + TestConfiguration.syndesisOperatorUrl(),
                "create",
                "-n", TestConfiguration.openShiftNamespace(),
                "-f", TestConfiguration.syndesisOperatorUrl()
        );
        log.debug(output);

        importProdImage("operator");

        log.info("Waiting for syndesis-operator to be ready");
        OpenShiftUtils.xtf().waiters()
                .areExactlyNPodsReady(1, "syndesis.io/component", "syndesis-operator")
                .interval(TimeUnit.SECONDS, 20)
                .timeout(TimeUnit.MINUTES, 10)
                .assertEventually();
    }

    /**
     * Used when deploying older versions of syndesis (currently used only in upgrade using operator tests).
     *
     * Can be removed when master is related to 7.4
     */
    private static void deployOperatorPre73Way() {
        log.info("Deploying operator using pre-7.3 way");
        final String output = OpenShiftBinaryClient.getInstance().executeCommandWithReturn(
                "Unable to create operator resource " + TestConfiguration.syndesisOperatorUrl(),
                "create",
                "-n", TestConfiguration.openShiftNamespace(),
                "-f", TestConfiguration.syndesisOperatorUrl()
        );
        log.debug(output);

        importProdImage("operator");

        log.info("Waiting for syndesis-operator to be ready");
        OpenShiftUtils.xtf().waiters()
                .areExactlyNPodsReady(1, "syndesis.io/component", "syndesis-operator")
                .interval(TimeUnit.SECONDS, 20)
                .timeout(TimeUnit.MINUTES, 10)
                .assertEventually();
    }

    private static void deploySyndesisViaOperator() {
        log.info("Deploying syndesis resource from " + TestConfiguration.syndesisOperatorTemplateUrl());
        try (InputStream is = new URL(TestConfiguration.syndesisOperatorTemplateUrl()).openStream()) {
            CustomResourceDefinition crd = OpenShiftUtils.client().customResourceDefinitions().load(is).get();
            Map<String, Object> integration = (Map)crd.getSpec().getAdditionalProperties().get("integration");
            integration.put("limit", 5);
            if (TestUtils.isJenkins()) {
                integration.put("stateCheckInterval", 150);
            }<commit_msg>[Deploy] Remove pre-7.3 deployment<commit_after>package io.syndesis.qe.templates;

import static org.assertj.core.api.Fail.fail;

import org.apache.commons.codec.binary.Base64;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;

import cz.xtf.openshift.OpenShiftBinaryClient;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.KubernetesList;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.ServiceAccount;
import io.fabric8.kubernetes.api.model.apiextensions.CustomResourceDefinition;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.utils.Serialization;
import io.fabric8.openshift.api.model.ImageStream;
import io.fabric8.openshift.api.model.ImageStreamList;
import io.fabric8.openshift.api.model.TagImportPolicy;
import io.fabric8.openshift.api.model.Template;
import io.syndesis.qe.Component;
import io.syndesis.qe.TestConfiguration;
import io.syndesis.qe.utils.HTTPResponse;
import io.syndesis.qe.utils.HttpUtils;
import io.syndesis.qe.utils.OpenShiftUtils;
import io.syndesis.qe.utils.TestUtils;
import io.syndesis.qe.utils.TodoUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class SyndesisTemplate {
    public static Template getTemplate() {
        try (InputStream is = new URL(TestConfiguration.syndesisTemplateUrl()).openStream()) {
            return OpenShiftUtils.client().templates().load(is).get();
        } catch (IOException ex) {
            throw new IllegalArgumentException("Unable to read template ", ex);
        }
    }

    public static ServiceAccount getSupportSA() {
        // Refresh the support SA URL as it can change during multiple tests executions
        try (InputStream is = new URL(TestConfiguration.syndesisTemplateSA()).openStream()) {
            return OpenShiftUtils.client().serviceAccounts().load(is).get();
        } catch (IOException ex) {
            throw new IllegalArgumentException("Unable to read SA ", ex);
        }
    }

    public static void deploy() {
        if (TestConfiguration.useOperator()) {
            deployUsingOperator();
        } else {
            deployUsingTemplate();
        }
    }

    public static void deployUsingTemplate() {
        log.info("Deploying using template");
        OpenShiftUtils.getInstance().cleanAndAssert();

        // get & create restricted SA
        OpenShiftUtils.getInstance().createServiceAccount(getSupportSA());
        // get token from SA `oc secrets get-token` && wait until created to prevent 404
        TestUtils.waitForEvent(Optional::isPresent,
                () -> OpenShiftUtils.getInstance().getSecrets().stream().filter(s -> s.getMetadata().getName().startsWith("syndesis-oauth-client-token")).findFirst(),
                TimeUnit.MINUTES,
                2,
                TimeUnit.SECONDS,
                5);

        Secret secret = OpenShiftUtils.getInstance().getSecrets().stream().filter(s -> s.getMetadata().getName().startsWith("syndesis-oauth-client-token")).findFirst().get();
        // token is Base64 encoded by default
        String oauthTokenEncoded = secret.getData().get("token");
        byte[] oauthTokenBytes = Base64.decodeBase64(oauthTokenEncoded);
        String oauthToken = new String(oauthTokenBytes);

        // get the template
        Template template = getTemplate();
        // set params
        Map<String, String> templateParams = new HashMap<>();
        templateParams.put("ROUTE_HOSTNAME", TestConfiguration.openShiftNamespace() + "." + TestConfiguration.openShiftRouteSuffix());
        templateParams.put("OPENSHIFT_MASTER", TestConfiguration.openShiftUrl());
        templateParams.put("OPENSHIFT_PROJECT", TestConfiguration.openShiftNamespace());
        templateParams.put("SAR_PROJECT", TestConfiguration.openShiftSARNamespace());
        templateParams.put("OPENSHIFT_OAUTH_CLIENT_SECRET", oauthToken);
        templateParams.put("TEST_SUPPORT_ENABLED", "true");
        templateParams.put("MAX_INTEGRATIONS_PER_USER", "5");
        if (TestUtils.isJenkins()) {
            templateParams.put("INTEGRATION_STATE_CHECK_INTERVAL", "150");
        }
        // process & create
        KubernetesList processedTemplate = OpenShiftUtils.getInstance().recreateAndProcessTemplate(template, templateParams);
        for (HasMetadata hasMetadata : processedTemplate.getItems()) {
            OpenShiftUtils.getInstance().createResources(hasMetadata);
        }

        //TODO: there's a bug in openshift-client, we need to initialize manually
        OpenShiftUtils.client().roleBindings().createOrReplaceWithNew()
                .withNewMetadata()
                    .withName("syndesis:editors")
                .endMetadata()
                .withNewRoleRef().withName("edit").endRoleRef()
                .addNewSubject().withKind("ServiceAccount").withName(Component.SERVER.getName()).withNamespace(TestConfiguration.openShiftNamespace()).endSubject()
                .addToUserNames(String.format("system:serviceaccount:%s:%s", TestConfiguration.openShiftNamespace(), Component.SERVER.getName()))
                .done();
        patchImageStreams();
        importProdImages();
    }

    private static void deployUsingOperator() {
        log.info("Deploying using Operator");
        if (!TestUtils.isUserAdmin()) {
            StringBuilder sb = new StringBuilder("\n");
            sb.append("****************************************************\n");
            sb.append("* Operator deployment needs user with admin rights *\n");
            sb.append("****************************************************\n");
            sb.append("If you are using minishift, you can use \"oc adm policy --as system:admin add-cluster-role-to-user cluster-admin developer\"\n");
            sb.append("Or use syndesis.config.template.use.operator=false\n");
            log.error(sb.toString());
            throw new RuntimeException(sb.toString());
        }

        OpenShiftUtils.getInstance().cleanAndAssert();
        deployCrd();
        deployOperator();
        importProdImages();
        deploySyndesisViaOperator();
        fixMavenRepos();
        patchImageStreams();
        // Prod template does have broker-amq deployment config defined for some reason, so delete it
        OpenShiftUtils.client().deploymentConfigs().withName("broker-amq").delete();
        TodoUtils.createDefaultRouteForTodo("todo2", "/");
    }

    private static void deployCrd() {
        log.info("Creating custom resource definition from " + TestConfiguration.syndesisOperatorCrdUrl());
        try (InputStream is = new URL(TestConfiguration.syndesisOperatorCrdUrl()).openStream()) {
            CustomResourceDefinition crd = OpenShiftUtils.client().customResourceDefinitions().load(is).get();
            OpenShiftUtils.client().customResourceDefinitions().create(crd);
        } catch (IOException ex) {
            throw new IllegalArgumentException("Unable to load CRD", ex);
        } catch (KubernetesClientException kce) {
            if (!kce.getMessage().contains("already exists")) {
                throw kce;
            }
        }
    }

    private static void deployOperator() {
        log.info("Deploying operator from " + TestConfiguration.syndesisOperatorUrl());
        final String output = OpenShiftBinaryClient.getInstance().executeCommandWithReturn(
                "Unable to process operator resource " + TestConfiguration.syndesisOperatorUrl(),
                "create",
                "-n", TestConfiguration.openShiftNamespace(),
                "-f", TestConfiguration.syndesisOperatorUrl()
        );
        log.debug(output);

        importProdImage("operator");

        log.info("Waiting for syndesis-operator to be ready");
        OpenShiftUtils.xtf().waiters()
                .areExactlyNPodsReady(1, "syndesis.io/component", "syndesis-operator")
                .interval(TimeUnit.SECONDS, 20)
                .timeout(TimeUnit.MINUTES, 10)
                .assertEventually();
    }

    private static void deploySyndesisViaOperator() {
        log.info("Deploying syndesis resource from " + TestConfiguration.syndesisOperatorTemplateUrl());
        try (InputStream is = new URL(TestConfiguration.syndesisOperatorTemplateUrl()).openStream()) {
            CustomResourceDefinition crd = OpenShiftUtils.client().customResourceDefinitions().load(is).get();
            Map<String, Object> integration = (Map)crd.getSpec().getAdditionalProperties().get("integration");
            integration.put("limit", 5);
            if (TestUtils.isJenkins()) {
                integration.put("stateCheckInterval", 150);
            }
{'fraction_non_alphanumeric': 0.1250063416366496, 'fraction_numerical': 0.005326974785652681, 'mean_word_length': 3.1118064246975385, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '830666', 'n_tokens_mistral': 5282, 'n_tokens_neox': 5054, 'n_words': 982}