<commit_before>/**
 * JBoss, Home of Professional Open Source.
 * Copyright 2014-2020 Red Hat, Inc., and individual contributors
 * as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jboss.pnc.indyrepositorymanager;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.commonjava.atlas.maven.ident.ref.ArtifactRef;
import org.commonjava.atlas.maven.ident.ref.SimpleArtifactRef;
import org.commonjava.atlas.maven.ident.util.ArtifactPathInfo;
import org.commonjava.atlas.npm.ident.ref.NpmPackageRef;
import org.commonjava.atlas.npm.ident.util.NpmPackagePathInfo;
import org.commonjava.indy.client.core.Indy;
import org.commonjava.indy.client.core.IndyClientException;
import org.commonjava.indy.client.core.module.IndyContentClientModule;
import org.commonjava.indy.folo.client.IndyFoloAdminClientModule;
import org.commonjava.indy.folo.dto.TrackedContentDTO;
import org.commonjava.indy.folo.dto.TrackedContentEntryDTO;
import org.commonjava.indy.model.core.HostedRepository;
import org.commonjava.indy.model.core.StoreKey;
import org.commonjava.indy.model.core.StoreType;
import org.commonjava.indy.promote.client.IndyPromoteClientModule;
import org.commonjava.indy.promote.model.AbstractPromoteResult;
import org.commonjava.indy.promote.model.PathsPromoteRequest;
import org.commonjava.indy.promote.model.PathsPromoteResult;
import org.commonjava.indy.promote.model.ValidationResult;
import org.jboss.pnc.constants.ReposiotryIdentifier;
import org.jboss.pnc.enums.ArtifactQuality;
import org.jboss.pnc.enums.BuildCategory;
import org.jboss.pnc.enums.RepositoryType;
import org.jboss.pnc.model.Artifact;
import org.jboss.pnc.model.TargetRepository;
import org.jboss.pnc.spi.coordinator.CompletionStatus;
import org.jboss.pnc.spi.repositorymanager.RepositoryManagerException;
import org.jboss.pnc.spi.repositorymanager.RepositoryManagerResult;
import org.jboss.pnc.spi.repositorymanager.model.RepositoryConnectionInfo;
import org.jboss.pnc.spi.repositorymanager.model.RepositorySession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.validation.ConstraintViolation;
import javax.validation.Validation;
import javax.validation.Validator;
import javax.validation.ValidatorFactory;
import java.io.File;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import static org.commonjava.indy.model.core.GenericPackageTypeDescriptor.GENERIC_PKG_KEY;
import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY;
import static org.commonjava.indy.pkg.npm.model.NPMPackageTypeDescriptor.NPM_PKG_KEY;
import static org.jboss.pnc.indyrepositorymanager.IndyRepositoryConstants.SHARED_IMPORTS_ID;

/**
 * {@link RepositorySession} implementation that works with the Maven {@link RepositoryManagerDriver} (which connects to
 * an Indy server instance for repository management). This session contains connection information for rendering Maven
 * settings.xml files and the like, along with the components necessary to extract the artifacts (dependencies, build
 * uploads) for the associated build.
 *
 * Artifact extraction also implies promotion of imported dependencies to a shared-imports Maven repository for safe
 * keeping. In the case of composed (chained) builds, it also implies promotion of the build output to the associated
 * build-set repository group, to expose them for use in successive builds in the chain.
 */
public class IndyRepositorySession implements RepositorySession {

    private final Logger logger = LoggerFactory.getLogger(getClass());
    private static final Logger userLog = LoggerFactory.getLogger("org.jboss.pnc._userlog_.build-executor");

    private static ValidatorFactory validatorFactory = Validation.buildDefaultValidatorFactory();

    private boolean isTempBuild;

    private Indy indy;
    private Indy serviceAccountIndy;
    private final String buildContentId;
    private final String packageType;
    /**
     * PackageType-specific internal repository name patterns.
     */
    private ArtifactFilter artifactFilter;

    private final RepositoryConnectionInfo connectionInfo;

    private final Validator validator;

    private String buildPromotionTarget;

    private BuildCategory buildCategory;

    private static Set<String> checksumSuffixes;
    static {
        checksumSuffixes = new HashSet<>(4);
        checksumSuffixes.add("md5");
        checksumSuffixes.add("sha1");
        checksumSuffixes.add("sha256");
        checksumSuffixes.add("sha512");
    }

    public IndyRepositorySession(
            Indy indy,
            Indy serviceAccountIndy,
            String buildContentId,
            String packageType,
            IndyRepositoryConnectionInfo info,
            ArtifactFilter artifactFilter,
            String buildPromotionTarget,
            BuildCategory buildCategory,
            boolean isTempBuild) {
        this.validator = validatorFactory.getValidator();
        this.indy = indy;
        this.serviceAccountIndy = serviceAccountIndy;
        this.buildContentId = buildContentId;
        this.packageType = packageType;
        this.artifactFilter = artifactFilter;
        this.connectionInfo = info;
        this.buildPromotionTarget = buildPromotionTarget;
        this.buildCategory = buildCategory;
        this.isTempBuild = isTempBuild;
    }

    @Override
    public String toString() {
        return "MavenRepositoryConfiguration " + this.hashCode();
    }

    @Override
    public RepositoryType getType() {
        return toRepoType(packageType);
    }

    @Override
    public String getBuildRepositoryId() {
        return buildContentId;
    }

    @Override
    public RepositoryConnectionInfo getConnectionInfo() {
        return connectionInfo;
    }

    /**
     * Retrieve tracking report from repository manager. Add each tracked download to the dependencies of the build
     * result. Add each tracked upload to the built artifacts of the build result. Promote uploaded artifacts to the
     * product-level storage. Finally delete the group associated with the completed build.
     *
     * @param liveBuild {@inheritDoc} - if true, the promotion of the collected artifacts and dependencies is done,
     *        tracking report is sealed and the build group is removed, if false it only reads the data without any
     *        actions
     */
    @Override
    public RepositoryManagerResult extractBuildArtifacts(final boolean liveBuild) throws RepositoryManagerException {

        TrackedContentDTO report = sealAndGetTrackingReport(liveBuild);

        Comparator<Artifact> comp = (one, two) -> one.getIdentifier().compareTo(two.getIdentifier());

        Uploads uploads = collectUploads(report);
        List<Artifact> uploadedArtifacts = uploads.getData();
        Collections.sort(uploadedArtifacts, comp);

        List<Artifact> downloadedArtifacts = null;
        String log = "";
        CompletionStatus status = CompletionStatus.SUCCESS;

        try {
            downloadedArtifacts = processDownloads(report, liveBuild);
            Collections.sort(downloadedArtifacts, comp);
        } catch (PromotionValidationException ex) {
            status = CompletionStatus.FAILED;
            log = ex.getMessage();
            logger.warn("Dependencies promotion failed. Error(s): {}", log);
            userLog.error("Built artifact promotion failed. Error(s): {}", log);
        }

        if (liveBuild) {
            deleteBuildGroup();
        }

        // if the promotion of dependencies succeeded...
        if (status == CompletionStatus.SUCCESS) {
            logger.info(
                    "Returning built artifacts / dependencies:\nUploads:\n  {}\n\nDownloads:\n  {}\n\n",
                    StringUtils.join(uploads.getData(), "\n  "),
                    StringUtils.join(downloadedArtifacts, "\n  "));

            if (liveBuild) {
                logger.info("BEGIN: promotion to build content set");<commit_msg>Fix log message.<commit_after>/**
 * JBoss, Home of Professional Open Source.
 * Copyright 2014-2020 Red Hat, Inc., and individual contributors
 * as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jboss.pnc.indyrepositorymanager;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.commonjava.atlas.maven.ident.ref.ArtifactRef;
import org.commonjava.atlas.maven.ident.ref.SimpleArtifactRef;
import org.commonjava.atlas.maven.ident.util.ArtifactPathInfo;
import org.commonjava.atlas.npm.ident.ref.NpmPackageRef;
import org.commonjava.atlas.npm.ident.util.NpmPackagePathInfo;
import org.commonjava.indy.client.core.Indy;
import org.commonjava.indy.client.core.IndyClientException;
import org.commonjava.indy.client.core.module.IndyContentClientModule;
import org.commonjava.indy.folo.client.IndyFoloAdminClientModule;
import org.commonjava.indy.folo.dto.TrackedContentDTO;
import org.commonjava.indy.folo.dto.TrackedContentEntryDTO;
import org.commonjava.indy.model.core.HostedRepository;
import org.commonjava.indy.model.core.StoreKey;
import org.commonjava.indy.model.core.StoreType;
import org.commonjava.indy.promote.client.IndyPromoteClientModule;
import org.commonjava.indy.promote.model.AbstractPromoteResult;
import org.commonjava.indy.promote.model.PathsPromoteRequest;
import org.commonjava.indy.promote.model.PathsPromoteResult;
import org.commonjava.indy.promote.model.ValidationResult;
import org.jboss.pnc.constants.ReposiotryIdentifier;
import org.jboss.pnc.enums.ArtifactQuality;
import org.jboss.pnc.enums.BuildCategory;
import org.jboss.pnc.enums.RepositoryType;
import org.jboss.pnc.model.Artifact;
import org.jboss.pnc.model.TargetRepository;
import org.jboss.pnc.spi.coordinator.CompletionStatus;
import org.jboss.pnc.spi.repositorymanager.RepositoryManagerException;
import org.jboss.pnc.spi.repositorymanager.RepositoryManagerResult;
import org.jboss.pnc.spi.repositorymanager.model.RepositoryConnectionInfo;
import org.jboss.pnc.spi.repositorymanager.model.RepositorySession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.validation.ConstraintViolation;
import javax.validation.Validation;
import javax.validation.Validator;
import javax.validation.ValidatorFactory;
import java.io.File;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import static org.commonjava.indy.model.core.GenericPackageTypeDescriptor.GENERIC_PKG_KEY;
import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY;
import static org.commonjava.indy.pkg.npm.model.NPMPackageTypeDescriptor.NPM_PKG_KEY;
import static org.jboss.pnc.indyrepositorymanager.IndyRepositoryConstants.SHARED_IMPORTS_ID;

/**
 * {@link RepositorySession} implementation that works with the Maven {@link RepositoryManagerDriver} (which connects to
 * an Indy server instance for repository management). This session contains connection information for rendering Maven
 * settings.xml files and the like, along with the components necessary to extract the artifacts (dependencies, build
 * uploads) for the associated build.
 *
 * Artifact extraction also implies promotion of imported dependencies to a shared-imports Maven repository for safe
 * keeping. In the case of composed (chained) builds, it also implies promotion of the build output to the associated
 * build-set repository group, to expose them for use in successive builds in the chain.
 */
public class IndyRepositorySession implements RepositorySession {

    private final Logger logger = LoggerFactory.getLogger(getClass());
    private static final Logger userLog = LoggerFactory.getLogger("org.jboss.pnc._userlog_.build-executor");

    private static ValidatorFactory validatorFactory = Validation.buildDefaultValidatorFactory();

    private boolean isTempBuild;

    private Indy indy;
    private Indy serviceAccountIndy;
    private final String buildContentId;
    private final String packageType;
    /**
     * PackageType-specific internal repository name patterns.
     */
    private ArtifactFilter artifactFilter;

    private final RepositoryConnectionInfo connectionInfo;

    private final Validator validator;

    private String buildPromotionTarget;

    private BuildCategory buildCategory;

    private static Set<String> checksumSuffixes;
    static {
        checksumSuffixes = new HashSet<>(4);
        checksumSuffixes.add("md5");
        checksumSuffixes.add("sha1");
        checksumSuffixes.add("sha256");
        checksumSuffixes.add("sha512");
    }

    public IndyRepositorySession(
            Indy indy,
            Indy serviceAccountIndy,
            String buildContentId,
            String packageType,
            IndyRepositoryConnectionInfo info,
            ArtifactFilter artifactFilter,
            String buildPromotionTarget,
            BuildCategory buildCategory,
            boolean isTempBuild) {
        this.validator = validatorFactory.getValidator();
        this.indy = indy;
        this.serviceAccountIndy = serviceAccountIndy;
        this.buildContentId = buildContentId;
        this.packageType = packageType;
        this.artifactFilter = artifactFilter;
        this.connectionInfo = info;
        this.buildPromotionTarget = buildPromotionTarget;
        this.buildCategory = buildCategory;
        this.isTempBuild = isTempBuild;
    }

    @Override
    public String toString() {
        return "MavenRepositoryConfiguration " + this.hashCode();
    }

    @Override
    public RepositoryType getType() {
        return toRepoType(packageType);
    }

    @Override
    public String getBuildRepositoryId() {
        return buildContentId;
    }

    @Override
    public RepositoryConnectionInfo getConnectionInfo() {
        return connectionInfo;
    }

    /**
     * Retrieve tracking report from repository manager. Add each tracked download to the dependencies of the build
     * result. Add each tracked upload to the built artifacts of the build result. Promote uploaded artifacts to the
     * product-level storage. Finally delete the group associated with the completed build.
     *
     * @param liveBuild {@inheritDoc} - if true, the promotion of the collected artifacts and dependencies is done,
     *        tracking report is sealed and the build group is removed, if false it only reads the data without any
     *        actions
     */
    @Override
    public RepositoryManagerResult extractBuildArtifacts(final boolean liveBuild) throws RepositoryManagerException {

        TrackedContentDTO report = sealAndGetTrackingReport(liveBuild);

        Comparator<Artifact> comp = (one, two) -> one.getIdentifier().compareTo(two.getIdentifier());

        Uploads uploads = collectUploads(report);
        List<Artifact> uploadedArtifacts = uploads.getData();
        Collections.sort(uploadedArtifacts, comp);

        List<Artifact> downloadedArtifacts = null;
        String log = "";
        CompletionStatus status = CompletionStatus.SUCCESS;

        try {
            downloadedArtifacts = processDownloads(report, liveBuild);
            Collections.sort(downloadedArtifacts, comp);
        } catch (PromotionValidationException ex) {
            status = CompletionStatus.FAILED;
            log = ex.getMessage();
            logger.warn("Dependencies promotion failed. Error(s): {}", log);
            userLog.error("Dependencies promotion failed. Error(s): {}", log);
        }

        if (liveBuild) {
            deleteBuildGroup();
        }

        // if the promotion of dependencies succeeded...
        if (status == CompletionStatus.SUCCESS) {
            logger.info(
                    "Returning built artifacts / dependencies:\nUploads:\n  {}\n\nDownloads:\n  {}\n\n",
                    StringUtils.join(uploads.getData(), "\n  "),
                    StringUtils.join(downloadedArtifacts, "\n  "));

            if (liveBuild) {
                logger.info("BEGIN: promotion to build content set");
{'fraction_non_alphanumeric': 0.08295217121237788, 'fraction_numerical': 0.002710486193460952, 'mean_word_length': 3.7416331994645247, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1896528', 'n_tokens_mistral': 4852, 'n_tokens_neox': 4659, 'n_words': 1236}