<commit_before>/**
 * JBoss, Home of Professional Open Source.
 * Copyright 2014-2020 Red Hat, Inc., and individual contributors
 * as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jboss.pnc.datastore;

import org.jboss.pnc.enums.RepositoryType;
import org.jboss.pnc.model.Artifact;
import org.jboss.pnc.model.BuildConfigSetRecord;
import org.jboss.pnc.model.BuildConfiguration;
import org.jboss.pnc.model.BuildConfigurationAudited;
import org.jboss.pnc.model.BuildConfigurationSet;
import org.jboss.pnc.model.BuildRecord;
import org.jboss.pnc.model.IdRev;
import org.jboss.pnc.model.TargetRepository;
import org.jboss.pnc.model.User;
import org.jboss.pnc.spi.coordinator.BuildTask;
import org.jboss.pnc.spi.datastore.Datastore;
import org.jboss.pnc.spi.datastore.repositories.ArtifactRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildConfigSetRecordRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildConfigurationAuditedRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildConfigurationRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildRecordRepository;
import org.jboss.pnc.spi.datastore.repositories.TargetRepositoryRepository;
import org.jboss.pnc.spi.datastore.repositories.UserRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ejb.Stateless;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.inject.Inject;

import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.jboss.pnc.common.util.CollectionUtils.ofNullableCollection;
import static org.jboss.pnc.spi.datastore.predicates.ArtifactPredicates.withIdentifierInAndBuilt;
import static org.jboss.pnc.spi.datastore.predicates.ArtifactPredicates.withSha256InAndBuilt;
import static org.jboss.pnc.spi.datastore.predicates.BuildConfigurationPredicates.withBuildConfigurationSetId;
import static org.jboss.pnc.spi.datastore.predicates.UserPredicates.withUserName;

@Stateless
public class DefaultDatastore implements Datastore {

    public static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

    private ArtifactRepository artifactRepository;

    private BuildRecordRepository buildRecordRepository;

    private BuildConfigurationRepository buildConfigurationRepository;

    private BuildConfigurationAuditedRepository buildConfigurationAuditedRepository;

    private BuildConfigSetRecordRepository buildConfigSetRecordRepository;

    private UserRepository userRepository;

    private TargetRepositoryRepository targetRepositoryRepository;

    public DefaultDatastore() {
    }

    @Inject
    public DefaultDatastore(
            ArtifactRepository artifactRepository,
            BuildRecordRepository buildRecordRepository,
            BuildConfigurationRepository buildConfigurationRepository,
            BuildConfigurationAuditedRepository buildConfigurationAuditedRepository,
            BuildConfigSetRecordRepository buildConfigSetRecordRepository,
            UserRepository userRepository,
            TargetRepositoryRepository targetRepositoryRepository) {
        this.artifactRepository = artifactRepository;
        this.buildRecordRepository = buildRecordRepository;
        this.buildConfigurationRepository = buildConfigurationRepository;
        this.buildConfigurationAuditedRepository = buildConfigurationAuditedRepository;
        this.buildConfigSetRecordRepository = buildConfigSetRecordRepository;
        this.userRepository = userRepository;
        this.targetRepositoryRepository = targetRepositoryRepository;
    }

    private static final String ARTIFACT_ALREADY_BUILT_CONFLICT_MESSAGE = "This artifact was already built in build #";

    @Override
    public Map<Artifact, String> checkForBuiltArtifacts(Collection<Artifact> artifacts) {
        Map<RepositoryType, Map<String, Artifact>> repoTypes = new HashMap<>();
        for (Artifact artifact : artifacts) {
            RepositoryType repoType = artifact.getTargetRepository().getRepositoryType();
            if (!repoTypes.containsKey(repoType)) {
                repoTypes.put(repoType, new HashMap<>());
            }
            Map<String, Artifact> identifiers = repoTypes.get(repoType);
            identifiers.put(artifact.getIdentifier(), artifact);
        }

        Map<Artifact, String> conflicts = new HashMap<>();
        for (RepositoryType repoType : repoTypes.keySet()) {
            Map<String, Artifact> identifiers = repoTypes.get(repoType);
            List<Artifact> conflicting = artifactRepository
                    .queryWithPredicates(withIdentifierInAndBuilt(identifiers.keySet()));
            for (Artifact conflict : conflicting) {
                if (conflict.getTargetRepository().getRepositoryType() == repoType) {
                    Artifact artifact = identifiers.get(conflict.getIdentifier());
                    conflicts
                            .put(artifact, ARTIFACT_ALREADY_BUILT_CONFLICT_MESSAGE + conflict.getBuildRecord().getId());
                }
            }
        }
        return conflicts;
    }

    @Override
    @TransactionAttribute(TransactionAttributeType.REQUIRED)
    public BuildRecord storeCompletedBuild(
            BuildRecord.Builder buildRecordBuilder,
            List<Artifact> builtArtifacts,
            List<Artifact> dependencies) {
        BuildRecord buildRecord = buildRecordBuilder.build(true);
        logger.debug("Storing completed build {}.", buildRecord);
        if (logger.isTraceEnabled()) {
            logger.trace("Build Log: {}.", buildRecord.getBuildLog());
        }

        Map<TargetRepository.IdentifierPath, TargetRepository> repositoriesCache = new HashMap<>();
        Map<Artifact.IdentifierSha256, Artifact> artifactCache = new HashMap<>();

        /**
         * Built artifacts must be saved before the dependencies. In case an artifact is built and the dependency
         * (re-downloaded), it must be linked to built artifacts repository.
         */
        logger.debug("Saving built artifacts ...");
        final Set<Artifact> savedBuiltArtifacts = saveArtifacts(builtArtifacts, repositoriesCache, artifactCache);

        logger.debug("Saving dependencies ...");
        buildRecord.setDependencies(saveArtifacts(dependencies, repositoriesCache, artifactCache));

        logger.debug("Done saving artifacts.");
        logger.trace("Saving build record {}.", buildRecord);
        buildRecord = buildRecordRepository.save(buildRecord);
        logger.debug("Build record {} saved.", buildRecord.getId());

        logger.trace("Setting artifacts as built.");
        for (Artifact builtArtifact : savedBuiltArtifacts) {
            builtArtifact.setBuildRecord(buildRecord);
        }

        return buildRecord;
    }

    @Override
    @TransactionAttribute(TransactionAttributeType.REQUIRED)
    public BuildRecord storeRecordForNoRebuild(BuildRecord buildRecord) {
        logger.debug("Storing record for not required build {}.", buildRecord);

        buildRecord = buildRecordRepository.save(buildRecord);
        logger.debug("Build record {} saved.", buildRecord.getId());<commit_msg>Set the artifact quality to temporary when temporary build finishes.<commit_after>/**
 * JBoss, Home of Professional Open Source.
 * Copyright 2014-2020 Red Hat, Inc., and individual contributors
 * as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jboss.pnc.datastore;

import org.jboss.pnc.enums.ArtifactQuality;
import org.jboss.pnc.enums.RepositoryType;
import org.jboss.pnc.model.Artifact;
import org.jboss.pnc.model.BuildConfigSetRecord;
import org.jboss.pnc.model.BuildConfiguration;
import org.jboss.pnc.model.BuildConfigurationAudited;
import org.jboss.pnc.model.BuildConfigurationSet;
import org.jboss.pnc.model.BuildRecord;
import org.jboss.pnc.model.IdRev;
import org.jboss.pnc.model.TargetRepository;
import org.jboss.pnc.model.User;
import org.jboss.pnc.spi.coordinator.BuildTask;
import org.jboss.pnc.spi.datastore.Datastore;
import org.jboss.pnc.spi.datastore.repositories.ArtifactRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildConfigSetRecordRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildConfigurationAuditedRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildConfigurationRepository;
import org.jboss.pnc.spi.datastore.repositories.BuildRecordRepository;
import org.jboss.pnc.spi.datastore.repositories.TargetRepositoryRepository;
import org.jboss.pnc.spi.datastore.repositories.UserRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ejb.Stateless;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.inject.Inject;

import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.jboss.pnc.common.util.CollectionUtils.ofNullableCollection;
import static org.jboss.pnc.spi.datastore.predicates.ArtifactPredicates.withIdentifierInAndBuilt;
import static org.jboss.pnc.spi.datastore.predicates.ArtifactPredicates.withSha256InAndBuilt;
import static org.jboss.pnc.spi.datastore.predicates.BuildConfigurationPredicates.withBuildConfigurationSetId;
import static org.jboss.pnc.spi.datastore.predicates.UserPredicates.withUserName;

@Stateless
public class DefaultDatastore implements Datastore {

    public static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

    private ArtifactRepository artifactRepository;

    private BuildRecordRepository buildRecordRepository;

    private BuildConfigurationRepository buildConfigurationRepository;

    private BuildConfigurationAuditedRepository buildConfigurationAuditedRepository;

    private BuildConfigSetRecordRepository buildConfigSetRecordRepository;

    private UserRepository userRepository;

    private TargetRepositoryRepository targetRepositoryRepository;

    public DefaultDatastore() {
    }

    @Inject
    public DefaultDatastore(
            ArtifactRepository artifactRepository,
            BuildRecordRepository buildRecordRepository,
            BuildConfigurationRepository buildConfigurationRepository,
            BuildConfigurationAuditedRepository buildConfigurationAuditedRepository,
            BuildConfigSetRecordRepository buildConfigSetRecordRepository,
            UserRepository userRepository,
            TargetRepositoryRepository targetRepositoryRepository) {
        this.artifactRepository = artifactRepository;
        this.buildRecordRepository = buildRecordRepository;
        this.buildConfigurationRepository = buildConfigurationRepository;
        this.buildConfigurationAuditedRepository = buildConfigurationAuditedRepository;
        this.buildConfigSetRecordRepository = buildConfigSetRecordRepository;
        this.userRepository = userRepository;
        this.targetRepositoryRepository = targetRepositoryRepository;
    }

    private static final String ARTIFACT_ALREADY_BUILT_CONFLICT_MESSAGE = "This artifact was already built in build #";

    @Override
    public Map<Artifact, String> checkForBuiltArtifacts(Collection<Artifact> artifacts) {
        Map<RepositoryType, Map<String, Artifact>> repoTypes = new HashMap<>();
        for (Artifact artifact : artifacts) {
            RepositoryType repoType = artifact.getTargetRepository().getRepositoryType();
            if (!repoTypes.containsKey(repoType)) {
                repoTypes.put(repoType, new HashMap<>());
            }
            Map<String, Artifact> identifiers = repoTypes.get(repoType);
            identifiers.put(artifact.getIdentifier(), artifact);
        }

        Map<Artifact, String> conflicts = new HashMap<>();
        for (RepositoryType repoType : repoTypes.keySet()) {
            Map<String, Artifact> identifiers = repoTypes.get(repoType);
            List<Artifact> conflicting = artifactRepository
                    .queryWithPredicates(withIdentifierInAndBuilt(identifiers.keySet()));
            for (Artifact conflict : conflicting) {
                if (conflict.getTargetRepository().getRepositoryType() == repoType) {
                    Artifact artifact = identifiers.get(conflict.getIdentifier());
                    conflicts
                            .put(artifact, ARTIFACT_ALREADY_BUILT_CONFLICT_MESSAGE + conflict.getBuildRecord().getId());
                }
            }
        }
        return conflicts;
    }

    @Override
    @TransactionAttribute(TransactionAttributeType.REQUIRED)
    public BuildRecord storeCompletedBuild(
            BuildRecord.Builder buildRecordBuilder,
            List<Artifact> builtArtifacts,
            List<Artifact> dependencies) {
        BuildRecord buildRecord = buildRecordBuilder.build(true);
        logger.debug("Storing completed build {}.", buildRecord);
        if (logger.isTraceEnabled()) {
            logger.trace("Build Log: {}.", buildRecord.getBuildLog());
        }

        Map<TargetRepository.IdentifierPath, TargetRepository> repositoriesCache = new HashMap<>();
        Map<Artifact.IdentifierSha256, Artifact> artifactCache = new HashMap<>();

        /**
         * Built artifacts must be saved before the dependencies. In case an artifact is built and the dependency
         * (re-downloaded), it must be linked to built artifacts repository.
         */
        logger.debug("Saving built artifacts ...");
        final Set<Artifact> savedBuiltArtifacts = saveArtifacts(builtArtifacts, repositoriesCache, artifactCache);

        logger.debug("Saving dependencies ...");
        buildRecord.setDependencies(saveArtifacts(dependencies, repositoriesCache, artifactCache));

        logger.debug("Done saving artifacts.");
        logger.trace("Saving build record {}.", buildRecord);
        buildRecord = buildRecordRepository.save(buildRecord);
        logger.debug("Build record {} saved.", buildRecord.getId());

        ArtifactQuality quality = buildRecord.isTemporaryBuild() ? ArtifactQuality.TEMPORARY : ArtifactQuality.NEW;
        logger.trace("Setting artifacts as built and their quality to {}.", quality);
        for (Artifact builtArtifact : savedBuiltArtifacts) {
            builtArtifact.setBuildRecord(buildRecord);
            builtArtifact.setArtifactQuality(quality);
        }

        return buildRecord;
    }

    @Override
    @TransactionAttribute(TransactionAttributeType.REQUIRED)
    public BuildRecord storeRecordForNoRebuild(BuildRecord buildRecord) {
        logger.debug("Storing record for not required build {}.", buildRecord);

        buildRecord = buildRecordRepository.save(buildRecord);
        logger.debug("Build record {} saved.", buildRecord.getId());
{'fraction_non_alphanumeric': 0.08429071696258525, 'fraction_numerical': 0.0024574553050316396, 'mean_word_length': 3.929739551786796, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4045483', 'n_tokens_mistral': 4280, 'n_tokens_neox': 4054, 'n_words': 911}