<commit_before>package de.tum.in.www1.artemis.service.connectors;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.Security;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.validation.constraints.NotNull;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.HiddenFileFilter;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.errors.UnsupportedCredentialItem;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.eclipse.jgit.transport.*;
import org.eclipse.jgit.transport.sshd.JGitKeyCache;
import org.eclipse.jgit.transport.sshd.KeyPasswordProvider;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import de.tum.in.www1.artemis.domain.*;
import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseParticipation;
import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
import de.tum.in.www1.artemis.exception.GitException;
import de.tum.in.www1.artemis.service.ZipFileService;
import de.tum.in.www1.artemis.web.rest.errors.EntityNotFoundException;

@Service
public class GitService {

    private final Logger log = LoggerFactory.getLogger(GitService.class);

    @Value("${artemis.version-control.url}")
    private URL gitUrl;

    @Value("${artemis.version-control.user}")
    private String gitUser;

    @Value("${artemis.version-control.password}")
    private String gitPassword;

    @Value("${artemis.version-control.token:#{null}}")
    private Optional<String> gitToken;

    @Value("${artemis.version-control.ssh-private-key-folder-path:#{null}}")
    private Optional<String> gitSshPrivateKeyPath;

    @Value("${artemis.version-control.ssh-private-key-password:#{null}}")
    private Optional<String> gitSshPrivateKeyPassphrase;

    @Value("${artemis.version-control.ssh-template-clone-url:#{null}}")
    private Optional<String> sshUrlTemplate;

    @Value("${artemis.repo-clone-path}")
    private String repoClonePath;

    @Value("${artemis.git.name}")
    private String artemisGitName;

    @Value("${artemis.git.email}")
    private String artemisGitEmail;

    private final Map<Path, Repository> cachedRepositories = new ConcurrentHashMap<>();

    private final Map<Path, Path> cloneInProgressOperations = new ConcurrentHashMap<>();

    private final ZipFileService zipFileService;

    private TransportConfigCallback sshCallback;

    private static final int JGIT_TIMEOUT_IN_SECONDS = 5;

    public GitService(ZipFileService zipFileService) {
        log.info("file.encoding=" + System.getProperty("file.encoding"));
        log.info("sun.jnu.encoding=" + System.getProperty("sun.jnu.encoding"));
        log.info("Default Charset=" + Charset.defaultCharset());
        log.info("Default Charset in Use=" + new OutputStreamWriter(new ByteArrayOutputStream()).getEncoding());
        this.zipFileService = zipFileService;
    }

    /**
     * initialize the GitService, in particular which authentication mechanism should be used
     * Artemis uses the following order for authentication:
     * 1. ssh key (if available)
     * 2. username + personal access token (if available)
     * 3. username + password
     */
    @PostConstruct
    public void init() {
        if (useSsh()) {
            log.info("GitService will use ssh keys as authentication method to interact with remote git repositories");
            configureSsh();
        }
        else if (gitToken.isPresent()) {
            log.info("GitService will use username + token as authentication method to interact with remote git repositories");
            CredentialsProvider.setDefault(new UsernamePasswordCredentialsProvider(gitUser, gitToken.get()));
        }
        else {
            log.info("GitService will use username + password as authentication method to interact with remote git repositories");
            CredentialsProvider.setDefault(new UsernamePasswordCredentialsProvider(gitUser, gitPassword));
        }
    }

    private void configureSsh() {

        // Note: this is an important statement, because otherwise the git operations over ssh might block, because the native random secure number generator is too slow, see e.g.
        // https://mattnworb.com/post/the-dangers-of-java-security-securerandom
        Security.insertProviderAt(new BouncyCastleProvider(), 1);   // 1 inserts this provider as the default one
<commit_msg>remove BouncyCastleProvider again because then other SSL operations might not work<commit_after>package de.tum.in.www1.artemis.service.connectors;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.validation.constraints.NotNull;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.HiddenFileFilter;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.errors.UnsupportedCredentialItem;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.eclipse.jgit.transport.*;
import org.eclipse.jgit.transport.sshd.JGitKeyCache;
import org.eclipse.jgit.transport.sshd.KeyPasswordProvider;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import de.tum.in.www1.artemis.domain.*;
import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseParticipation;
import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
import de.tum.in.www1.artemis.exception.GitException;
import de.tum.in.www1.artemis.service.ZipFileService;
import de.tum.in.www1.artemis.web.rest.errors.EntityNotFoundException;

@Service
public class GitService {

    private final Logger log = LoggerFactory.getLogger(GitService.class);

    @Value("${artemis.version-control.url}")
    private URL gitUrl;

    @Value("${artemis.version-control.user}")
    private String gitUser;

    @Value("${artemis.version-control.password}")
    private String gitPassword;

    @Value("${artemis.version-control.token:#{null}}")
    private Optional<String> gitToken;

    @Value("${artemis.version-control.ssh-private-key-folder-path:#{null}}")
    private Optional<String> gitSshPrivateKeyPath;

    @Value("${artemis.version-control.ssh-private-key-password:#{null}}")
    private Optional<String> gitSshPrivateKeyPassphrase;

    @Value("${artemis.version-control.ssh-template-clone-url:#{null}}")
    private Optional<String> sshUrlTemplate;

    @Value("${artemis.repo-clone-path}")
    private String repoClonePath;

    @Value("${artemis.git.name}")
    private String artemisGitName;

    @Value("${artemis.git.email}")
    private String artemisGitEmail;

    private final Map<Path, Repository> cachedRepositories = new ConcurrentHashMap<>();

    private final Map<Path, Path> cloneInProgressOperations = new ConcurrentHashMap<>();

    private final ZipFileService zipFileService;

    private TransportConfigCallback sshCallback;

    private static final int JGIT_TIMEOUT_IN_SECONDS = 5;

    public GitService(ZipFileService zipFileService) {
        log.info("file.encoding=" + System.getProperty("file.encoding"));
        log.info("sun.jnu.encoding=" + System.getProperty("sun.jnu.encoding"));
        log.info("Default Charset=" + Charset.defaultCharset());
        log.info("Default Charset in Use=" + new OutputStreamWriter(new ByteArrayOutputStream()).getEncoding());
        this.zipFileService = zipFileService;
    }

    /**
     * initialize the GitService, in particular which authentication mechanism should be used
     * Artemis uses the following order for authentication:
     * 1. ssh key (if available)
     * 2. username + personal access token (if available)
     * 3. username + password
     */
    @PostConstruct
    public void init() {
        if (useSsh()) {
            log.info("GitService will use ssh keys as authentication method to interact with remote git repositories");
            configureSsh();
        }
        else if (gitToken.isPresent()) {
            log.info("GitService will use username + token as authentication method to interact with remote git repositories");
            CredentialsProvider.setDefault(new UsernamePasswordCredentialsProvider(gitUser, gitToken.get()));
        }
        else {
            log.info("GitService will use username + password as authentication method to interact with remote git repositories");
            CredentialsProvider.setDefault(new UsernamePasswordCredentialsProvider(gitUser, gitPassword));
        }
    }

    private void configureSsh() {

        // Note: this is an important statement, because otherwise the git operations over ssh might block, because the native random secure number generator is too slow, see e.g.
        // https://mattnworb.com/post/the-dangers-of-java-security-securerandom
        // Security.insertProviderAt(new BouncyCastleProvider(), 1); // 1 inserts this provider as the default one

{'fraction_non_alphanumeric': 0.10984230560087004, 'fraction_numerical': 0.0027188689505165853, 'mean_word_length': 4.435960591133005, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3634955', 'n_tokens_mistral': 3267, 'n_tokens_neox': 3124, 'n_words': 664}