<commit_before>package fr.inria.spirals.repairnator.process;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import fr.inria.spirals.jtravis.entities.Build;
import fr.inria.spirals.jtravis.entities.BuildStatus;
import fr.inria.spirals.jtravis.entities.Job;
import fr.inria.spirals.jtravis.entities.Log;
import fr.inria.spirals.jtravis.entities.Repository;
import fr.inria.spirals.jtravis.entities.TestsInformation;
import fr.inria.spirals.jtravis.helpers.BuildHelper;
import fr.inria.spirals.jtravis.helpers.RepositoryHelper;
import fr.inria.spirals.repairnator.process.step.AbstractStep;

/**
 * This class aims to provide utility methods to scan the projects and get
 * failing builds
 *
 * @author <NAME>
 */
public class ProjectScanner {
    private final Logger logger = LoggerFactory.getLogger(ProjectScanner.class);
    private int totalRepoNumber;
    private int totalRepoUsingTravis;
    private int totalScannedBuilds;
    private int totalPRBuilds;
    private int totalBuildInJava;
    private int totalJavaPassingBuilds;
    private int totalBuildInJavaFailing;
    private int totalBuildInJavaFailingWithFailingTests;
    private Date dateStart;
    private Date dateFinish;

    private Collection<String> slugs;
    private Collection<Repository> repositories;
    private Collection<Integer> buildsId;
    private Date limitDate;

    public ProjectScanner(int lookupHours) {
        this.slugs = new HashSet<String>();
        this.repositories = new HashSet<Repository>();

        Calendar limitCal = Calendar.getInstance();
        limitCal.add(Calendar.HOUR_OF_DAY, -lookupHours);
        this.limitDate = limitCal.getTime();
    }

    public int getTotalPRBuilds() {
        return totalPRBuilds;
    }

    public int getTotalBuildInJava() {
        return totalBuildInJava;
    }

    public int getTotalBuildInJavaFailing() {
        return totalBuildInJavaFailing;
    }

    public Date getLimitDate() {
        return limitDate;
    }

    public int getTotalJavaPassingBuilds() {
        return totalJavaPassingBuilds;
    }

    public int getTotalRepoNumber() {
        return totalRepoNumber;
    }

    public int getTotalRepoUsingTravis() {
        return totalRepoUsingTravis;
    }

    public int getTotalBuildInJavaFailingWithFailingTests() {
        return totalBuildInJavaFailingWithFailingTests;
    }

    public int getTotalScannedBuilds() {
        return totalScannedBuilds;
    }

    public Collection<String> getSlugs() {
        return slugs;
    }

    public Collection<Repository> getRepositories() {
        return repositories;
    }

    public List<Build> getListOfFailingBuildFromGivenBuildIds(String path) throws IOException {
        this.dateStart = new Date();
        List<String> buildsIds = getFileContent(path);
        this.totalScannedBuilds = buildsIds.size();

        this.buildsId = new ArrayList<Integer>();

        List<Build> result = new ArrayList<Build>();

        for (String s : buildsIds) {
            int buildId;
            try {
                buildId = Integer.parseInt(s);
            } catch (NumberFormatException e) {
                this.logger.error("Error while reading build ids from input: " + e.getMessage());
                continue;
            }

            Build build = BuildHelper.getBuildFromId(buildId, null);
            if (build == null) {
                this.logger.warn("The following build cannot be retrieved: " + buildId);
                continue;
            }
            if (testBuild(build, true)) {
                result.add(build);
                this.buildsId.add(build.getId());
            }
        }

        this.totalRepoNumber = this.repositories.size();
        this.totalRepoUsingTravis = this.repositories.size();
        this.totalBuildInJavaFailingWithFailingTests = result.size();
        this.dateFinish = new Date();

        return result;
    }

    /**
     * Take a filepath as input containing a list of projects to scan. Check
     * last build of each project. And finally returns the list of failing
     * builds.
     *
     * @param path
     *            A path to a file formatted to contain a slug name of project
     *            per line (ex: INRIA/spoon)
     * @return a list of failing builds
     * @throws IOException
     */
    public List<Build> getListOfFailingBuildFromProjects(String path) throws IOException {
        return getListOfBuildsFromProjectsByBuildStatus(path, true);
    }

    public List<Build> getListOfPassingBuildsFromProjects(String path) throws IOException {
        return getListOfBuildsFromProjectsByBuildStatus(path, false);
    }

    private List<Build> getListOfBuildsFromProjectsByBuildStatus(String path, boolean targetFailing)
            throws IOException {
        this.dateStart = new Date();
        List<String> slugs = getFileContent(path);
        this.totalRepoNumber = slugs.size();

        List<Repository> repos = getListOfValidRepository(slugs);
        List<Build> builds = getListOfBuildsFromRepo(repos, targetFailing);

        this.dateFinish = new Date();
        return builds;
    }

    private List<String> getFileContent(String path) throws IOException {
        List<String> result = new ArrayList<String>();
        File file = new File(path);
        BufferedReader reader = new BufferedReader(new FileReader(file));
        while (reader.ready()) {
            result.add(reader.readLine().trim());
        }
        return result;
    }

    private List<Repository> getListOfValidRepository(List<String> allSlugs) {
        List<Repository> result = new ArrayList<Repository>();

        for (String slug : allSlugs) {
            this.logger.debug("Get repo " + slug);
            Repository repo = RepositoryHelper.getRepositoryFromSlug(slug);
            if (repo != null) {
                Build lastBuild = repo.getLastBuild(false);
                if (lastBuild != null) {
                    result.add(repo);
                } else {
                    this.logger.info("It seems that the repo " + slug + " does not have any Travis build.");
                }

            } else {
                this.logger.warn("Can't examine repo : " + slug);
            }
        }

        this.totalRepoUsingTravis = result.size();
        return result;
    }

    private boolean testBuild(Build build, boolean targetFailing) {
        if (build.isPullRequest()) {
            this.totalPRBuilds++;
        }

        Repository repo = build.getRepository();
        if (build.getConfig().getLanguage().equals("java")) {
            this.totalBuildInJava++;

            this.logger.debug("Repo " + repo.getSlug() + " with java language - build " + build.getId() + " - Status : "
                    + build.getBuildStatus().name());
            if (build.getBuildStatus() == BuildStatus.FAILED) {
                this.totalBuildInJavaFailing++;

                for (Job job : build.getJobs()) {
                    Log jobLog = job.getLog();

                    if (jobLog != null) {
                        TestsInformation testInfo = jobLog.getTestsInformation();

                        if (testInfo.getFailing() > 0) {
                            this.totalBuildInJavaFailingWithFailingTests++;
                            if (targetFailing) {
                                this.slugs.add(repo.getSlug());
                                this.repositories.add(repo);
                                return true;
                            }
                        }<commit_msg>RepairNator: fix a bug if the build tool is unknown<commit_after>package fr.inria.spirals.repairnator.process;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import fr.inria.spirals.jtravis.entities.Build;
import fr.inria.spirals.jtravis.entities.BuildStatus;
import fr.inria.spirals.jtravis.entities.Job;
import fr.inria.spirals.jtravis.entities.Log;
import fr.inria.spirals.jtravis.entities.Repository;
import fr.inria.spirals.jtravis.entities.TestsInformation;
import fr.inria.spirals.jtravis.helpers.BuildHelper;
import fr.inria.spirals.jtravis.helpers.RepositoryHelper;
import fr.inria.spirals.repairnator.process.step.AbstractStep;

/**
 * This class aims to provide utility methods to scan the projects and get
 * failing builds
 *
 * @author <NAME>
 */
public class ProjectScanner {
    private final Logger logger = LoggerFactory.getLogger(ProjectScanner.class);
    private int totalRepoNumber;
    private int totalRepoUsingTravis;
    private int totalScannedBuilds;
    private int totalPRBuilds;
    private int totalBuildInJava;
    private int totalJavaPassingBuilds;
    private int totalBuildInJavaFailing;
    private int totalBuildInJavaFailingWithFailingTests;
    private Date dateStart;
    private Date dateFinish;

    private Collection<String> slugs;
    private Collection<Repository> repositories;
    private Collection<Integer> buildsId;
    private Date limitDate;

    public ProjectScanner(int lookupHours) {
        this.slugs = new HashSet<String>();
        this.repositories = new HashSet<Repository>();

        Calendar limitCal = Calendar.getInstance();
        limitCal.add(Calendar.HOUR_OF_DAY, -lookupHours);
        this.limitDate = limitCal.getTime();
    }

    public int getTotalPRBuilds() {
        return totalPRBuilds;
    }

    public int getTotalBuildInJava() {
        return totalBuildInJava;
    }

    public int getTotalBuildInJavaFailing() {
        return totalBuildInJavaFailing;
    }

    public Date getLimitDate() {
        return limitDate;
    }

    public int getTotalJavaPassingBuilds() {
        return totalJavaPassingBuilds;
    }

    public int getTotalRepoNumber() {
        return totalRepoNumber;
    }

    public int getTotalRepoUsingTravis() {
        return totalRepoUsingTravis;
    }

    public int getTotalBuildInJavaFailingWithFailingTests() {
        return totalBuildInJavaFailingWithFailingTests;
    }

    public int getTotalScannedBuilds() {
        return totalScannedBuilds;
    }

    public Collection<String> getSlugs() {
        return slugs;
    }

    public Collection<Repository> getRepositories() {
        return repositories;
    }

    public List<Build> getListOfFailingBuildFromGivenBuildIds(String path) throws IOException {
        this.dateStart = new Date();
        List<String> buildsIds = getFileContent(path);
        this.totalScannedBuilds = buildsIds.size();

        this.buildsId = new ArrayList<Integer>();

        List<Build> result = new ArrayList<Build>();

        for (String s : buildsIds) {
            int buildId;
            try {
                buildId = Integer.parseInt(s);
            } catch (NumberFormatException e) {
                this.logger.error("Error while reading build ids from input: " + e.getMessage());
                continue;
            }

            Build build = BuildHelper.getBuildFromId(buildId, null);
            if (build == null) {
                this.logger.warn("The following build cannot be retrieved: " + buildId);
                continue;
            }
            if (testBuild(build, true)) {
                result.add(build);
                this.buildsId.add(build.getId());
            }
        }

        this.totalRepoNumber = this.repositories.size();
        this.totalRepoUsingTravis = this.repositories.size();
        this.totalBuildInJavaFailingWithFailingTests = result.size();
        this.dateFinish = new Date();

        return result;
    }

    /**
     * Take a filepath as input containing a list of projects to scan. Check
     * last build of each project. And finally returns the list of failing
     * builds.
     *
     * @param path
     *            A path to a file formatted to contain a slug name of project
     *            per line (ex: INRIA/spoon)
     * @return a list of failing builds
     * @throws IOException
     */
    public List<Build> getListOfFailingBuildFromProjects(String path) throws IOException {
        return getListOfBuildsFromProjectsByBuildStatus(path, true);
    }

    public List<Build> getListOfPassingBuildsFromProjects(String path) throws IOException {
        return getListOfBuildsFromProjectsByBuildStatus(path, false);
    }

    private List<Build> getListOfBuildsFromProjectsByBuildStatus(String path, boolean targetFailing)
            throws IOException {
        this.dateStart = new Date();
        List<String> slugs = getFileContent(path);
        this.totalRepoNumber = slugs.size();

        List<Repository> repos = getListOfValidRepository(slugs);
        List<Build> builds = getListOfBuildsFromRepo(repos, targetFailing);

        this.dateFinish = new Date();
        return builds;
    }

    private List<String> getFileContent(String path) throws IOException {
        List<String> result = new ArrayList<String>();
        File file = new File(path);
        BufferedReader reader = new BufferedReader(new FileReader(file));
        while (reader.ready()) {
            result.add(reader.readLine().trim());
        }
        return result;
    }

    private List<Repository> getListOfValidRepository(List<String> allSlugs) {
        List<Repository> result = new ArrayList<Repository>();

        for (String slug : allSlugs) {
            this.logger.debug("Get repo " + slug);
            Repository repo = RepositoryHelper.getRepositoryFromSlug(slug);
            if (repo != null) {
                Build lastBuild = repo.getLastBuild(false);
                if (lastBuild != null) {
                    result.add(repo);
                } else {
                    this.logger.info("It seems that the repo " + slug + " does not have any Travis build.");
                }

            } else {
                this.logger.warn("Can't examine repo : " + slug);
            }
        }

        this.totalRepoUsingTravis = result.size();
        return result;
    }

    private boolean testBuild(Build build, boolean targetFailing) {
        if (build.isPullRequest()) {
            this.totalPRBuilds++;
        }

        Repository repo = build.getRepository();
        if (build.getConfig().getLanguage().equals("java")) {
            this.totalBuildInJava++;

            this.logger.debug("Repo " + repo.getSlug() + " with java language - build " + build.getId() + " - Status : "
                    + build.getBuildStatus().name());
            if (build.getBuildStatus() == BuildStatus.FAILED) {
                this.totalBuildInJavaFailing++;

                for (Job job : build.getJobs()) {
                    Log jobLog = job.getLog();

                    if (jobLog != null) {
                        TestsInformation testInfo = jobLog.getTestsInformation();

                        // testInfo can be null if the build tool is unknown
                        if (testInfo != null && testInfo.getFailing() > 0) {
                            this.totalBuildInJavaFailingWithFailingTests++;
                            if (targetFailing) {
                                this.slugs.add(repo.getSlug());
                                this.repositories.add(repo);
                                return true;
                            }
                        }
{'fraction_non_alphanumeric': 0.09193740966505373, 'fraction_numerical': 0.0003770502105197009, 'mean_word_length': 2.3216447505739928, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5656152', 'n_tokens_mistral': 4386, 'n_tokens_neox': 4263, 'n_words': 993}