<commit_before>package org.mockito.release.notes.contributors;

import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.json.simple.DeserializationException;
import org.json.simple.JsonObject;
import org.mockito.release.notes.model.ProjectContributor;
import org.mockito.release.notes.util.GitHubListFetcher;
import org.mockito.release.notes.util.GitHubObjectFetcher;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

public class GitHubAllContributorsFetcher {

    private static final Logger LOG = Logging.getLogger(GitHubAllContributorsFetcher.class);

    ProjectContributorsSet fetchAllContributorsForProject(String repository, String readOnlyAuthToken) {
        LOG.lifecycle("  Querying GitHub API for all contributors for project");
        ProjectContributorsSet result = new DefaultProjectContributorsSet();

        try {
            GitHubProjectContributors contributors =
                    GitHubProjectContributors.authenticatingWith(repository, readOnlyAuthToken).build();

            while(contributors.hasNextPage()) {
                List<JsonObject> page = contributors.nextPage();
                result.addAllContributors(extractContributors(page, readOnlyAuthToken));
            }
        } catch (Exception e) {
            //TODO we need to eliminate below from the codebase here and everywhere, else to
            //Let's throw an exception!!!
            e.printStackTrace();
        }
        return result;
    }

    private Set<ProjectContributor> extractContributors(List<JsonObject> page, String readOnlyAuthToken) throws IOException, DeserializationException {
        Set<ProjectContributor> result = new HashSet<ProjectContributor>();
        for (JsonObject contributor : page) {
            String url = (String) contributor.get("url");
            GitHubObjectFetcher userFetcher = new GitHubObjectFetcher(url, readOnlyAuthToken);
            JsonObject user = userFetcher.getPage();
            result.add(GitHubAllContributorsJson.toContributor(contributor, user));
        }
        return result;
    }

    private static class GitHubProjectContributors {
        private final GitHubListFetcher fetcher;
        private List<JsonObject> lastFetchedPage;

        static GitHubProjectContributorsBuilder authenticatingWith(String repository, String readOnlyAuthToken) {
            return new GitHubProjectContributorsBuilder(repository, readOnlyAuthToken);
        }

        private GitHubProjectContributors(String nextPageUrl) {
            fetcher = new GitHubListFetcher(nextPageUrl);
        }

        public boolean hasNextPage() {
            return fetcher.hasNextPage();
        }

        public List<JsonObject> nextPage() throws IOException, DeserializationException {
            lastFetchedPage = fetcher.nextPage();
            return lastFetchedPage;
        }
    }

    private static class GitHubProjectContributorsBuilder {

        private final String repository;
        private final String readOnlyAuthToken;

        public GitHubProjectContributorsBuilder(String repository, String readOnlyAuthToken) {
            this.repository = repository;
            this.readOnlyAuthToken = readOnlyAuthToken;
        }

        GitHubProjectContributors build() {
            // see API doc: https://developer.github.com/v3/repos/#list-contributors
            String nextPageUrl = "https://api.github.com/repos/" + repository + "/contributors" +
                    "?access_token=" + readOnlyAuthToken +
                    "&per_page=100";
            return new GitHubProjectContributors(nextPageUrl);
        }
    }
}
<commit_msg>Added documentation and a TODO<commit_after>package org.mockito.release.notes.contributors;

import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.json.simple.DeserializationException;
import org.json.simple.JsonObject;
import org.mockito.release.notes.model.ProjectContributor;
import org.mockito.release.notes.util.GitHubListFetcher;
import org.mockito.release.notes.util.GitHubObjectFetcher;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

public class GitHubAllContributorsFetcher {

    private static final Logger LOG = Logging.getLogger(GitHubAllContributorsFetcher.class);

    ProjectContributorsSet fetchAllContributorsForProject(String repository, String readOnlyAuthToken) {
        LOG.lifecycle("  Querying GitHub API for all contributors for project");
        ProjectContributorsSet result = new DefaultProjectContributorsSet();

        try {
            GitHubProjectContributors contributors =
                    GitHubProjectContributors.authenticatingWith(repository, readOnlyAuthToken).build();

            while(contributors.hasNextPage()) {
                List<JsonObject> page = contributors.nextPage();
                result.addAllContributors(extractContributors(page, readOnlyAuthToken));
            }
        } catch (Exception e) {
            //TODO we need to eliminate below from the codebase here and everywhere, else to
            //Let's throw an exception!!!
            e.printStackTrace();
        }
        return result;
    }

    private Set<ProjectContributor> extractContributors(List<JsonObject> page, String readOnlyAuthToken) throws IOException, DeserializationException {
        Set<ProjectContributor> result = new HashSet<ProjectContributor>();
        for (JsonObject contributor : page) {
            //Since returned contributor does not have 'name' element, we need to fetch the user data to get his name
            //TODO add static caching of this. Names don't change that often, let's just cache this forever in build cache.
            String url = (String) contributor.get("url");
            GitHubObjectFetcher userFetcher = new GitHubObjectFetcher(url, readOnlyAuthToken);
            JsonObject user = userFetcher.getPage();
            result.add(GitHubAllContributorsJson.toContributor(contributor, user));
        }
        return result;
    }

    private static class GitHubProjectContributors {
        private final GitHubListFetcher fetcher;
        private List<JsonObject> lastFetchedPage;

        static GitHubProjectContributorsBuilder authenticatingWith(String repository, String readOnlyAuthToken) {
            return new GitHubProjectContributorsBuilder(repository, readOnlyAuthToken);
        }

        private GitHubProjectContributors(String nextPageUrl) {
            fetcher = new GitHubListFetcher(nextPageUrl);
        }

        public boolean hasNextPage() {
            return fetcher.hasNextPage();
        }

        public List<JsonObject> nextPage() throws IOException, DeserializationException {
            lastFetchedPage = fetcher.nextPage();
            return lastFetchedPage;
        }
    }

    private static class GitHubProjectContributorsBuilder {

        private final String repository;
        private final String readOnlyAuthToken;

        public GitHubProjectContributorsBuilder(String repository, String readOnlyAuthToken) {
            this.repository = repository;
            this.readOnlyAuthToken = readOnlyAuthToken;
        }

        GitHubProjectContributors build() {
            // see API doc: https://developer.github.com/v3/repos/#list-contributors
            String nextPageUrl = "https://api.github.com/repos/" + repository + "/contributors" +
                    "?access_token=" + readOnlyAuthToken +
                    "&per_page=100";
            return new GitHubProjectContributors(nextPageUrl);
        }
    }
}

{'fraction_non_alphanumeric': 0.07640978673295826, 'fraction_numerical': 0.0010467094073007982, 'mean_word_length': 2.9874804381846634, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4744775', 'n_tokens_mistral': 1995, 'n_tokens_neox': 1861, 'n_words': 440}