<commit_before>package com.gh4a.utils;

import android.content.Context;
import android.graphics.Color;
import android.net.Uri;
import android.text.TextUtils;

import com.gh4a.ApiRequestException;
import com.gh4a.Gh4Application;
import com.gh4a.R;
import com.meisolsson.githubsdk.model.Commit;
import com.meisolsson.githubsdk.model.GitHubCommentBase;
import com.meisolsson.githubsdk.model.Label;
import com.meisolsson.githubsdk.model.Page;
import com.meisolsson.githubsdk.model.User;
import com.meisolsson.githubsdk.model.git.GitUser;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;

import retrofit2.Response;

public class ApiHelpers {
    public interface IssueState {
        String OPEN = "open";
        String CLOSED = "closed";
        String MERGED = "merged";
        String UNMERGED = "unmerged";
    }

    public static final Comparator<GitHubCommentBase> COMMENT_COMPARATOR = new Comparator<GitHubCommentBase>() {
        @Override
        public int compare(GitHubCommentBase lhs, GitHubCommentBase rhs) {
            if (lhs.createdAt() == null) {
                return 1;
            }
            if (rhs.createdAt() == null) {
                return -1;
            }
            return lhs.createdAt().compareTo(rhs.createdAt());
        }
    };

    //RepositoryCommit
    public static String getAuthorName(Context context, Commit commit) {
        if (commit.author() != null) {
            return commit.author().login();
        }
        if (commit.commit().author() != null) {
            return commit.commit().author().name();
        }
        return context.getString(R.string.unknown);
    }

    public static String getAuthorLogin(Commit commit) {
        if (commit.author() != null) {
            return commit.author().login();
        }
        return null;
    }

    public static String getCommitterName(Context context, Commit commit) {
        if (commit.committer() != null) {
            return commit.committer().login();
        }
        if (commit.commit().committer() != null) {
            return commit.commit().committer().name();
        }
        return context.getString(R.string.unknown);
    }

    public static boolean authorEqualsCommitter(Commit commit) {
        if (commit.committer() != null && commit.author() != null) {
            return TextUtils.equals(commit.committer().login(), commit.author().login());
        }

        GitUser author = commit.commit().author();
        GitUser committer = commit.commit().committer();
        if (author.email() != null && committer.email() != null) {
            return TextUtils.equals(author.email(), committer.email());
        }
        return TextUtils.equals(author.name(), committer.name());
    }

    public static String getUserLogin(Context context, User user) {
        if (user != null && user.login() != null) {
            return user.login();
        }
        return context.getString(R.string.unknown);
    }

    public static int colorForLabel(Label label) {
        return Color.parseColor("#" + label.color());
    }

    public static boolean userEquals(User lhs, User rhs) {
        if (lhs == null || rhs == null) {
            return false;
        }
        return loginEquals(lhs.login(), rhs.login());
    }

    public static boolean loginEquals(User user, String login) {
        if (user == null) {
            return false;
        }
        return loginEquals(user.login(), login);
    }

    public static boolean loginEquals(String user, String login) {
        return user != null && user.equalsIgnoreCase(login);
    }

    public static Uri normalizeUri(Uri uri) {
        if (uri == null || uri.getAuthority() == null) {
            return uri;
        }

        // Only normalize API links
        if (!uri.getPath().contains("/api/v3/") && !uri.getAuthority().contains("api.")) {
            return uri;
        }

        String path = uri.getPath()
                .replace("/api/v3/", "/")
                .replace("repos/", "")
                .replace("commits/", "commit/")
                .replace("pulls/", "pull/");

        String authority = uri.getAuthority()
                .replace("api.", "");

        return uri.buildUpon()
                .path(path)
                .authority(authority)
                .build();
    }

    public static String md5(String input) {
        try {
            MessageDigest digest = MessageDigest.getInstance("MD5");
            digest.update(input.getBytes());
            byte[] messageDigest = digest.digest();

            StringBuilder builder = new StringBuilder();
            for (byte b : messageDigest) {
                String hexString = Integer.toHexString(0xFF & b);
                while (hexString.length() < 2) {
                    hexString = "0" + hexString;
                }
                builder.append(hexString);
            }
            return builder.toString();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
        return "";
    }

    public static <T> T throwOnFailure(Response<T> response) throws IOException {
        if (response.code() == HttpURLConnection.HTTP_UNAUTHORIZED) {
            Gh4Application.get().logout();
        }
        if (!response.isSuccessful()) {
            throw new ApiRequestException(response);
        }
        return response.body();
    }

    public static class Pager<T> {
        public interface PageProvider<T> {
            Page<T> providePage(long page) throws IOException;
        }

        public static <T> List<T> fetchAllPages(PageProvider<T> provider) throws IOException {
            List<T> result = new ArrayList<>();
            int nextPage = 1;
            do {
                Page<T> page = provider.providePage(nextPage);
                result.addAll(page.items());
                nextPage = page.next() != null ? page.next() : 0;
            } while (nextPage > 0);
            return result;
        }
    }
}<commit_msg>Add a helper to deal with API pagination.<commit_after>package com.gh4a.utils;

import android.content.Context;
import android.graphics.Color;
import android.net.Uri;
import android.text.TextUtils;

import com.gh4a.ApiRequestException;
import com.gh4a.Gh4Application;
import com.gh4a.R;
import com.meisolsson.githubsdk.model.Commit;
import com.meisolsson.githubsdk.model.GitHubCommentBase;
import com.meisolsson.githubsdk.model.Label;
import com.meisolsson.githubsdk.model.Page;
import com.meisolsson.githubsdk.model.User;
import com.meisolsson.githubsdk.model.git.GitUser;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;

import io.reactivex.Observable;
import io.reactivex.Single;
import retrofit2.Response;

public class ApiHelpers {
    public interface IssueState {
        String OPEN = "open";
        String CLOSED = "closed";
        String MERGED = "merged";
        String UNMERGED = "unmerged";
    }

    public static final Comparator<GitHubCommentBase> COMMENT_COMPARATOR = new Comparator<GitHubCommentBase>() {
        @Override
        public int compare(GitHubCommentBase lhs, GitHubCommentBase rhs) {
            if (lhs.createdAt() == null) {
                return 1;
            }
            if (rhs.createdAt() == null) {
                return -1;
            }
            return lhs.createdAt().compareTo(rhs.createdAt());
        }
    };

    //RepositoryCommit
    public static String getAuthorName(Context context, Commit commit) {
        if (commit.author() != null) {
            return commit.author().login();
        }
        if (commit.commit().author() != null) {
            return commit.commit().author().name();
        }
        return context.getString(R.string.unknown);
    }

    public static String getAuthorLogin(Commit commit) {
        if (commit.author() != null) {
            return commit.author().login();
        }
        return null;
    }

    public static String getCommitterName(Context context, Commit commit) {
        if (commit.committer() != null) {
            return commit.committer().login();
        }
        if (commit.commit().committer() != null) {
            return commit.commit().committer().name();
        }
        return context.getString(R.string.unknown);
    }

    public static boolean authorEqualsCommitter(Commit commit) {
        if (commit.committer() != null && commit.author() != null) {
            return TextUtils.equals(commit.committer().login(), commit.author().login());
        }

        GitUser author = commit.commit().author();
        GitUser committer = commit.commit().committer();
        if (author.email() != null && committer.email() != null) {
            return TextUtils.equals(author.email(), committer.email());
        }
        return TextUtils.equals(author.name(), committer.name());
    }

    public static String getUserLogin(Context context, User user) {
        if (user != null && user.login() != null) {
            return user.login();
        }
        return context.getString(R.string.unknown);
    }

    public static int colorForLabel(Label label) {
        return Color.parseColor("#" + label.color());
    }

    public static boolean userEquals(User lhs, User rhs) {
        if (lhs == null || rhs == null) {
            return false;
        }
        return loginEquals(lhs.login(), rhs.login());
    }

    public static boolean loginEquals(User user, String login) {
        if (user == null) {
            return false;
        }
        return loginEquals(user.login(), login);
    }

    public static boolean loginEquals(String user, String login) {
        return user != null && user.equalsIgnoreCase(login);
    }

    public static Uri normalizeUri(Uri uri) {
        if (uri == null || uri.getAuthority() == null) {
            return uri;
        }

        // Only normalize API links
        if (!uri.getPath().contains("/api/v3/") && !uri.getAuthority().contains("api.")) {
            return uri;
        }

        String path = uri.getPath()
                .replace("/api/v3/", "/")
                .replace("repos/", "")
                .replace("commits/", "commit/")
                .replace("pulls/", "pull/");

        String authority = uri.getAuthority()
                .replace("api.", "");

        return uri.buildUpon()
                .path(path)
                .authority(authority)
                .build();
    }

    public static String md5(String input) {
        try {
            MessageDigest digest = MessageDigest.getInstance("MD5");
            digest.update(input.getBytes());
            byte[] messageDigest = digest.digest();

            StringBuilder builder = new StringBuilder();
            for (byte b : messageDigest) {
                String hexString = Integer.toHexString(0xFF & b);
                while (hexString.length() < 2) {
                    hexString = "0" + hexString;
                }
                builder.append(hexString);
            }
            return builder.toString();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
        return "";
    }

    public static <T> T throwOnFailure(Response<T> response) throws IOException {
        if (response.code() == HttpURLConnection.HTTP_UNAUTHORIZED) {
            Gh4Application.get().logout();
        }
        if (!response.isSuccessful()) {
            throw new ApiRequestException(response);
        }
        return response.body();
    }

    public static class Pager<T> {
        public interface PageProvider<T> {
            Page<T> providePage(long page) throws IOException;
        }

        public static <T> List<T> fetchAllPages(PageProvider<T> provider) throws IOException {
            List<T> result = new ArrayList<>();
            int nextPage = 1;
            do {
                Page<T> page = provider.providePage(nextPage);
                result.addAll(page.items());
                nextPage = page.next() != null ? page.next() : 0;
            } while (nextPage > 0);
            return result;
        }
    }

    public static class PageIterator<T> {
        public interface PageProducer<T> {
            Single<Response<Page<T>>> getPage(long page);
        }

        public static <T> Observable<List<T>> toObservable(PageProducer<T> producer) {
            return pageToObservable(producer, 1);
        }

        public static <T> Single<List<T>> toSingle(PageProducer<T> producer) {
            return toObservable(producer)
                    .toList()
                    .map(lists -> {
                        List<T> result = new ArrayList<T>();
                        for (List<T> l : lists) {
                            result.addAll(l);
                        }
                        return result;
                    });
        }

        private static <T> Observable<List<T>> pageToObservable(PageProducer<T> producer, int page) {
            return producer.getPage(page)
                    .toObservable()
                    .compose(result -> evaluateError(result))
                    .compose(result -> chain(producer, result));
        }

        private static <T> Observable<Page<T>> evaluateError(Observable<Response<Page<T>>> input) {
            return input.map(response -> {
                throwOnFailure(response);
                return response.body();
            });
        }

        private static <T> Observable<List<T>> chain(PageProducer<T> producer, Observable<Page<T>> input) {
            return input.concatMap(page -> {
                if (page.next() == null) {
                    return Observable.empty();
                }
                return Observable.just(page.items())
                        .concatWith(pageToObservable(producer, page.next()));
            });
        }
    }
}
{'fraction_non_alphanumeric': 0.12249663859599462, 'fraction_numerical': 0.0027598896044158236, 'mean_word_length': 1.9953370072064434, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '668195', 'n_tokens_mistral': 3874, 'n_tokens_neox': 3734, 'n_words': 847}