<commit_before>package org.kohsuke.github;

import com.fasterxml.jackson.databind.exc.MismatchedInputException;
import com.fasterxml.jackson.databind.exc.ValueInstantiationException;
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
import org.hamcrest.CoreMatchers;
import org.junit.Test;

import java.io.IOException;
import java.time.Duration;
import java.util.Date;

import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.core.IsInstanceOf.instanceOf;

/**
 * Test showing the behavior of OkHttpConnector with and without cache.
 * <p>
 * Key take aways:
 *
 * <ul>
 * <li>These tests are artificial and intended to highlight the differences in behavior between scenarios. However, the
 * differences they indicate are stark.</li>
 * <li>Caching reduces rate limit consumption by at least a factor of two in even the simplest case.</li>
 * <li>The OkHttp cache is pretty smart and will often connect read and write requests made on the same client and
 * invalidate caches.</li>
 * <li>Changes made outside the current client cause the OkHttp cache to return stale data. This is expected and correct
 * behavior.</li>
 * <li>"max-age=0" addresses the problem of external changes by revalidating caches for each request. This produces the
 * same number of requests as OkHttp without caching, but those requests only count towards the GitHub rate limit if
 * data has changes.</li>
 * </ul>
 *
 * @author <NAME>
 */
public class GHRateLimitTest extends AbstractGitHubWireMockTest {

    GHRateLimit rateLimit = null;
    GHRateLimit previousLimit = null;

    public GHRateLimitTest() {
        useDefaultGitHub = false;
    }

    @Override
    protected WireMockConfiguration getWireMockOptions() {
        return super.getWireMockOptions().extensions(templating.newResponseTransformer());
    }

    @Test
    public void testGitHubRateLimit() throws Exception {
        // Customized response that templates the date to keep things working
        snapshotNotAllowed();

        assertThat(mockGitHub.getRequestCount(), equalTo(0));

        // 4897 is just the what the limit was when the snapshot was taken
        previousLimit = GHRateLimit.fromHeaderRecord(new GHRateLimit.Record(5000,
                4897,
                (templating.testStartDate.getTime() + Duration.ofHours(1).toMillis()) / 1000L));

        // -------------------------------------------------------------
        // /user gets response with rate limit information
        gitHub = getGitHubBuilder().withEndpoint(mockGitHub.apiServer().baseUrl()).build();
        gitHub.getMyself();

        assertThat(mockGitHub.getRequestCount(), equalTo(1));

        // Since we already had rate limit info these don't request again
        rateLimit = gitHub.lastRateLimit();
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining());
        previousLimit = rateLimit;

        GHRateLimit headerRateLimit = rateLimit;

        // Give this a moment
        Thread.sleep(1000);

        // ratelimit() uses headerRateLimit if available and headerRateLimit is not expired
        assertThat(gitHub.rateLimit(), equalTo(headerRateLimit));

        assertThat(mockGitHub.getRequestCount(), equalTo(1));

        // Give this a moment
        Thread.sleep(1000);

        // Always requests new info
        rateLimit = gitHub.getRateLimit();
        assertThat(mockGitHub.getRequestCount(), equalTo(2));

        // Because remaining and reset date are unchanged, the header should be unchanged as well
        assertThat(gitHub.lastRateLimit(), sameInstance(headerRateLimit));

        // rate limit request is free, remaining is unchanged
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining());
        previousLimit = rateLimit;

        // Give this a moment
        Thread.sleep(1000);

        // Always requests new info
        rateLimit = gitHub.getRateLimit();
        assertThat(mockGitHub.getRequestCount(), equalTo(3));

        // Because remaining and reset date are unchanged, the header should be unchanged as well
        assertThat(gitHub.lastRateLimit(), sameInstance(headerRateLimit));

        // rate limit request is free, remaining is unchanged
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining());
        previousLimit = rateLimit;

        gitHub.getOrganization(GITHUB_API_TEST_ORG);
        assertThat(mockGitHub.getRequestCount(), equalTo(4));

        // Because remaining has changed the header should be different
        assertThat(gitHub.lastRateLimit(), not(sameInstance(headerRateLimit)));
        assertThat(gitHub.lastRateLimit(), not(equalTo(headerRateLimit)));
        rateLimit = gitHub.lastRateLimit();

        // Org costs limit to query
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining() - 1);

        previousLimit = rateLimit;
        headerRateLimit = rateLimit;

        // ratelimit() should prefer headerRateLimit when it is most recent and not expired
        assertThat(gitHub.rateLimit(), sameInstance(headerRateLimit));

        assertThat(mockGitHub.getRequestCount(), equalTo(4));

        // AT THIS POINT WE SIMULATE A RATE LIMIT RESET

        // Give this a moment
        Thread.sleep(2000);

        // Always requests new info
        rateLimit = gitHub.getRateLimit();
        assertThat(mockGitHub.getRequestCount(), equalTo(5));

        // rate limit request is free, remaining is unchanged date is later
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining(), true);
        previousLimit = rateLimit;

        // When getRateLimit() succeeds, headerRateLimit updates as usual as well (if needed)
        // These are separate instances, but should be equal
        assertThat(gitHub.rateLimit(), not(sameInstance(rateLimit)));

        // Verify different record instances can be compared
        assertThat(gitHub.rateLimit().getCore(), equalTo(rateLimit.getCore()));

        // Verify different instances can be compared
        // TODO: This is not work currently because the header rate limit has unknowns for records other than core.
        // assertThat(gitHub.rateLimit(), equalTo(rateLimit));

        assertThat(gitHub.rateLimit(), not(sameInstance(headerRateLimit)));
        assertThat(gitHub.rateLimit(), sameInstance(gitHub.lastRateLimit()));

        assertThat(mockGitHub.getRequestCount(), equalTo(5));
    }

    private void verifyRateLimitValues(GHRateLimit previousLimit, int remaining) {
        verifyRateLimitValues(previousLimit, remaining, false);
    }

    private void verifyRateLimitValues(GHRateLimit previousLimit, int remaining, boolean changedResetDate) {
        // newer or unchange
        int resetComparisionValue = changedResetDate ? 1 : 0;

        // Basic checks of values
        assertThat(rateLimit, notNullValue());
        assertThat(rateLimit.getLimit(), equalTo(previousLimit.getLimit()));
        assertThat(rateLimit.getRemaining(), equalTo(remaining));

        // Check that the reset date of the current limit is not older than the previous one
        assertThat(rateLimit.getResetDate().compareTo(previousLimit.getResetDate()), equalTo(resetComparisionValue));

        // Additional checks for record values
        assertThat(rateLimit.getCore().getLimit(), equalTo(rateLimit.getLimit()));
        assertThat(rateLimit.getCore().getRemaining(), equalTo(rateLimit.getRemaining()));
        assertThat(rateLimit.getCore().getResetEpochSeconds(), equalTo(rateLimit.getResetEpochSeconds()));
        assertThat(rateLimit.getCore().getResetDate(), equalTo(rateLimit.getResetDate()));

        // Additional checks for deprecated values
        assertThat(rateLimit.limit, equalTo(rateLimit.getLimit()));
        assertThat(rateLimit.remaining, equalTo(rateLimit.getRemaining()));
        assertThat(rateLimit.reset.getTime(), equalTo(rateLimit.getResetEpochSeconds()));
    }

    @Test
    public void testGitHubEnterpriseDoesNotHaveRateLimit() throws Exception {
        // Customized response that results in file not found the same as GitHub Enterprise
        snapshotNotAllowed();
        assertThat(mockGitHub.getRequestCount(), equalTo(0));
        GHRateLimit rateLimit = null;

        Date lastReset = new Date(System.currentTimeMillis() / 1000L);

        // Give this a moment
        Thread.sleep(1000);
<commit_msg>Make rate limit tests tolerant of one second timing differences<commit_after>package org.kohsuke.github;

import com.fasterxml.jackson.databind.exc.MismatchedInputException;
import com.fasterxml.jackson.databind.exc.ValueInstantiationException;
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
import org.hamcrest.CoreMatchers;
import org.junit.Test;

import java.io.IOException;
import java.time.Duration;
import java.util.Date;

import static org.hamcrest.Matchers.*;
import static org.hamcrest.core.IsInstanceOf.instanceOf;

/**
 * Test showing the behavior of OkHttpConnector with and without cache.
 * <p>
 * Key take aways:
 *
 * <ul>
 * <li>These tests are artificial and intended to highlight the differences in behavior between scenarios. However, the
 * differences they indicate are stark.</li>
 * <li>Caching reduces rate limit consumption by at least a factor of two in even the simplest case.</li>
 * <li>The OkHttp cache is pretty smart and will often connect read and write requests made on the same client and
 * invalidate caches.</li>
 * <li>Changes made outside the current client cause the OkHttp cache to return stale data. This is expected and correct
 * behavior.</li>
 * <li>"max-age=0" addresses the problem of external changes by revalidating caches for each request. This produces the
 * same number of requests as OkHttp without caching, but those requests only count towards the GitHub rate limit if
 * data has changes.</li>
 * </ul>
 *
 * @author <NAME>
 */
public class GHRateLimitTest extends AbstractGitHubWireMockTest {

    GHRateLimit rateLimit = null;
    GHRateLimit previousLimit = null;

    public GHRateLimitTest() {
        useDefaultGitHub = false;
    }

    @Override
    protected WireMockConfiguration getWireMockOptions() {
        return super.getWireMockOptions().extensions(templating.newResponseTransformer());
    }

    @Test
    public void testGitHubRateLimit() throws Exception {
        // Customized response that templates the date to keep things working
        snapshotNotAllowed();

        assertThat(mockGitHub.getRequestCount(), equalTo(0));

        // 4897 is just the what the limit was when the snapshot was taken
        previousLimit = GHRateLimit.fromHeaderRecord(new GHRateLimit.Record(5000,
                4897,
                (templating.testStartDate.getTime() + Duration.ofHours(1).toMillis()) / 1000L));

        // -------------------------------------------------------------
        // /user gets response with rate limit information
        gitHub = getGitHubBuilder().withEndpoint(mockGitHub.apiServer().baseUrl()).build();
        gitHub.getMyself();

        assertThat(mockGitHub.getRequestCount(), equalTo(1));

        // Since we already had rate limit info these don't request again
        rateLimit = gitHub.lastRateLimit();
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining());
        previousLimit = rateLimit;

        GHRateLimit headerRateLimit = rateLimit;

        // Give this a moment
        Thread.sleep(1000);

        // ratelimit() uses headerRateLimit if available and headerRateLimit is not expired
        assertThat(gitHub.rateLimit(), equalTo(headerRateLimit));

        assertThat(mockGitHub.getRequestCount(), equalTo(1));

        // Give this a moment
        Thread.sleep(1000);

        // Always requests new info
        rateLimit = gitHub.getRateLimit();
        assertThat(mockGitHub.getRequestCount(), equalTo(2));

        // Because remaining and reset date are unchanged, the header should be unchanged as well
        assertThat(gitHub.lastRateLimit(), sameInstance(headerRateLimit));

        // rate limit request is free, remaining is unchanged
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining());
        previousLimit = rateLimit;

        // Give this a moment
        Thread.sleep(1000);

        // Always requests new info
        rateLimit = gitHub.getRateLimit();
        assertThat(mockGitHub.getRequestCount(), equalTo(3));

        // Because remaining and reset date are unchanged, the header should be unchanged as well
        assertThat(gitHub.lastRateLimit(), sameInstance(headerRateLimit));

        // rate limit request is free, remaining is unchanged
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining());
        previousLimit = rateLimit;

        gitHub.getOrganization(GITHUB_API_TEST_ORG);
        assertThat(mockGitHub.getRequestCount(), equalTo(4));

        // Because remaining has changed the header should be different
        assertThat(gitHub.lastRateLimit(), not(sameInstance(headerRateLimit)));
        assertThat(gitHub.lastRateLimit(), not(equalTo(headerRateLimit)));
        rateLimit = gitHub.lastRateLimit();

        // Org costs limit to query
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining() - 1);

        previousLimit = rateLimit;
        headerRateLimit = rateLimit;

        // ratelimit() should prefer headerRateLimit when it is most recent and not expired
        assertThat(gitHub.rateLimit(), sameInstance(headerRateLimit));

        assertThat(mockGitHub.getRequestCount(), equalTo(4));

        // AT THIS POINT WE SIMULATE A RATE LIMIT RESET

        // Give this a moment
        Thread.sleep(2000);

        // Always requests new info
        rateLimit = gitHub.getRateLimit();
        assertThat(mockGitHub.getRequestCount(), equalTo(5));

        // rate limit request is free, remaining is unchanged date is later
        verifyRateLimitValues(previousLimit, previousLimit.getRemaining(), true);
        previousLimit = rateLimit;

        // When getRateLimit() succeeds, headerRateLimit updates as usual as well (if needed)
        // These are separate instances, but should be equal
        assertThat(gitHub.rateLimit(), not(sameInstance(rateLimit)));

        // Verify different record instances can be compared
        assertThat(gitHub.rateLimit().getCore(), equalTo(rateLimit.getCore()));

        // Verify different instances can be compared
        // TODO: This is not work currently because the header rate limit has unknowns for records other than core.
        // assertThat(gitHub.rateLimit(), equalTo(rateLimit));

        assertThat(gitHub.rateLimit(), not(sameInstance(headerRateLimit)));
        assertThat(gitHub.rateLimit(), sameInstance(gitHub.lastRateLimit()));

        assertThat(mockGitHub.getRequestCount(), equalTo(5));
    }

    private void verifyRateLimitValues(GHRateLimit previousLimit, int remaining) {
        verifyRateLimitValues(previousLimit, remaining, false);
    }

    private void verifyRateLimitValues(GHRateLimit previousLimit, int remaining, boolean changedResetDate) {
        // Basic checks of values
        assertThat(rateLimit, notNullValue());
        assertThat(rateLimit.getLimit(), equalTo(previousLimit.getLimit()));
        assertThat(rateLimit.getRemaining(), equalTo(remaining));

        // Check that the reset date of the current limit is not older than the previous one
        long diffMillis = rateLimit.getResetDate().getTime() - previousLimit.getResetDate().getTime();

        assertThat(diffMillis, greaterThanOrEqualTo(0L));
        if (changedResetDate) {
            assertThat(diffMillis, greaterThan(1000L));
        } else {
            assertThat(diffMillis, lessThanOrEqualTo(1000L));
        }

        // Additional checks for record values
        assertThat(rateLimit.getCore().getLimit(), equalTo(rateLimit.getLimit()));
        assertThat(rateLimit.getCore().getRemaining(), equalTo(rateLimit.getRemaining()));
        assertThat(rateLimit.getCore().getResetEpochSeconds(), equalTo(rateLimit.getResetEpochSeconds()));
        assertThat(rateLimit.getCore().getResetDate(), equalTo(rateLimit.getResetDate()));

        // Additional checks for deprecated values
        assertThat(rateLimit.limit, equalTo(rateLimit.getLimit()));
        assertThat(rateLimit.remaining, equalTo(rateLimit.getRemaining()));
        assertThat(rateLimit.reset.getTime(), equalTo(rateLimit.getResetEpochSeconds()));
    }

    @Test
    public void testGitHubEnterpriseDoesNotHaveRateLimit() throws Exception {
        // Customized response that results in file not found the same as GitHub Enterprise
        snapshotNotAllowed();
        assertThat(mockGitHub.getRequestCount(), equalTo(0));
        GHRateLimit rateLimit = null;

        Date lastReset = new Date(System.currentTimeMillis() / 1000L);

        // Give this a moment
        Thread.sleep(1000);

{'fraction_non_alphanumeric': 0.09915399633201206, 'fraction_numerical': 0.006921848192628527, 'mean_word_length': 3.3023670145075084, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2976518', 'n_tokens_mistral': 4405, 'n_tokens_neox': 4231, 'n_words': 1234}