<commit_before>package langpop.sites

class Stackoverflow extends CodingSite {
    // This is the date of the oldest data in stackoverflow
    static final String OLDEST_DATE = '2008-07-31'
    static final String SITE_NAME = 'stackoverflow'

    // TODO: handle API limitations in instance
    private static final int API_SLEEP_TIME = 60000
    // Uses a custom filter that only returns quota_remaining and total
    // (https://api.stackexchange.com/docs/create-filter#unsafe=false&filter=!GeF-5sUcKK53)&run=true)
    private static final String API_URL = 'https://api.stackexchange.com/2.2/search?todate=%s&site=stackoverflow&tagged=%s&filter=!GeF-5sUcKK53)'
    // Try this many times on API failures before giving up
    private static final int MAX_API_TRIES = 20

    private String apiKey

    @Override
    Date getOldestDate() {
        return Date.parse('yyyy-MM-dd', OLDEST_DATE)
    }

    @Override
    Integer getScore(String langName, Date date) {
        def url = String.format(API_URL, encodeDate(date), encodeLangName(langName))
        def result = getResult(url)

        log.debug("StackOverflow API daily quota remaining: ${result.quota_remaining}")

        return result.total<commit_msg>Update stackoverflow filter to include backoff field<commit_after>package langpop.sites

class Stackoverflow extends CodingSite {
    // This is the date of the oldest data in stackoverflow
    static final String OLDEST_DATE = '2008-07-31'
    static final String SITE_NAME = 'stackoverflow'

    /* TODO: handle API limitations in instance (https://stackapps.com/a/3057/41977)
     *  - Don't make more than 30 requests/second
     *  - Handle backoff field
     */
    private static final int API_SLEEP_TIME = 60000
    // Uses a custom filter that only returns backoff, quota_remaining, and total
    // (https://api.stackexchange.com/docs/create-filter#unsafe=false&filter=!.UE8F0bVg4M-_Ii4&run=true)
    private static final String API_URL = 'https://api.stackexchange.com/2.2/search?todate=%s&site=stackoverflow&tagged=%s&filter=!.UE8F0bVg4M-_Ii4'
    // Try this many times on API failures before giving up
    private static final int MAX_API_TRIES = 20

    private String apiKey

    @Override
    Date getOldestDate() {
        return Date.parse('yyyy-MM-dd', OLDEST_DATE)
    }

    @Override
    Integer getScore(String langName, Date date) {
        def url = String.format(API_URL, encodeDate(date), encodeLangName(langName))
        def result = getResult(url)

        log.debug("StackOverflow API daily quota remaining: ${result.quota_remaining}")

        return result.total
{'fraction_non_alphanumeric': 0.09230180007659901, 'fraction_numerical': 0.022596706242818843, 'mean_word_length': 3.511226252158895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5711472', 'n_tokens_mistral': 849, 'n_tokens_neox': 788, 'n_words': 220}