<commit_before>import os
import time
import click
import logging
import requests

import datetime
import humanize
from tqdm.auto import tqdm
from functools import wraps


log = logging.getLogger("twarc")


def rate_limit(f, tries=30):
    """
    A decorator to handle rate limiting from the Twitter v2 API. If
    a rate limit error is encountered we will sleep until we can
    issue the API call again.
    """

    @wraps(f)
    def new_f(*args, **kwargs):
        errors = 0
        while True:
            resp = f(*args, **kwargs)
            if resp.status_code in [200, 201]:
                errors = 0
                return resp
            elif resp.status_code == 429:
                reset = int(resp.headers["x-rate-limit-reset"])
                now = time.time()

                # The time to sleep depends on having an accurate system time,
                # so check to see if there's something really bad happening
                # to warn the user.
                target_sleep_seconds = reset - now

                # Never sleep longer than 15 minutes, as that is the basis for
                # all of the read time based rate limits in the Twitter API
                seconds = min(901, max(10, (target_sleep_seconds + 10)))

                if target_sleep_seconds >= 900:
                    # If we need to sleep for more than a rate limit period, the
                    # system clock could be wrong.
                    log.warning(
                        "Detected overlong sleep interval - is your system clock accurate? "
                        "An accurate system time is needed to calculate how long to sleep for."
                    )

                log.warning("rate limit exceeded: sleeping %s secs", seconds)
                time.sleep(seconds)
            elif resp.status_code >= 500:
                errors += 1
                if errors > tries:
                    log.warning(f"too many errors ({tries}) from Twitter, giving up")
                    resp.raise_for_status()
                seconds = errors ** 2
                log.warning(
                    "caught %s from Twitter API, sleeping %s", resp.status_code, seconds
                )
                time.sleep(seconds)
            else:
                log.error("Unexpected HTTP response: %s", resp)
                resp.raise_for_status()

    return new_f


def catch_request_exceptions(f, tries=30):<commit_msg>Also warn on negative sleep time<commit_after>import os
import time
import click
import logging
import requests

import datetime
import humanize
from tqdm.auto import tqdm
from functools import wraps


log = logging.getLogger("twarc")


def rate_limit(f, tries=30):
    """
    A decorator to handle rate limiting from the Twitter v2 API. If
    a rate limit error is encountered we will sleep until we can
    issue the API call again.
    """

    @wraps(f)
    def new_f(*args, **kwargs):
        errors = 0
        while True:
            resp = f(*args, **kwargs)
            if resp.status_code in [200, 201]:
                errors = 0
                return resp
            elif resp.status_code == 429:
                reset = int(resp.headers["x-rate-limit-reset"])
                now = time.time()

                # The time to sleep depends on having an accurate system time,
                # so check to see if there's something really bad happening
                # to warn the user.
                target_sleep_seconds = reset - now

                # Never sleep longer than 15 minutes, as that is the basis for
                # all of the read time based rate limits in the Twitter API
                seconds = min(901, max(10, (target_sleep_seconds + 10)))

                if target_sleep_seconds >= 900:
                    # If we need to sleep for more than a rate limit period, the
                    # system clock could be wrong.
                    log.warning(
                        "Detected overlong sleep interval - is your system clock accurate? "
                        "An accurate system time is needed to calculate how long to sleep for, "
                        "and data collection might be slowed."
                    )
                elif target_sleep_seconds < 0:
                    # If we need to sleep for negative time something weird might be up.
                    log.warning(
                        "Detected negative sleep interval - is your system clock accurate? "
                        "If your system time is running fast, rate limiting may not be "
                        "effective."
                    )


                log.warning("rate limit exceeded: sleeping %s secs", seconds)
                time.sleep(seconds)
            elif resp.status_code >= 500:
                errors += 1
                if errors > tries:
                    log.warning(f"too many errors ({tries}) from Twitter, giving up")
                    resp.raise_for_status()
                seconds = errors ** 2
                log.warning(
                    "caught %s from Twitter API, sleeping %s", resp.status_code, seconds
                )
                time.sleep(seconds)
            else:
                log.error("Unexpected HTTP response: %s", resp)
                resp.raise_for_status()

    return new_f


def catch_request_exceptions(f, tries=30):
{'fraction_non_alphanumeric': 0.06753731343283582, 'fraction_numerical': 0.0125, 'mean_word_length': 1.5063113604488079, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1961522', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1245, 'n_words': 519}