<commit_before># -*- coding: utf-8 -*-
"""global variables for the API URL and rate limiting."""
import functools
from halo import Halo
from ratelimiter import RateLimiter
import time
import threading

ROOT = "https://api.crimsonhexagon.com/api/"

ONE_MINUTE = 60
MAX_CALLS = 120


class SpinnerLimiter(RateLimiter):
    """Rate limiting with spinner information."""

    def __enter__(self):
        """When limiter sleeps, show spinner with wait time."""
        with self._lock:
            # We want to ensure that no more than max_calls were run in the allowed
            # period. For this, we store the last timestamps of each call and run
            # the rate verification upon each __enter__ call.
            if len(self.calls) >= self.max_calls:
                until = time.time() + self.period - self._timespan
                if self.callback:
                    t = threading.Thread(target=self.callback, args=(until, ))
                    t.daemon = True
                    t.start()
                sleeptime = until - time.time()
                with Halo(text="Rate Limit Reached. (Sleeping for {} seconds)".
                          format(round(sleeptime))):
                    if sleeptime > 0:
                        time.sleep(sleeptime)
            return self


def response_handler(f):
    """Ensure responses do not contain errors, and Rate Limit is obeyed."""

    @SpinnerLimiter(max_calls=MAX_CALLS, period=ONE_MINUTE)
    @functools.wraps(f)
    def wrapped(*args, **kwargs):
        response = f(*args, **kwargs)
        if response.status_code not in [200, 201, 202]:
            raise ValueError("Something Went Wrong." + response.text)
        elif ("status" in response.json()
              ) and response.json()["status"] == "error":
            raise ValueError("Something Went Wrong. " + response.text)
        return response.json()

    return wrapped
<commit_msg>fixed rate limiting decorator to reset request count after waiting<commit_after># -*- coding: utf-8 -*-
"""global variables for the API URL and rate limiting."""
import functools
from halo import Halo
from ratelimiter import RateLimiter
import time
import threading
import collections

ROOT = "https://api.crimsonhexagon.com/api/"

ONE_MINUTE = 60
MAX_CALLS = 120


class SpinnerLimiter(RateLimiter):
    """Rate limiting with spinner information."""

    def __enter__(self):
        """When limiter sleeps, show spinner with wait time."""
        with self._lock:
            # We want to ensure that no more than max_calls were run in the allowed
            # period. For this, we store the last timestamps of each call and run
            # the rate verification upon each __enter__ call.
            if len(self.calls) >= self.max_calls:
                until = time.time() + self.period - self._timespan
                if self.callback:
                    t = threading.Thread(target=self.callback, args=(until, ))
                    t.daemon = True
                    t.start()
                sleeptime = until - time.time()
                with Halo(text="Rate Limit Reached. (Sleeping for {} seconds)".
                          format(round(sleeptime))):
                    if sleeptime > 0:
                        time.sleep(sleeptime)
                        self.calls = collections.deque()
            return self


def response_handler(f):
    """Ensure responses do not contain errors, and Rate Limit is obeyed."""

    @SpinnerLimiter(max_calls=MAX_CALLS, period=ONE_MINUTE)
    @functools.wraps(f)
    def wrapped(*args, **kwargs):
        response = f(*args, **kwargs)
        if response.status_code not in [200, 201, 202]:
            raise ValueError("Something Went Wrong." + response.text)
        elif ("status" in response.json()
              ) and response.json()["status"] == "error":
            raise ValueError("Something Went Wrong. " + response.text)
        return response.json()

    return wrapped

{'fraction_non_alphanumeric': 0.10027784794139934, 'fraction_numerical': 0.008082849204344532, 'mean_word_length': 2.130434782608696, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3251200', 'n_tokens_mistral': 1112, 'n_tokens_neox': 1053, 'n_words': 347}