<commit_before>"""
Curious' HTTP client is a thin wrapper over the `requests` library, running in threads.

This is because there is (currently) no native curio HTTP library.
"""
import functools
import logging
import sys
import time
import typing
import weakref
from math import ceil
from urllib.parse import quote

import curio

import curious
from curious.exc import Forbidden, HTTPException, NotFound, Unauthorized
from curious.http.curio_http import ClientSession, Response


# HTTP exceptions, used to raise errors.


class HTTPClient(object):
    API_BASE = "https://discordapp.com/api/v6"
    GUILD_BASE = API_BASE + "/guilds/{guild_id}"
    CHANNEL_BASE = API_BASE + "/channels/{channel_id}"

    USER_ME = API_BASE + "/users/@me"

    def __init__(self, token: str):
        #: The token used for all requests.
        self.token = token

        # Calculated headers
        headers = {
            "Authorization": "Bot {}".format(self.token),
            "User-Agent": "DiscordBot (https://github.com/SunDwarf/curious {0}) Python/{1[0]}.{1[1]}"
                          " curio/{2}".format(curious.__version__, sys.version_info,
                                              curio.__version__)
        }

        self.session = ClientSession()
        self.session.headers = headers

        #: Ratelimit buckets.
        self._rate_limits = weakref.WeakValueDictionary()

        #: Ratelimit remaining times
        self._ratelimit_remaining = {}

        self.logger = logging.getLogger("curious.http")

    def get_ratelimit_lock(self, bucket: object):
        """
        Gets a ratelimit lock from the dict if it exists, otherwise creates a new one.
        """
        try:
            return self._rate_limits[bucket]
        except KeyError:
            lock = curio.Lock()
            self._rate_limits[bucket] = lock
            return lock

    # Special wrapper functions

    async def __perform_request(self, *args, **kwargs) -> Response:
        """
        A wrapper for requests' request that uses curio.abide().

        This only performs the request - it does NOT do any ratelimiting.
        Hence, this method is unsafe, and is private. It should not be used by client code.
        """
        partial = functools.partial(self.session.request, *args, **kwargs)
        coro = curio.timeout_after(5, curio.abide(partial))
        return await coro

    async def get_response_data(self, response: Response) -> typing.Union[str, dict]:
        """
        Return either the text of a request or the JSON.

        :param response: The response to use.
        """
        if response.headers.get("Content-Type", None) == "application/json":
            return await response.json()

        return await response.text()

    async def request(self, bucket: object, *args, **kwargs):
        """
        Makes a rate-limited request.

        This will respect Discord's X-Ratelimit-Limit headers to make requests.

        :param bucket: The bucket this request falls under.
        """
        # Okay, an English explaination of how this works.
        # First, it loads the curio-based lock from the defaultdict of lock, keyed by bucket.
        # Then, it tries to acquire the lock. This is held by one request at a time, naturally.

        # Normally, the lock is immediately released upon a request finishing, which allows the next request to
        # handle it. However, once X-Ratelimit-Remaining is 0, we don't want any more requests to be made until the
        # time limit is over. So the request sleeps for (X-RateLimit-Reset - time.time()) seconds, then unlocks the
        # lock.

        lock = self.get_ratelimit_lock(bucket)
        try:
            await lock.acquire()

            if bucket in self._ratelimit_remaining:
                # Make sure we have enough tries left.
                tries, reset_time = self._ratelimit_remaining[bucket]

                if tries <= 0:
                    # We need to sleep for a bit before we can start making another request.
                    sleep_time = ceil(reset_time - time.time())
                    self.logger.debug("Sleeping with lock open for {} seconds.".format(sleep_time))
                    await curio.sleep(sleep_time)

            for tries in range(0, 5):
                # Make the request.
                response = await self.session.request(*args, **kwargs)
                self.logger.debug("{} {} => {}".format(kwargs.get("method", "???"),
                                                       kwargs.get("url", "???"),
                                                       response.status_code))

                if response.status_code == 502:
                    # 502 means that we can retry without worrying about ratelimits.
                    # Perform exponential backoff to prevent spamming discord.
                    sleep_time = 1 + (tries * 2)
                    await curio.sleep(sleep_time)
                    continue

                if response.status_code == 429:
                    # This is bad!
                    # But it's okay, we can handle it.
                    self.logger.warning("Hit a 429 in bucket {}. Check your clock!".format(bucket))
                    sleep_time = ceil(int(response.headers["Retry-After"]) / 1000)
                    await curio.sleep(sleep_time)
                    continue

                # Extract ratelimit headers.
                remaining = int(response.headers.get("X-Ratelimit-Remaining", 1))
                reset = int(response.headers.get("X-Ratelimit-Reset", 1))<commit_msg>Retry on 500 errors too (when discord is dying.)<commit_after>"""
Curious' HTTP client is a thin wrapper over the `requests` library, running in threads.

This is because there is (currently) no native curio HTTP library.
"""
import functools
import logging
import sys
import time
import typing
import weakref
from math import ceil
from urllib.parse import quote

import curio

import curious
from curious.exc import Forbidden, HTTPException, NotFound, Unauthorized
from curious.http.curio_http import ClientSession, Response


# HTTP exceptions, used to raise errors.


class HTTPClient(object):
    API_BASE = "https://discordapp.com/api/v6"
    GUILD_BASE = API_BASE + "/guilds/{guild_id}"
    CHANNEL_BASE = API_BASE + "/channels/{channel_id}"

    USER_ME = API_BASE + "/users/@me"

    def __init__(self, token: str):
        #: The token used for all requests.
        self.token = token

        # Calculated headers
        headers = {
            "Authorization": "Bot {}".format(self.token),
            "User-Agent": "DiscordBot (https://github.com/SunDwarf/curious {0}) Python/{1[0]}.{1[1]}"
                          " curio/{2}".format(curious.__version__, sys.version_info,
                                              curio.__version__)
        }

        self.session = ClientSession()
        self.session.headers = headers

        #: Ratelimit buckets.
        self._rate_limits = weakref.WeakValueDictionary()

        #: Ratelimit remaining times
        self._ratelimit_remaining = {}

        self.logger = logging.getLogger("curious.http")

    def get_ratelimit_lock(self, bucket: object):
        """
        Gets a ratelimit lock from the dict if it exists, otherwise creates a new one.
        """
        try:
            return self._rate_limits[bucket]
        except KeyError:
            lock = curio.Lock()
            self._rate_limits[bucket] = lock
            return lock

    # Special wrapper functions

    async def __perform_request(self, *args, **kwargs) -> Response:
        """
        A wrapper for requests' request that uses curio.abide().

        This only performs the request - it does NOT do any ratelimiting.
        Hence, this method is unsafe, and is private. It should not be used by client code.
        """
        partial = functools.partial(self.session.request, *args, **kwargs)
        coro = curio.timeout_after(5, curio.abide(partial))
        return await coro

    async def get_response_data(self, response: Response) -> typing.Union[str, dict]:
        """
        Return either the text of a request or the JSON.

        :param response: The response to use.
        """
        if response.headers.get("Content-Type", None) == "application/json":
            return await response.json()

        return await response.text()

    async def request(self, bucket: object, *args, **kwargs):
        """
        Makes a rate-limited request.

        This will respect Discord's X-Ratelimit-Limit headers to make requests.

        :param bucket: The bucket this request falls under.
        """
        # Okay, an English explaination of how this works.
        # First, it loads the curio-based lock from the defaultdict of lock, keyed by bucket.
        # Then, it tries to acquire the lock. This is held by one request at a time, naturally.

        # Normally, the lock is immediately released upon a request finishing, which allows the next request to
        # handle it. However, once X-Ratelimit-Remaining is 0, we don't want any more requests to be made until the
        # time limit is over. So the request sleeps for (X-RateLimit-Reset - time.time()) seconds, then unlocks the
        # lock.

        lock = self.get_ratelimit_lock(bucket)
        try:
            await lock.acquire()

            if bucket in self._ratelimit_remaining:
                # Make sure we have enough tries left.
                tries, reset_time = self._ratelimit_remaining[bucket]

                if tries <= 0:
                    # We need to sleep for a bit before we can start making another request.
                    sleep_time = ceil(reset_time - time.time())
                    self.logger.debug("Sleeping with lock open for {} seconds.".format(sleep_time))
                    await curio.sleep(sleep_time)

            for tries in range(0, 5):
                # Make the request.
                response = await self.session.request(*args, **kwargs)
                self.logger.debug("{} {} => {}".format(kwargs.get("method", "???"),
                                                       kwargs.get("url", "???"),
                                                       response.status_code))

                if response.status_code in [500, 502]:
                    # 502 means that we can retry without worrying about ratelimits.
                    # Perform exponential backoff to prevent spamming discord.
                    sleep_time = 1 + (tries * 2)
                    await curio.sleep(sleep_time)
                    continue

                if response.status_code == 429:
                    # This is bad!
                    # But it's okay, we can handle it.
                    self.logger.warning("Hit a 429 in bucket {}. Check your clock!".format(bucket))
                    sleep_time = ceil(int(response.headers["Retry-After"]) / 1000)
                    await curio.sleep(sleep_time)
                    continue

                # Extract ratelimit headers.
                remaining = int(response.headers.get("X-Ratelimit-Remaining", 1))
                reset = int(response.headers.get("X-Ratelimit-Reset", 1))
{'fraction_non_alphanumeric': 0.09181672312355144, 'fraction_numerical': 0.006239971474416117, 'mean_word_length': 1.9376800209478922, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '89626', 'n_tokens_mistral': 3026, 'n_tokens_neox': 2838, 'n_words': 971}