<commit_before>import asyncio

import aiofiles
import aiohttp
import aiohttp.client_exceptions
from aiopath import AsyncPath
from sanic.log import logger

EXCEPTIONS = (
    aiohttp.client_exceptions.ClientConnectionError,
    aiohttp.client_exceptions.InvalidURL,
    aiohttp.client_exceptions.TooManyRedirects,
    AssertionError,
    asyncio.TimeoutError,
    UnicodeError,
)


async def fetch(url: str, **kwargs) -> tuple[int, dict | str]:
    async with aiohttp.ClientSession() as session:
        try:
            async with session.get(url, timeout=10, **kwargs) as response:

                try:
                    message = await response.json()
                except aiohttp.client_exceptions.ContentTypeError:
                    message = await response.text()

                return response.status, message

        except EXCEPTIONS as e:
            message = str(e).strip("() ") or e.__class__.__name__

            return 500, message


async def download(url: str, path: AsyncPath) -> bool:
    async with aiohttp.ClientSession() as session:
        try:
            async with session.get(url, timeout=10) as response:

                if response.history and response.history[0].status == 302:
                    logger.error(f"302 response from {url}")
                    return False

                if response.status == 200:
                    f = await aiofiles.open(path, mode="wb")  # type: ignore
                    await f.write(await response.read())
                    await f.close()
                    return True

                logger.error(f"{response.status} response from {url}")

        except EXCEPTIONS as e:
            message = str(e).strip("() ") or e.__class__.__name__
            logger.error(f"Invalid response from {url}: {message}")

    return False<commit_msg>Reject all background images that redirect<commit_after>import asyncio

import aiofiles
import aiohttp
import aiohttp.client_exceptions
from aiopath import AsyncPath
from sanic.log import logger

EXCEPTIONS = (
    aiohttp.client_exceptions.ClientConnectionError,
    aiohttp.client_exceptions.InvalidURL,
    aiohttp.client_exceptions.TooManyRedirects,
    AssertionError,
    asyncio.TimeoutError,
    UnicodeError,
)


async def fetch(url: str, **kwargs) -> tuple[int, dict | str]:
    async with aiohttp.ClientSession() as session:
        try:
            async with session.get(url, timeout=10, **kwargs) as response:

                try:
                    message = await response.json()
                except aiohttp.client_exceptions.ContentTypeError:
                    message = await response.text()

                return response.status, message

        except EXCEPTIONS as e:
            message = str(e).strip("() ") or e.__class__.__name__

            return 500, message


async def download(url: str, path: AsyncPath) -> bool:
    async with aiohttp.ClientSession() as session:
        try:
            async with session.get(url, timeout=10) as response:

                if response.history:
                    logger.error(f"3xx response from {url}")
                    return False

                if response.status == 200:
                    logger.info(f"200 response from {url}")
                    f = await aiofiles.open(path, mode="wb")  # type: ignore
                    await f.write(await response.read())
                    await f.close()
                    return True

                logger.error(f"{response.status} response from {url}")

        except EXCEPTIONS as e:
            message = str(e).strip("() ") or e.__class__.__name__
            logger.error(f"5xx response from {url}: {message}")

    return False
{'fraction_non_alphanumeric': 0.08997568224804108, 'fraction_numerical': 0.008646311807619562, 'mean_word_length': 2.036915504511895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5785284', 'n_tokens_mistral': 1037, 'n_tokens_neox': 967, 'n_words': 269}