<commit_before>from typing import Optional, Iterable
from idiotic import config
import aiohttp
import asyncio


class MissingResource(Exception):
    pass


class Resource:
    def __init__(self):
        self.available = False
        self.static = False

    async def try_check(self):
        if not self.available:
            raise MissingResource(self)

    def available_hosts(self, config: config.Config) -> Optional[Iterable[str]]:
        """Return a list of hosts where this resource is statically guaranteed to be available.
        This is used during the pre-allocation phase of blocks to determine which nodes the scheduler
        should not attempt to schedule a block on.
        If this resource cannot be statically determined, it should return None or a list of all hosts."""

        return None

    async def run(self):
        pass


class HTTPResource(Resource):
    def __init__(self, address):
        self.address = address
        super().__init__()

    async def run(self):
        while True:
            async with aiohttp.ClientSession() as client:
                async with client.get(self.address) as response:
                    if response.status == 200:
                        self.available = True
                    else:
                        self.available = False
                await asyncio.sleep(10)
<commit_msg>Use HEAD request for HTTP resources<commit_after>from typing import Optional, Iterable
from idiotic import config
import aiohttp
import asyncio


class MissingResource(Exception):
    pass


class Resource:
    def __init__(self):
        self.available = False
        self.static = False

    async def try_check(self):
        if not self.available:
            raise MissingResource(self)

    def available_hosts(self, config: config.Config) -> Optional[Iterable[str]]:
        """Return a list of hosts where this resource is statically guaranteed to be available.
        This is used during the pre-allocation phase of blocks to determine which nodes the scheduler
        should not attempt to schedule a block on.
        If this resource cannot be statically determined, it should return None or a list of all hosts."""

        return None

    async def run(self):
        pass


class HTTPResource(Resource):
    def __init__(self, address):
        self.address = address
        super().__init__()

    async def run(self):
        while True:
            async with aiohttp.ClientSession() as client:
                async with client.head(self.address) as response:
                    if response.status == 200:
                        self.available = True
                    else:
                        self.available = False
                await asyncio.sleep(10)

{'fraction_non_alphanumeric': 0.06311207834602829, 'fraction_numerical': 0.003627130939426913, 'mean_word_length': 2.1556064073226544, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3470527', 'n_tokens_mistral': 687, 'n_tokens_neox': 645, 'n_words': 267}