<commit_before>"""Maintain host infos."""

import asyncio
import collections
import random
import datetime as DT
import urllib.parse as UP


class HostPool:
    """Maintain host infos."""

    dyn_delay_table = {  # dyn_delay_factor -> second
        0: 0,
        1: 5,
        2: 10,
        3: 20,
        4: 30,
        5: 40,
        6: 50,
        7: 60,
        8: 90,
        9: 120,
        10: 180,
        11: 240,
        12: 300,
        13: 600,
        14: 900,
        15: 1200,
        16: 1500,
        17: 1800,
        18: 2100,
        19: 2400,
        20: 3600,
    }

    def __init__(self, config, loop):
        """Init host infos."""
        self.config = config
        self.loop = loop

        self.hosts = collections.defaultdict(self.__default_host_info)

    def __default_host_info(self):
        return {
            'dyn_delay_factor': 0,
            'dyn_delay_changed': DT.datetime.utcnow(),
            'semaphore': asyncio.Semaphore(
                value=self.config.per_host_concurrent,
                loop=self.loop),
        }

    def __get_host(self, url):
        netloc = UP.urlparse(url).netloc

        return self.hosts[netloc]

    def __get_dyn_delay_factor(self, url):
        host = self.__get_host(url)

        return host.get('dyn_delay_factor')

    def __increase_dyn_delay_factor(self, url):
        host = self.__get_host(url)
        host['dyn_delay_factor'] = min(20, host['dyn_delay_factor'] + 1)
        host['dyn_delay_changed'] = DT.datetime.utcnow()

    def __decrease_dyn_delay_factor(self, url):
        host = self.__get_host(url)
        host['dyn_delay_factor'] = max(0, host['dyn_delay_factor'] - 1)
        host['dyn_delay_changed'] = DT.datetime.utcnow()

    def get_delay_sec(self, url):
        """Get delay seconds for the url (based on host)."""
        dyn_delay_factor = self.__get_dyn_delay_factor(url)<commit_msg>let dyn_delay increasing less sensitivity<commit_after>"""Maintain host infos."""

import asyncio
import collections
import random
import datetime as DT
import urllib.parse as UP


class HostPool:
    """Maintain host infos."""

    dyn_delay_table = {  # dyn_delay_factor -> second
        0: 0,
        1: 5,
        2: 10,
        3: 20,
        4: 30,
        5: 40,
        6: 50,
        7: 60,
        8: 90,
        9: 120,
        10: 180,
        11: 240,
        12: 300,
        13: 600,
        14: 900,
        15: 1200,
        16: 1500,
        17: 1800,
        18: 2100,
        19: 2400,
        20: 3600,
    }

    dyn_delay_inc_sensitivity = DT.timedelta(seconds=10)

    def __init__(self, config, loop):
        """Init host infos."""
        self.config = config
        self.loop = loop

        self.hosts = collections.defaultdict(self.__default_host_info)

    def __default_host_info(self):
        return {
            'dyn_delay_factor': 0,
            'dyn_delay_changed': DT.datetime.utcnow(),
            'semaphore': asyncio.Semaphore(
                value=self.config.per_host_concurrent,
                loop=self.loop),
        }

    def __get_host(self, url):
        netloc = UP.urlparse(url).netloc

        return self.hosts[netloc]

    def __get_dyn_delay_factor(self, url):
        host = self.__get_host(url)

        return host.get('dyn_delay_factor')

    def __increase_dyn_delay_factor(self, url):
        host = self.__get_host(url)

        now = DT.datetime.utcnow()
        active_time = (host['dyn_delay_changed']
                       + self.dyn_delay_inc_sensitivity)

        if now > active_time:
            host['dyn_delay_factor'] = min(20, host['dyn_delay_factor'] + 1)
            host['dyn_delay_changed'] = DT.datetime.utcnow()

    def __decrease_dyn_delay_factor(self, url):
        host = self.__get_host(url)

        host['dyn_delay_factor'] = max(0, host['dyn_delay_factor'] - 1)
        host['dyn_delay_changed'] = DT.datetime.utcnow()

    def get_delay_sec(self, url):
        """Get delay seconds for the url (based on host)."""
        dyn_delay_factor = self.__get_dyn_delay_factor(url)
{'fraction_non_alphanumeric': 0.11375921375921376, 'fraction_numerical': 0.047665847665847666, 'mean_word_length': 2.0403286034353996, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2336470', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1308, 'n_words': 295}