<commit_before>class HrefType:
    def __init__(self, name, handlers: list, inner_types=list()):
        self.name = name
        self.handlers = handlers
        self.inner_types = inner_types

    def set_url_prefix(self, url_prefix: str):
        handlers = [h for h in self.handlers if h.relative_href]
        for handler in handlers:
            handler.set_full_pattern(url_prefix)


class HrefHandler:
    parent_placeholder = '$parent'

    def __init__(self, path: str, pattern: str, value_key, relative_href=True, get_values_func=None, set_data_func=None):
        self.path = path.split('.')
        self.pattern = pattern.lstrip('/')
        self.value_keys = value_key if isinstance(value_key, list) else [value_key]
        self.relative_href = relative_href
        self.get_values_func = get_values_func or self._get_values
        self.set_data_func = set_data_func or self._set_data
        self.full_pattern = self.pattern

    def set_full_pattern(self, url_prefix: str):
        self.full_pattern = '{0}/{1}'.format(url_prefix.rstrip('/'), self.pattern)

    @staticmethod
    def _get_values(data: dict, parent_data: dict, value_keys: list):
        values = []
        for key in value_keys:
            if key.startswith(HrefHandler.parent_placeholder + '.'):
                value = parent_data.get(key[len(HrefHandler.parent_placeholder) + 1:])
            else:
                value = data.get(key)
            values.append(value)
        return values

    def get_values(self, data: dict, parent_data: dict):
        return self.get_values_func(data, parent_data, self.value_keys)

    @staticmethod
    def _set_data(data: dict, parent_data: dict, key: str, host: str, values: list, pattern: str):
        data[key] = '{0}/{1}'.format(host, pattern.lstrip('/').format(*values))

    def set_data(self, data: dict, parent_data: dict, key: str, host: str, values: list):
        if all([i is not None for i in values]):
            self.set_data_func(data, parent_data, key, host, values, self.full_pattern)


class HrefTypes:
    def __new__(cls, *args, **kwds):
        it = cls.__dict__.get("__it__")
        if it is None:
            cls.__it__ = it = object.__new__(cls)
            it.__init__()
            it.href_types = {}
        return it

    def __init__(self):
        pass

    def add_type(self, href_type: HrefType):
        if href_type.name in self.href_types:
            raise Exception('HrefType already exists: {0}'.format(href_type.name))
        self.href_types[href_type.name] = href_type

    def resolve(self, data, href_type_name: str, request):
        href_type = self.href_types[href_type_name]
        self._resolve(data, href_type, request)

    def _resolve(self, data, href_type: HrefType, request):
        """
        Resolves all hrefs of provided data using request's information
        :param data: list or dict being processed
        :param request: original http request containing the host information in order to build the hrefs
        """
        if isinstance(data, list):
            for i in data:
                self._resolve(i, href_type, request)
        elif isinstance(data, dict):
            # resolve hrefs
            for handler in href_type.handlers:
                for next_data, key in self.navigate(data, handler.path):
                    # calculate href and set it to the data
                    values = handler.get_values(next_data, data)
                    host = '{0}://{1}'.format(request.protocol, request.host)
                    handler.set_data(next_data, data, key, host, values)
            # resolve inners
            for path, inner_type in href_type.inner_types:
                inner_href_type = self.href_types[inner_type]
                for inner_data, key in self.navigate(data, path.split('.')):
                    if key in inner_data:
                        self._resolve(inner_data[key], inner_href_type, request)

    def navigate(self, data: dict, path: list):<commit_msg>performance improvement<commit_after>class HrefType:
    def __init__(self, name, handlers: list, inner_types=list()):
        self.name = name
        self.handlers = handlers
        self.inner_types = inner_types

    def set_url_prefix(self, url_prefix: str):
        handlers = [h for h in self.handlers if h.relative_href]
        for handler in handlers:
            handler.set_full_pattern(url_prefix)


class HrefHandler:
    parent_placeholder = '$parent'

    def __init__(self, path: str, pattern: str, value_key, relative_href=True, get_values_func=None, set_data_func=None):
        self.path = path.split('.')
        self.pattern = pattern.lstrip('/')
        self.value_keys = value_key if isinstance(value_key, list) else [value_key]
        self.relative_href = relative_href
        self.get_values_func = get_values_func or self._get_values
        self.set_data_func = set_data_func or self._set_data
        self.full_pattern = self.pattern

    def set_full_pattern(self, url_prefix: str):
        self.full_pattern = '{0}/{1}'.format(url_prefix.rstrip('/'), self.pattern)

    @staticmethod
    def _get_values(data: dict, parent_data: dict, value_keys: list):
        values = []
        for key in value_keys:
            if key.startswith(HrefHandler.parent_placeholder + '.'):
                value = parent_data.get(key[len(HrefHandler.parent_placeholder) + 1:])
            else:
                value = data.get(key)
            values.append(value)
        return values

    def get_values(self, data: dict, parent_data: dict):
        return self.get_values_func(data, parent_data, self.value_keys)

    @staticmethod
    def _set_data(data: dict, parent_data: dict, key: str, base_url: str, values: list, pattern: str):
        data[key] = '{0}/{1}'.format(base_url, pattern.lstrip('/').format(*values))

    def set_data(self, data: dict, parent_data: dict, key: str, base_url: str, values: list):
        if all([i is not None for i in values]):
            self.set_data_func(data, parent_data, key, base_url, values, self.full_pattern)


class HrefTypes:
    def __new__(cls, *args, **kwds):
        it = cls.__dict__.get("__it__")
        if it is None:
            cls.__it__ = it = object.__new__(cls)
            it.__init__()
            it.href_types = {}
        return it

    def __init__(self):
        pass

    def add_type(self, href_type: HrefType):
        if href_type.name in self.href_types:
            raise Exception('HrefType already exists: {0}'.format(href_type.name))
        self.href_types[href_type.name] = href_type

    def resolve(self, data, href_type_name: str, request):
        href_type = self.href_types[href_type_name]
        base_url = '{0}://{1}'.format(request.protocol, request.host)
        self._resolve(data, href_type, base_url)

    def _resolve(self, data, href_type: HrefType, base_url: str):
        """
        Resolves all hrefs of provided data using request's information
        :param data: list or dict being processed
        :param base_url: host base url
        """
        if isinstance(data, list):
            for i in data:
                self._resolve(i, href_type, base_url)
        elif isinstance(data, dict):
            # resolve hrefs
            for handler in href_type.handlers:
                for next_data, key in self.navigate(data, handler.path):
                    # calculate href and set it to the data
                    values = handler.get_values(next_data, data)
                    handler.set_data(next_data, data, key, base_url, values)
            # resolve inners
            for path, inner_type in href_type.inner_types:
                inner_href_type = self.href_types[inner_type]
                for inner_data, key in self.navigate(data, path.split('.')):
                    if key in inner_data:
                        self._resolve(inner_data[key], inner_href_type, base_url)

    def navigate(self, data: dict, path: list):
{'fraction_non_alphanumeric': 0.10315868638756581, 'fraction_numerical': 0.002005515166708448, 'mean_word_length': 2.4616052060737528, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5079176', 'n_tokens_mistral': 2419, 'n_tokens_neox': 2358, 'n_words': 611}