<commit_before># future
from __future__ import annotations

# stdlib
import socket
from typing import Any
from typing import Callable
from typing import Dict as TypeDict
from typing import List as TypeList
from typing import Optional
from typing import Tuple as TypeTuple
from typing import Union

# relative
from .lib import find_available_port


class BadGrammar(Exception):
    pass


class GrammarVerb:
    def __init__(
        self,
        command: str,
        full_sentence: TypeList[TypeDict[str, Any]],
        abbreviations: TypeDict[int, TypeList[Optional[str]]],
    ) -> None:
        self.grammar: TypeList[Union[GrammarTerm, HostGrammarTerm]] = []
        self.command = command
        self.full_sentence = full_sentence
        self.abbreviations = abbreviations

    def get_named_term_grammar(self, name: str) -> GrammarTerm:
        for term in self.grammar:
            if term.name == name and isinstance(term, GrammarTerm):
                return term
        raise BadGrammar(f"GrammarTerm with {name} not found in {self.grammar}")

    def get_named_term_hostgrammar(self, name: str) -> HostGrammarTerm:
        for term in self.grammar:
            if term.name == name and isinstance(term, HostGrammarTerm):
                return term
        raise BadGrammar(f"HostGrammarTerm with {name} not found in {self.grammar}")

    def get_named_term_type(
        self, name: str, term_type: Optional[str] = None
    ) -> Union[GrammarTerm, HostGrammarTerm]:
        if term_type == "host":
            return self.get_named_term_hostgrammar(name=name)
        return self.get_named_term_grammar(name=name)

    def set_named_term_type(
        self, name: str, new_term: GrammarTerm, term_type: Optional[str] = None
    ) -> None:
        new_grammar = []
        for term in self.grammar:
            found = False
            if term.name == name:
                if term_type is not None and term.type == term_type:
                    found = True
                elif term_type is None:
                    found = True
            if not found:
                new_grammar.append(term)
            else:
                new_grammar.append(new_term)
        self.grammar = new_grammar

    def load_grammar(
        self, grammar: TypeList[Union[GrammarTerm, HostGrammarTerm]]
    ) -> None:
        self.grammar = grammar


class GrammarTerm:
    def __init__(
        self,
        type: str,
        name: str,
        default: Optional[Union[str, Callable]] = None,
        options: TypeList = [],
        example: Optional[str] = None,
        **kwargs: TypeDict[str, Any],
    ) -> None:
        self.raw_input: Optional[str] = None
        self.input: Optional[str] = None
        self.type = type
        self.name = name
        self.default = default
        self.options = options
        self.example = example

    @property
    def snake_input(self) -> Optional[str]:
        if self.input:
            return self.input.lower().replace(" ", "_")
        return None

    def __repr__(self) -> str:
        return f"<{type(self).__name__}: {self.name}<{self.type}>: {self.input} [raw: {self.raw_input}]>"

    def get_example(self) -> str:
        return_value = self.example if self.example else self.default
        if callable(return_value):
            return_value = return_value()
        return str(return_value)

    # no op
    def custom_parsing(self, input: str) -> str:
        return input

    def parse_input(self, input: Optional[str]) -> None:
        self.raw_input = input
        if input is None and self.default is None:
            raise BadGrammar(
                f"{self.name} has no default, please use one of the following options: {self.options}"
            )
        if input is None:
            if isinstance(self.default, str):
                input = self.default
            elif callable(self.default):
                input = self.default()

        if input not in self.options:
            raise BadGrammar(
                f"{input} is not valid for {self.name} please use one of the following options: {self.options}"
            )

        self.input = self.custom_parsing(input=input) if input else input


class HostGrammarTerm(GrammarTerm):
    @property
    def host(self) -> Optional[str]:
        return self.parts()[0]

    @property
    def port(self) -> Optional[int]:
        return self.parts()[1]

    @property
    def search(self) -> bool:
        return bool(self.parts()[2])

    @property
    def free_port(self) -> int:
        if self.port is None:
            raise BadGrammar(
                f"{type(self)} unable to check if port {self.port} is free"
            )
        return find_available_port(host="localhost", port=self.port, search=self.search)
<commit_msg>Fixed issue causing the tool to break because of MyPy change<commit_after># future
from __future__ import annotations

# stdlib
import socket
from typing import Any
from typing import Callable
from typing import Dict as TypeDict
from typing import List as TypeList
from typing import Optional
from typing import Tuple as TypeTuple
from typing import Union

# relative
from .lib import find_available_port


class BadGrammar(Exception):
    pass


class GrammarVerb:
    def __init__(
        self,
        command: str,
        full_sentence: TypeList[TypeDict[str, Any]],
        abbreviations: TypeDict[int, TypeList[Optional[str]]],
    ) -> None:
        self.grammar: TypeList[Union[GrammarTerm, HostGrammarTerm]] = []
        self.command = command
        self.full_sentence = full_sentence
        self.abbreviations = abbreviations

    def get_named_term_grammar(self, name: str) -> GrammarTerm:
        for term in self.grammar:
            if term.name == name and isinstance(term, GrammarTerm):
                return term
        raise BadGrammar(f"GrammarTerm with {name} not found in {self.grammar}")

    def get_named_term_hostgrammar(self, name: str) -> HostGrammarTerm:
        for term in self.grammar:
            if term.name == name and isinstance(term, HostGrammarTerm):
                return term
        raise BadGrammar(f"HostGrammarTerm with {name} not found in {self.grammar}")

    def get_named_term_type(
        self, name: str, term_type: Optional[str] = None
    ) -> Union[GrammarTerm, HostGrammarTerm]:
        if term_type == "host":
            return self.get_named_term_hostgrammar(name=name)
        return self.get_named_term_grammar(name=name)

    def set_named_term_type(
        self, name: str, new_term: GrammarTerm, term_type: Optional[str] = None
    ) -> None:
        new_grammar = []
        for term in self.grammar:
            found = False
            if term.name == name:
                if term_type is not None and term.type == term_type:
                    found = True
                elif term_type is None:
                    found = True
            if not found:
                new_grammar.append(term)
            else:
                new_grammar.append(new_term)
        self.grammar = new_grammar

    def load_grammar(
        self, grammar: TypeList[Union[GrammarTerm, HostGrammarTerm]]
    ) -> None:
        self.grammar = grammar


class GrammarTerm:
    def __init__(
        self,
        type: str,
        name: str,
        default: Optional[Union[str, Callable]] = None,
        options: TypeList = [],
        example: Optional[str] = None,
        **kwargs: TypeDict[str, Any],
    ) -> None:
        self.raw_input: Optional[str] = None
        self.input: Optional[str] = None
        self.type = type
        self.name = name
        self.default = default
        self.options = options
        self.example = example

    @property
    def snake_input(self) -> Optional[str]:
        if self.input:
            return self.input.lower().replace(" ", "_")
        return None

    def __repr__(self) -> str:
        return f"<{type(self).__name__}: {self.name}<{self.type}>: {self.input} [raw: {self.raw_input}]>"

    def get_example(self) -> str:
        return_value = self.example if self.example else self.default
        if callable(return_value):
            return_value = return_value()
        return str(return_value)

    # no op
    def custom_parsing(self, input: str) -> str:
        return input

    def parse_input(self, input: Optional[str]) -> None:
        self.raw_input = input
        if input is None and self.default is None:
            raise BadGrammar(
                f"{self.name} has no default, please use one of the following options: {self.options}"
            )
        if input is None:
            if isinstance(self.default, str):
                input = self.default
            elif callable(self.default):
                input = self.default()

        if len(self.options) > 0 and input not in self.options:
            raise BadGrammar(
                f"{input} is not valid for {self.name} please use one of the following options: {self.options}"
            )

        self.input = self.custom_parsing(input=input) if input else input


class HostGrammarTerm(GrammarTerm):
    @property
    def host(self) -> Optional[str]:
        return self.parts()[0]

    @property
    def port(self) -> Optional[int]:
        return self.parts()[1]

    @property
    def search(self) -> bool:
        return bool(self.parts()[2])

    @property
    def free_port(self) -> int:
        if self.port is None:
            raise BadGrammar(
                f"{type(self)} unable to check if port {self.port} is free"
            )
        return find_available_port(host="localhost", port=self.port, search=self.search)

{'fraction_non_alphanumeric': 0.09475324675324676, 'fraction_numerical': 0.0007272727272727272, 'mean_word_length': 2.189529489728297, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3810219', 'n_tokens_mistral': 2842, 'n_tokens_neox': 2754, 'n_words': 789}