<commit_before>from gen.path import Path
import gen.smartdict as smartdict
import re
import gen.env as env

_IS_TITLE = re.compile(r'^[-\w]+(?: [-\w]+)*$')
def is_title(x):
    return bool(_IS_TITLE.match(x))

_IS_FILENAME = re.compile(r'^[_a-zA-Z0-9][-_a-zA-Z0-9]*$')
def is_filename(x):
    return bool(_IS_FILENAME.match(x))

_IS_DOMAIN = re.compile(r'^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?' 
                        r'(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$')
def is_domain(x):
    return bool(_IS_DOMAIN.match(x))

_IS_VERSION = re.compile(r'^[-+.A-Za-z0-9]+$')
def is_version(x):
    return bool(_IS_VERSION.match(x))

_IS_HASH = re.compile(r'^[0-9a-f]+$')
def is_hash(x):
    return bool(_IS_HASH.match(x))

_IS_CVAR = re.compile(r'^[-_A-Za-z0-9]+(?:\.[-_A-Za-z0-9]+)+')
def is_cvar(x):
    return bool(_IS_CVAR.match(x))

class Title(smartdict.Key):
    """A title environment variable.

    Must be a sequence of 'words' separated by individual spaces.
    Words may contain alphanumeric characterss, hyphens, and
    underscores.
    """

    def __init__(self, name, default=None):
        smartdict.Key.__init__(self, name)
        self._default = default

    def default(self, instance):
        if self._default is not None:
            return self.default_var(instance, self._default)
        smartdict.Key.default(self, instance)

    @staticmethod
    def check(value):
        if not isinstance(value, str):
            raise TypeError('invalid title')
        if not is_title(value):
            raise ValueError('invalid title')
        return value

class Filename(smartdict.Key):
    """A filename environment variable.

    Must be a sequence of alphanumeric characters, underscores, and
    hyphens.  A hyphen may not be the first character.  The default
    value is derived from another key by replacing invalid character
    sequences with an underscore.
    """

    def __init__(self, name, default=None):
        smartdict.Key.__init__(self, name)
        self._default = default

    def default(self, instance):
        if self._default is not None:
            value = self.default_var(instance, self._default)
            return re.sub('[^-A-Za-z0-9]+', '_', value)
        smartdict.Key.default(self, instance)

    @staticmethod
    def check(value):
        if not isinstance(value, str):
            raise TypeError('invalid filename')
        if not is_filename(value):
            raise ValueError('invalid filename')
        return value

class DomainName(smartdict.Key):
    """A domain name environment variable."""

    def check(self, value):
        if not isinstance(value, str):
            raise TypeError('invalid domain name')
        if not is_domain(value):
            raise ValueError('invalid domain name')
        return value

class Version(smartdict.Key):
    """A version number environment variable."""

    def check(self, value):
        if not isinstance(value, str):
            raise TypeError('invalid version')
        if not is_version(value):
            raise ValueError('invalid version')
        return value
<commit_msg>build: fix error in smart dict defaults<commit_after>from gen.path import Path
import gen.smartdict as smartdict
import re
import gen.env as env

_IS_TITLE = re.compile(r'^[-\w]+(?: [-\w]+)*$')
def is_title(x):
    return bool(_IS_TITLE.match(x))

_IS_FILENAME = re.compile(r'^[_a-zA-Z0-9][-_a-zA-Z0-9]*$')
def is_filename(x):
    return bool(_IS_FILENAME.match(x))

_IS_DOMAIN = re.compile(r'^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?' 
                        r'(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$')
def is_domain(x):
    return bool(_IS_DOMAIN.match(x))

_IS_VERSION = re.compile(r'^[-+.A-Za-z0-9]+$')
def is_version(x):
    return bool(_IS_VERSION.match(x))

_IS_HASH = re.compile(r'^[0-9a-f]+$')
def is_hash(x):
    return bool(_IS_HASH.match(x))

_IS_CVAR = re.compile(r'^[-_A-Za-z0-9]+(?:\.[-_A-Za-z0-9]+)+')
def is_cvar(x):
    return bool(_IS_CVAR.match(x))

class Title(smartdict.Key):
    """A title environment variable.

    Must be a sequence of 'words' separated by individual spaces.
    Words may contain alphanumeric characterss, hyphens, and
    underscores.
    """

    def __init__(self, name, default=None):
        smartdict.Key.__init__(self, name)
        self._default = default

    def default(self, instance):
        if self._default is not None:
            return self.default_var(instance, self._default)
        return smartdict.Key.default(self, instance)

    @staticmethod
    def check(value):
        if not isinstance(value, str):
            raise TypeError('invalid title')
        if not is_title(value):
            raise ValueError('invalid title')
        return value

class Filename(smartdict.Key):
    """A filename environment variable.

    Must be a sequence of alphanumeric characters, underscores, and
    hyphens.  A hyphen may not be the first character.  The default
    value is derived from another key by replacing invalid character
    sequences with an underscore.
    """

    def __init__(self, name, default=None):
        smartdict.Key.__init__(self, name)
        self._default = default

    def default(self, instance):
        if self._default is not None:
            value = self.default_var(instance, self._default)
            return re.sub('[^-A-Za-z0-9]+', '_', value)
        return smartdict.Key.default(self, instance)

    @staticmethod
    def check(value):
        if not isinstance(value, str):
            raise TypeError('invalid filename')
        if not is_filename(value):
            raise ValueError('invalid filename')
        return value

class DomainName(smartdict.Key):
    """A domain name environment variable."""

    def check(self, value):
        if not isinstance(value, str):
            raise TypeError('invalid domain name')
        if not is_domain(value):
            raise ValueError('invalid domain name')
        return value

class Version(smartdict.Key):
    """A version number environment variable."""

    def check(self, value):
        if not isinstance(value, str):
            raise TypeError('invalid version')
        if not is_version(value):
            raise ValueError('invalid version')
        return value

{'fraction_non_alphanumeric': 0.13428755226761016, 'fraction_numerical': 0.008362817626246381, 'mean_word_length': 2.8223724646588813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '70655', 'n_tokens_mistral': 1998, 'n_tokens_neox': 1951, 'n_words': 506}