<commit_before># -*- coding: utf-8 -*-


def _is_tokenized(word):
    """
    Runs some checks to see if the word is tokenized.
    Note: if this functions returns True doesn't mean is really tokenized, but
    if returns False you know it's not tokenized propperly.
    """

    # FIXME: add harder checks
    return not ((word.endswith(".") or word.endswith(",")) and
                word[:-1].isalpha())


class Sentence(list):
    def __init__(self, iterable=None, position=None):
        if position is not None:
            if not(0 <= position <= 1):
                raise ValueError("Position must be between 0 and 1")
            self.position = position
        if iterable is not None:
            super(Sentence, self).__init__(iterable)
        else:
            super(Sentence, self).__init__()

    def check_is_tokenized(self):
        message = u"Word {!r} is not tokenized"
        for word in self:
            if not _is_tokenized(word):
                raise ValueError(message.format(word))

    def to_text(self):
        return ' '.join(self).encode('utf-8')


class SentencePair(list):
    def __init__(self, sentence_a, sentence_b, aligned=None):
        super(SentencePair, self).__init__([sentence_a, sentence_b])
        self.a = sentence_a
        self.b = sentence_b
        self.aligned = aligned


class ScoreFunction(object):
    def __init__(self, min_bound, max_bound):<commit_msg>removed position<commit_after># -*- coding: utf-8 -*-


def _is_tokenized(word):
    """
    Runs some checks to see if the word is tokenized.
    Note: if this functions returns True doesn't mean is really tokenized, but
    if returns False you know it's not tokenized propperly.
    """
    # FIXME: add harder checks
    return not ((word.endswith(".") or word.endswith(",")) and
                word[:-1].isalpha())


class Sentence(list):
    def __init__(self, iterable=None):
        if iterable is not None:
            super(Sentence, self).__init__(iterable)
        else:
            super(Sentence, self).__init__()

    def check_is_tokenized(self):
        message = u"Word {!r} is not tokenized"
        for word in self:
            if not _is_tokenized(word):
                raise ValueError(message.format(word))

    def to_text(self):
        return ' '.join(self).encode('utf-8')


class SentencePair(list):
    def __init__(self, sentence_a, sentence_b, aligned=None):
        super(SentencePair, self).__init__([sentence_a, sentence_b])
        self.a = sentence_a
        self.b = sentence_b
        self.aligned = aligned


class ScoreFunction(object):
    def __init__(self, min_bound, max_bound):
{'fraction_non_alphanumeric': 0.11127933383800151, 'fraction_numerical': 0.003785011355034065, 'mean_word_length': 2.557200538358008, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5406800', 'n_tokens_mistral': 806, 'n_tokens_neox': 778, 'n_words': 230}