<commit_before>class AbstractCompilable(object):
    """
    An abstract class that defines an API to transform itself into other types
    of expressions.
    """
    def as_str(self):
        raise NotImplementedError

    def __eq__(self, other):
        if isinstance(other, str):
            return self.as_str() == other
        if isinstance(other, self.__class__):
            return self.__dict__ == other.__dict__
        else:
            return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(tuple(sorted(self.__dict__.items())))

    def __repr__(self):
        return '<%s %s>' % (self.__class__.__name__, repr(self.as_str()))


class Token(AbstractCompilable):

    def __init__(self, string):
        assert isinstance(string, str)
        self._string = string

    @property
    def string(self):
        return self._string

    def as_str(self):
        return self.string

    def __str__(self):
        return self.as_str()


class Reference(Token):
    def __init__(self, number, parent=None):
        super(Reference, self).__init__(number)
        assert isinstance(number, str)
        assert isinstance(parent, Token) or parent is None
        self._parent = parent
<commit_msg>Removed unused class.<commit_after>class Token(object):

    def __init__(self, string):
        assert isinstance(string, str)
        self._string = string

    @property
    def string(self):
        return self._string

    def as_str(self):
        return self.string

    def __str__(self):
        return self.as_str()

    def __eq__(self, other):
        if isinstance(other, str):
            return self.as_str() == other
        if isinstance(other, self.__class__):
            return self.__dict__ == other.__dict__
        else:
            return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(tuple(sorted(self.__dict__.items())))

    def __repr__(self):
        return '<%s %s>' % (self.__class__.__name__, repr(self.as_str()))


class Reference(Token):
    def __init__(self, number, parent=None):
        super(Reference, self).__init__(number)
        assert isinstance(number, str)
        assert isinstance(parent, Token) or parent is None
        self._parent = parent

{'fraction_non_alphanumeric': 0.10499359795134443, 'fraction_numerical': 0.0, 'mean_word_length': 2.5089820359281436, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1190811', 'n_tokens_mistral': 668, 'n_tokens_neox': 646, 'n_words': 179}