<commit_before>class XNode:
    def __init__(self):
        pass

class PTree(list):
    """
    """
    __slots__ = ['type', 'result', 'data']

    def __init__(self, type):
        super(PTree, self).__init__()
        self.type = type
        self.data = type
        self.result = None

    def eval(self, handle):
        if handle:
            self.result = handle(*self)

    def val(self):
        return self.result

# class Token(namedtuple('Token', ('data', 'type', 'value'))):
    # def val(self):
        # return self.value
    # 
    # def tlen(self):
        # return 1
# 
    # def clen(self):
        # return len(self.data)
# 
    # def __repr__(self):
        # return '%s(%s)' % (self.type.__name__, repr(self.data))

class Token:
    __slots__=['data', 'offset', 'type', 'value']

    def __init__(self, data, type=None, cast=None, offset=0):
        self.data = data
        self.value = cast(data) if cast else data
        self.type = type
        self.offset = offset

    def val(self):
        return self.value
    
    def tlen(self):
        return 1

    def clen(self):
        return len(self.data)

    def __repr__(self):
        return '%s(%s)' % (self.type.__name__, repr(self.data))

class TSeq(list):
    """
    This is meant to be returned by XNode's instances
    that extract strings from a given doc sequentially.
    """

    def __init__(self, *args):
        self.extend(args)

    def clen(self):
        count = 0
        for ind in self:
            count += ind.clen()
        return count

class TokType:
    @classmethod
    def validate(cls, slc):
        tok = slc.get()
        if tok and cls.istype(tok):
            return tok

    @classmethod
    def istype(cls, tok):
        return tok.type is cls

class TokVal:
    def __init__(self, data):
        self.data = data
        self.type = TokVal

    @classmethod
    def validate(slc):
        tok = slc.get()
        if tok and self.istype(tok):
            return tok

    def istype(self, tok):
        return self.data == tok.data

    def __repr__(self):
        return 'TokVal(%s)' % repr(self.data)

class Eof(TokType):
    pass

class Sof(TokType):
    pass<commit_msg>Fixing bug due to changings.<commit_after>class XNode:
    def __init__(self):
        pass

class PTree(list):
    """
    """
    __slots__ = ['type', 'result', 'data']

    def __init__(self, type):
        super(PTree, self).__init__()
        self.type = type
        self.data = type
        self.result = None

    def eval(self, handle):
        if handle:
            self.result = handle(*self)

    def val(self):
        return self.result

# class Token(namedtuple('Token', ('data', 'type', 'value'))):
    # def val(self):
        # return self.value
    # 
    # def tlen(self):
        # return 1
# 
    # def clen(self):
        # return len(self.data)
# 
    # def __repr__(self):
        # return '%s(%s)' % (self.type.__name__, repr(self.data))

class Token:
    __slots__=['data', 'offset', 'type', 'value']

    def __init__(self, data, type=None, cast=None, offset=0):
        self.data = data
        self.value = cast(data) if cast else data
        self.type = type
        self.offset = offset

    def val(self):
        return self.value
    
    def tlen(self):
        return 1

    def clen(self):
        return len(self.data)

    def __repr__(self):
        return '%s(%s)' % (self.type.__name__, repr(self.data))

class TSeq(list):
    """
    This is meant to be returned by XNode's instances
    that extract strings from a given doc sequentially.
    """

    def __init__(self, *args):
        self.extend(args)

    def clen(self):
        count = 0
        for ind in self:
            count += ind.clen()
        return count

class TokType:
    @classmethod
    def validate(cls, slc):
        tok = slc.get()
        if tok and cls.istype(tok):
            return tok

    @classmethod
    def istype(cls, tok):
        return tok.type is cls

class TokVal:
    def __init__(self, data):
        self.data = data
        self.type = TokVal

    def validate(self, slc):
        tok = slc.get()
        if tok and self.istype(tok):
            return tok

    def istype(self, tok):
        return self.data == tok.data

    def __repr__(self):
        return 'TokVal(%s)' % repr(self.data)

class Eof(TokType):
    pass

class Sof(TokType):
    pass
{'fraction_non_alphanumeric': 0.12796532846715328, 'fraction_numerical': 0.0018248175182481751, 'mean_word_length': 1.9911323328785813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453225', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1397, 'n_words': 387}