<commit_before>import string
from enum import Enum, auto

class Symbol(Enum):
    ERRORTOKEN = auto()
    BEGINMARKER = auto()
    ENDMARKER = auto()
    NEWLINE = auto()
    NAME = auto()
    NUMBER = auto()
    STRING = auto()
    LPAR = auto()
    RPAR = auto()
    LSQB = auto()
    RSQB = auto()
    LBRACE = auto()
    RBRACE = auto()
    COLON = auto()
    COMMA = auto()
    SEMI = auto()
    AT = auto()
    PLUS = auto()
    MINUS = auto()
    STAR = auto()
    SLASH = auto()
    DOUBLESLASH = auto()
    VBAR = auto()
    AMPER = auto()
    LESS = auto()
    GREATER = auto()
    EQUAL = auto()
    DOT = auto()
    PERCENT = auto()
    EQEQUAL = auto()
    NOTEQUAL = auto()
    LESSEQUAL = auto()
    GREATEREQUAL = auto()
    TILDE = auto()
    CIRCUMFLEX = auto()
    LEFTSHIFT = auto()
    RIGHTSHIFT = auto()
    DOUBLESTAR = auto()
    PLUSEQUAL = auto()
    MINEQUAL = auto()
    STAREQUAL = auto()
    SLASHEQUAL = auto()
    PERCENTEQUAL = auto()
    AMPEREQUAL = auto()
    VBAREQUAL = auto()
    CIRCUMFLEXEQUAL = auto()
    LEFTSHIFTEQUAL = auto()
    RIGHTSHIFTEQUAL = auto()
    DOUBLESTAREQUAL = auto()
    DOUBLESLASHEQUAL = auto()


class Token:
    def __init__(self, symbol, text, linenum, colnum):
        self.symbol = symbol
        self.text = text
        self.linenum = linenum
        self.colnum = colnum


    def __repr__(self):
        return f'<{self.__class__.__name__} {self.symbol} {self.text!r} {self.linenum} {self.colnum}>'


class Tokenizer:
    def __init__(self):
        pass

    def tokenize(self, text):
        tokens = []
        i = 0
        linenum = 0
        colnum = 0

        while i < len(text):
            c = text[i]

            if c in ('\n', '\r'):
                token = Token(Symbol.NEWLINE, text[i:i + 1], linenum, colnum)
                tokens.append(token)
                linenum += 1
                colnum = 0
                i += 1
            elif c in '_' + string.ascii_letters:
                j = i

                while c in '_' + string.ascii_letters + string.digits:
                    j += 1
                    c = text[j]

                token = Token(Symbol.NAME, text[i:j], linenum, colnum)
                tokens.append(token)
                colnum += j - i
                i = j
            elif c in string.digits:
                # FIXME: possible issue if something invalid is just behind number
                if text[i + 1] == 'x':
                    # hex
                    j = i + 2
                    c = text[j]

                    while c in string.hexdigits:
                        j += 1
                        c = text[j]
                elif text[i + 1] == 'o':
                    # octal
                    j = i + 2
                    c = text[j]

                    while c in '01234567':
                        j += 1
                        c = text[j]
                elif text[i + 1] == 'o':
                    # binary
                    j = i + 2
                    c = text[j]

                    while c in '01234567':
                        j += 1
                        c = text[j]

                token = Token(Symbol.NAME, text[i:j], linenum, colnum)
                tokens.append(token)
                colnum += j - i
                i = j
            else:
                colnum += 1
                i += 1

        return tokens


if __name__ == '__main__':
    text = '''
    _aaa aaa a_a_
    122 __a
    '''

    tokenizer = Tokenizer()
    tokens = tokenizer.tokenize(text)
    print('tokens:', tokens)<commit_msg>tokenize string, number, name<commit_after>import string
from enum import Enum, auto

class Symbol(Enum):
    ERRORTOKEN = auto()
    BEGINMARKER = auto()
    ENDMARKER = auto()
    NEWLINE = auto()
    NAME = auto()
    NUMBER = auto()
    STRING = auto()
    LPAR = auto()
    RPAR = auto()
    LSQB = auto()
    RSQB = auto()
    LBRACE = auto()
    RBRACE = auto()
    COLON = auto()
    COMMA = auto()
    SEMI = auto()
    AT = auto()
    PLUS = auto()
    MINUS = auto()
    STAR = auto()
    SLASH = auto()
    DOUBLESLASH = auto()
    VBAR = auto()
    AMPER = auto()
    LESS = auto()
    GREATER = auto()
    EQUAL = auto()
    DOT = auto()
    PERCENT = auto()
    EQEQUAL = auto()
    NOTEQUAL = auto()
    LESSEQUAL = auto()
    GREATEREQUAL = auto()
    TILDE = auto()
    CIRCUMFLEX = auto()
    LEFTSHIFT = auto()
    RIGHTSHIFT = auto()
    DOUBLESTAR = auto()
    PLUSEQUAL = auto()
    MINEQUAL = auto()
    STAREQUAL = auto()
    SLASHEQUAL = auto()
    PERCENTEQUAL = auto()
    AMPEREQUAL = auto()
    VBAREQUAL = auto()
    CIRCUMFLEXEQUAL = auto()
    LEFTSHIFTEQUAL = auto()
    RIGHTSHIFTEQUAL = auto()
    DOUBLESTAREQUAL = auto()
    DOUBLESLASHEQUAL = auto()


class Token:
    def __init__(self, symbol, text, linenum, colnum):
        self.symbol = symbol
        self.text = text
        self.linenum = linenum
        self.colnum = colnum


    def __repr__(self):
        return f'<{self.__class__.__name__} {self.symbol} {self.text!r} {self.linenum} {self.colnum}>'


class Tokenizer:
    def __init__(self):
        pass

    def tokenize(self, text):
        tokens = []
        i = 0
        linenum = 0
        colnum = 0

        while i < len(text):
            c = text[i]

            if c in ('\n', '\r'):
                token = Token(Symbol.NEWLINE, text[i:i + 1], linenum, colnum)
                tokens.append(token)
                linenum += 1
                colnum = 0
                i += 1
            elif c in '_' + string.ascii_letters:
                j = i

                while c in '_' + string.ascii_letters + string.digits:
                    j += 1
                    c = text[j]

                token = Token(Symbol.NAME, text[i:j], linenum, colnum)
                tokens.append(token)
                colnum += j - i
                i = j
            elif c in string.digits:
                # FIXME: possible issue if something invalid is just behind number
                if text[i + 1] == 'x':
                    # hex
                    j = i + 2
                    c = text[j]

                    while c in string.hexdigits:
                        j += 1
                        c = text[j]
                elif text[i + 1] == 'o':
                    # octal
                    j = i + 2
                    c = text[j]

                    while c in '01234567':
                        j += 1
                        c = text[j]
                elif text[i + 1] == 'o':
                    # binary
                    j = i + 2
                    c = text[j]

                    while c in '01':
                        j += 1
                        c = text[j]
                else:
                    # decimal
                    j = i + 2
                    c = text[j]

                    while c in string.digits:
                        j += 1
                        c = text[j]

                token = Token(Symbol.NUMBER, text[i:j], linenum, colnum)
                tokens.append(token)
                colnum += j - i
                i = j
            elif c == '\'':
                # ' string
                j = i + 1
                c = text[j]

                while c is not '\'':
                    if c == '\\':
                        j += 2
                    else:
                        j += 1

                    print(f'c: {c!r}')
                    c = text[j]

                j += 1

                token = Token(Symbol.STRING, text[i:j], linenum, colnum)
                tokens.append(token)
                colnum += j - i
                i = j
            else:
                colnum += 1
                i += 1

        return tokens


if __name__ == '__main__':
    text = r'''
    _aaa aaa a_a_
    122 __a 0x12f
    0b111 0o333
    123g 'asasa' 'frfr \' asda'
    '''

    tokenizer = Tokenizer()
    tokens = tokenizer.tokenize(text)
    print('tokens:', tokens)
{'fraction_non_alphanumeric': 0.11019180142910869, 'fraction_numerical': 0.011282437006393382, 'mean_word_length': 1.1144977471508084, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3170856', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2328, 'n_words': 657}