<commit_before>import sys
from queue import Queue

from rply import LexerGenerator, Token

REPL_CONTINUE = object()

INFIX_OPERATORS = {'OPPLUS',
                   'OPMINUS',
                   'OPTIMES',
                   'PERCENT',
                   'OPDIV',
                   'OPLEQ',
                   'OPGEQ',
                   'OPEQ',
                   'OPNEQ',
                   'OPLT',
                   'OPGT',
                   'OPAND',
                   'OPOR',
                   'OPIS',
                   'PIPELINE',
                   'PIPELINE_BIND',
                   'PIPELINE_FIRST',
                   'PIPELINE_FIRST_BIND',
                   'PIPELINE_SEND',
                   'PIPELINE_MULTI_SEND'}


def _set_keyword(token):
    if token.name == 'NAME':
        for rule in klg.rules:
            if rule.matches(token.value, 0):
                token.name = rule.name
                break


def mod_lex(lexer, repl_mode=False):
    paren_openers = {'LPAREN', 'LBRACE', 'LBRACK'}
    paren_closers = {'RPAREN', 'RBRACE', 'RBRACK'}

    token_queue = Queue()
    indent_level = [0]
    ignore_newline = False
    paren_level = 0
    tab_len = 4

    def handle_newline(token):
        text = token.getstr()
        indent_str = text.rsplit('\n', 1)[1]
        indent = indent_str.count(' ') + indent_str.count('\t') * tab_len
        if indent > indent_level[-1]:
            indent_level.append(indent)
            indent_token = Token('INDENT', indent_str)
            indent_token.source_pos = token.getsourcepos()
            token_queue.put(indent_token)
        else:
            while indent < indent_level[-1]:
                indent_level.pop()
                dedent_token = Token('DEDENT', indent_str)
                token_queue.put(dedent_token)
        return token

    for token in lexer:
        while not token_queue.empty():
            queued_token = token_queue.get()
            if queued_token.gettokentype() in paren_openers:
                paren_level += 1
            elif queued_token.gettokentype() in paren_closers:
                paren_level -= 1
            ignore_newline = (paren_level > 0)

            if queued_token.gettokentype() == 'NAME' and queued_token.getstr().startswith('&'):
                amp = Token('AMP', '&')
                amp.source_pos = queued_token.getsourcepos()
                comma = Token('COMMA', ',')
                amp.source_pos = queued_token.getsourcepos()
                name = Token('NAME', queued_token.getstr()[1:])
                name.source_pos = queued_token.getsourcepos()
                yield amp
                yield comma
                yield name
            else:
                yield queued_token

        if token.name == 'NAME':
            for rule in klg.rules:
                if rule.matches(token.value, 0):
                    token.name = rule.name
                    print(token)
                    print(token.gettokentype())
                    break
        if token.gettokentype() in INFIX_OPERATORS:
            print("TRUE!")
            ahead_token = next(lexer)
            print(ahead_token)
            if ahead_token.gettokentype() == 'NEWLINE':
                pass
            else:
                token_queue.put(ahead_token)
        elif token.gettokentype() == 'NEWLINE':
            try:
                ahead_token = next(lexer)
                _set_keyword(ahead_token)
            except StopIteration:
                ahead_token = None
            if not (ignore_newline or ((ahead_token is not None)
                                       and (ahead_token.gettokentype() in INFIX_OPERATORS))):
                yield handle_newline(token)
            if ahead_token is not None:
                token_queue.put(ahead_token)
            continue

        if token.gettokentype() in paren_openers:
            paren_level += 1
        elif token.gettokentype() in paren_closers:
            paren_level -= 1
        ignore_newline = (paren_level > 0)

        if token.gettokentype() == 'NAME' and token.getstr().startswith('&'):
            amp = Token('AMP', '&')
            amp.source_pos = token.getsourcepos()
            comma = Token('COMMA', ',')
            amp.source_pos = token.getsourcepos()
            name = Token('NAME', token.getstr()[1:])
            name.source_pos = token.getsourcepos()
            yield amp
            yield comma
            yield name
        else:
            yield token

    if repl_mode and len(indent_level) > 1:
        yield REPL_CONTINUE
    elif repl_mode and paren_level > 0:
        yield REPL_CONTINUE
    else:
        while not token_queue.empty():
            yield token_queue.get()

        while len(indent_level) > 1:
            indent_level.pop()
            yield Token('DEDENT', '')


def lex(input, repl_mode=False, debug=True):
    if True:
        print(list(mod_lex(lg.build().lex(input), repl_mode)), file=sys.stderr)

    return mod_lex(lg.build().lex(input), repl_mode)


lg = LexerGenerator()

lg.add('TQUOTE_STR', r'(?x)"""(?:|[^\\]|\\.|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8})*"""')<commit_msg>Oops\!<commit_after>import sys
from queue import Queue

from rply import LexerGenerator, Token

REPL_CONTINUE = object()

INFIX_OPERATORS = {'OPPLUS',
                   'OPMINUS',
                   'OPTIMES',
                   'PERCENT',
                   'OPDIV',
                   'OPLEQ',
                   'OPGEQ',
                   'OPEQ',
                   'OPNEQ',
                   'OPLT',
                   'OPGT',
                   'OPAND',
                   'OPOR',
                   'OPIS',
                   'PIPELINE',
                   'PIPELINE_BIND',
                   'PIPELINE_FIRST',
                   'PIPELINE_FIRST_BIND',
                   'PIPELINE_SEND',
                   'PIPELINE_MULTI_SEND'}


def _set_keyword(token):
    if token.name == 'NAME':
        for rule in klg.rules:
            if rule.matches(token.value, 0):
                token.name = rule.name
                break


def mod_lex(lexer, repl_mode=False):
    paren_openers = {'LPAREN', 'LBRACE', 'LBRACK'}
    paren_closers = {'RPAREN', 'RBRACE', 'RBRACK'}

    token_queue = Queue()
    indent_level = [0]
    ignore_newline = False
    paren_level = 0
    tab_len = 4

    def handle_newline(token):
        text = token.getstr()
        indent_str = text.rsplit('\n', 1)[1]
        indent = indent_str.count(' ') + indent_str.count('\t') * tab_len
        if indent > indent_level[-1]:
            indent_level.append(indent)
            indent_token = Token('INDENT', indent_str)
            indent_token.source_pos = token.getsourcepos()
            token_queue.put(indent_token)
        else:
            while indent < indent_level[-1]:
                indent_level.pop()
                dedent_token = Token('DEDENT', indent_str)
                token_queue.put(dedent_token)
        return token

    for token in lexer:
        while not token_queue.empty():
            queued_token = token_queue.get()
            if queued_token.gettokentype() in paren_openers:
                paren_level += 1
            elif queued_token.gettokentype() in paren_closers:
                paren_level -= 1
            ignore_newline = (paren_level > 0)

            if queued_token.gettokentype() == 'NAME' and queued_token.getstr().startswith('&'):
                amp = Token('AMP', '&')
                amp.source_pos = queued_token.getsourcepos()
                comma = Token('COMMA', ',')
                amp.source_pos = queued_token.getsourcepos()
                name = Token('NAME', queued_token.getstr()[1:])
                name.source_pos = queued_token.getsourcepos()
                yield amp
                yield comma
                yield name
            else:
                yield queued_token

        if token.name == 'NAME':
            for rule in klg.rules:
                if rule.matches(token.value, 0):
                    token.name = rule.name
                    break
        if token.gettokentype() in INFIX_OPERATORS:
            ahead_token = next(lexer)
            if ahead_token.gettokentype() == 'NEWLINE':
                pass
            else:
                token_queue.put(ahead_token)
        elif token.gettokentype() == 'NEWLINE':
            try:
                ahead_token = next(lexer)
                _set_keyword(ahead_token)
            except StopIteration:
                ahead_token = None
            if not (ignore_newline or ((ahead_token is not None)
                                       and (ahead_token.gettokentype() in INFIX_OPERATORS))):
                yield handle_newline(token)
            if ahead_token is not None:
                token_queue.put(ahead_token)
            continue

        if token.gettokentype() in paren_openers:
            paren_level += 1
        elif token.gettokentype() in paren_closers:
            paren_level -= 1
        ignore_newline = (paren_level > 0)

        if token.gettokentype() == 'NAME' and token.getstr().startswith('&'):
            amp = Token('AMP', '&')
            amp.source_pos = token.getsourcepos()
            comma = Token('COMMA', ',')
            amp.source_pos = token.getsourcepos()
            name = Token('NAME', token.getstr()[1:])
            name.source_pos = token.getsourcepos()
            yield amp
            yield comma
            yield name
        else:
            yield token

    if repl_mode and len(indent_level) > 1:
        yield REPL_CONTINUE
    elif repl_mode and paren_level > 0:
        yield REPL_CONTINUE
    else:
        while not token_queue.empty():
            yield token_queue.get()

        while len(indent_level) > 1:
            indent_level.pop()
            yield Token('DEDENT', '')


def lex(input, repl_mode=False, debug=True):
    if debug:
        print(list(mod_lex(lg.build().lex(input), repl_mode)), file=sys.stderr)

    return mod_lex(lg.build().lex(input), repl_mode)


lg = LexerGenerator()

lg.add('TQUOTE_STR', r'(?x)"""(?:|[^\\]|\\.|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8})*"""')
{'fraction_non_alphanumeric': 0.11173514631983447, 'fraction_numerical': 0.00571484875357178, 'mean_word_length': 1.5368657835541115, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1640697', 'n_tokens_mistral': 3170, 'n_tokens_neox': 2990, 'n_words': 591}