<commit_before># -*- coding: utf-8 -*-

from analyzer import analyse
from opcodes import opcodes
from re import match

import inspect
from binascii import hexlify

asm65_tokens = [
    dict(type='T_INSTRUCTION', regex=r'^(ADC|AND|ASL|BCC|BCS|BEQ|BIT|BMI|BNE|BPL|BRK|BVC|BVS|CLC|CLD|CLI|CLV|CMP|CPX|CPY|DEC|DEX|DEY|EOR|INC|INX|INY|JMP|JSR|LDA|LDX|LDY|LSR|NOP|ORA|PHA|PHP|PLA|PLP|ROL|ROR|RTI|RTS|SBC|SEC|SED|SEI|STA|STX|STY|TAX|TAY|TSX|TXA|TXS|TYA)', store=True),
    dict(type='T_ADDRESS', regex=r'\$([\dA-F]{2,4})', store=True),
    dict(type='T_NUMBER', regex=r'\#\$?([\dA-F]{2})', store=True),
    dict(type='T_SEPARATOR', regex=r'^,', store=True),
    dict(type='T_REGISTER', regex=r'^(X|Y)', store=True),
    dict(type='T_OPEN', regex=r'^\(', store=True),
    dict(type='T_CLOSE', regex=r'^\)', store=True),
    dict(type='T_LABEL', regex=r'^[a-z][a-z\d]*', store=True),
    dict(type='T_FUNCTION', regex=r'^\.[a-z]+', store=True),
    dict(type='T_NUM', regex=r'^[\d]+', store=True), #TODO
    dict(type='T_ENDLINE', regex=r'^\n', store=True),
    dict(type='T_WHITESPACE', regex=r'^[ \t]+', store=False),
    dict(type='T_COMMENT', regex=r'^;[^\n]*', store=False)
]

def t_endline (tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_ENDLINE':
        return True
    return False

def t_instruction (tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_INSTRUCTION':
        return True
    return False

def t_zeropage (tokens,index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_ADDRESS' and len(token['value']) == 3:
        return True
    return False

def t_address(tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_ADDRESS': #and len(token['value']) == 5:
        return True
    return False

def t_number(tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_NUMBER': #and len(token['value']) == 5:
        return True
    return False

def t_separator(tokens , index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_SEPARATOR' and token['value'] == ',':
        return True
    return False

def t_register_x(tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_REGISTER' and token['value'] == 'X':
        return True
    return False

def t_register_y(tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_REGISTER' and token['value'] == 'Y':
        return True
    return False

def t_open(tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_OPEN' and token['value'] == '(':
        return True
    return False

def t_close(tokens, index):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == 'T_CLOSE' and token['value'] == ')':
        return True
    return False

asm65_bnf = [
    dict(type='S_IMMEDIATE', short='imm', bnf=[t_instruction, t_number]),
    dict(type='S_ZEROPAGE_X', short='zpx', bnf=[t_instruction, t_zeropage, t_separator, t_register_x]),
    dict(type='S_ZEROPAGE_Y', short='zpy', bnf=[t_instruction, t_zeropage, t_separator, t_register_y]),
    dict(type='S_ZEROPAGE', short='zp', bnf=[t_instruction, t_zeropage]),
    dict(type='S_ABSOLUTE_X', short='absx', bnf=[t_instruction, t_address, t_separator, t_register_x]),
    dict(type='S_ABSOLUTE_Y', short='absy', bnf=[t_instruction, t_address, t_separator, t_register_y]),
    dict(type='S_ABSOLUTE', short='abs', bnf=[t_instruction, t_address]),<commit_msg>refactoring on the look_ahead functions<commit_after># -*- coding: utf-8 -*-

from analyzer import analyse
from opcodes import opcodes
from re import match

import inspect
from binascii import hexlify

asm65_tokens = [
    dict(type='T_INSTRUCTION', regex=r'^(ADC|AND|ASL|BCC|BCS|BEQ|BIT|BMI|BNE|BPL|BRK|BVC|BVS|CLC|CLD|CLI|CLV|CMP|CPX|CPY|DEC|DEX|DEY|EOR|INC|INX|INY|JMP|JSR|LDA|LDX|LDY|LSR|NOP|ORA|PHA|PHP|PLA|PLP|ROL|ROR|RTI|RTS|SBC|SEC|SED|SEI|STA|STX|STY|TAX|TAY|TSX|TXA|TXS|TYA)', store=True),
    dict(type='T_ADDRESS', regex=r'\$([\dA-F]{2,4})', store=True),
    dict(type='T_NUMBER', regex=r'\#\$?([\dA-F]{2})', store=True),
    dict(type='T_SEPARATOR', regex=r'^,', store=True),
    dict(type='T_REGISTER', regex=r'^(X|Y)', store=True),
    dict(type='T_OPEN', regex=r'^\(', store=True),
    dict(type='T_CLOSE', regex=r'^\)', store=True),
    dict(type='T_LABEL', regex=r'^[a-z][a-z\d]*', store=True),
    dict(type='T_FUNCTION', regex=r'^\.[a-z]+', store=True),
    dict(type='T_NUM', regex=r'^[\d]+', store=True), #TODO
    dict(type='T_ENDLINE', regex=r'^\n', store=True),
    dict(type='T_WHITESPACE', regex=r'^[ \t]+', store=False),
    dict(type='T_COMMENT', regex=r'^;[^\n]*', store=False)
]

def look_ahead(tokens, index, type, value = None):
    if index > len(tokens) - 1:
        return False
    token = tokens[index]
    if token['type'] == type:
        if value == None or token['value'] == value:
            return True
    return False

def t_endline (tokens, index):
    return look_ahead(tokens, index, 'T_ENDLINE')

def t_instruction (tokens, index):
    return look_ahead(tokens, index, 'T_INSTRUCTION')

def t_zeropage (tokens,index):
    lh = look_ahead(tokens, index, 'T_ADDRESS')
    if lh and len(tokens[index]['value']) == 3:
        return True
    return False

def t_address(tokens, index):
    return look_ahead(tokens, index, 'T_ADDRESS')

def t_number(tokens, index):
    return look_ahead(tokens, index, 'T_NUMBER')

def t_separator(tokens , index):
    return look_ahead(tokens, index, 'T_SEPARATOR')

def t_register_x(tokens, index):
    return look_ahead(tokens, index, 'T_REGISTER', 'X')

def t_register_y(tokens, index):
    return look_ahead(tokens, index, 'T_REGISTER', 'Y')

def t_open(tokens, index):
    return look_ahead(tokens, index, 'T_OPEN', '(')

def t_close(tokens, index):
    return look_ahead(tokens, index, 'T_CLOSE', ')')

asm65_bnf = [
    dict(type='S_IMMEDIATE', short='imm', bnf=[t_instruction, t_number]),
    dict(type='S_ZEROPAGE_X', short='zpx', bnf=[t_instruction, t_zeropage, t_separator, t_register_x]),
    dict(type='S_ZEROPAGE_Y', short='zpy', bnf=[t_instruction, t_zeropage, t_separator, t_register_y]),
    dict(type='S_ZEROPAGE', short='zp', bnf=[t_instruction, t_zeropage]),
    dict(type='S_ABSOLUTE_X', short='absx', bnf=[t_instruction, t_address, t_separator, t_register_x]),
    dict(type='S_ABSOLUTE_Y', short='absy', bnf=[t_instruction, t_address, t_separator, t_register_y]),
    dict(type='S_ABSOLUTE', short='abs', bnf=[t_instruction, t_address]),
{'fraction_non_alphanumeric': 0.18241979571284708, 'fraction_numerical': 0.004459789958279384, 'mean_word_length': 4.508716323296355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1308616', 'n_tokens_mistral': 2738, 'n_tokens_neox': 2651, 'n_words': 516}