<commit_before>"""
Generating and interpreting numsed opcodes.
"""


import sys
import re
import dis
from StringIO import StringIO  # Python2
#from io import StringIO  # Python3
import transformer


# -- Disassemble -------------------------------------------------------------


def disassemble(source, trace=False):

     # compile
    with open(source) as f:
        script = f.read()

    code = compile(script, source, "exec")

    # disassemble
    old_stdout = sys.stdout
    result = StringIO()
    sys.stdout = result
    dis.dis(code)
    sys.stdout = old_stdout
    code = result.getvalue().splitlines()

    # trace if requested
    if trace:
        for instr in code:
            print instr

    # return list of instructions
    return code


# -- Disassemble to numsed opcodes -------------------------------------------


def make_opcode_module(source, trace=False):

    if 1 == 1:
        global BINARY_ADD, BINARY_MULTIPLY
        def BINARY_ADD(): return 'BINARY_ADD'
        def BINARY_MULTIPLY(): return 'BINARY_MULTIPLY'

    # transform to positive form
    transformer.transform(source, '~.py')

    # disassemble
    dis_code = disassemble('~.py', trace=False)

    # simplify dis code
    dis_code = prepared_dis_code(dis_code)

    # convert dis codes to numsed codes
    newcode3 = opcodes(dis_code, trace)

    # return list of instructions
    return newcode3


def opcodes(dis_code, trace=False):
    newcode = []
    newcode.append('STARTUP')

    # add dummy context to be removed by final RETURN_VALUE
    newcode.append('MAKE_CONTEXT')
    # add dummy pointer address to be taken by final RETURN_VALUE
    newcode.append('LOAD_CONST end_of_script')

    # normalize disassembly labels and opcode arguments
    newcode.extend(dis_code)

    # inline helper functions (is_positive, negative, divide_by_ten)
    newcode = inline_helper_opcodes(newcode)

    # handle function arguments and context
    tmp = []
    for instr in newcode:
        if instr.startswith('FUNCTION'):
            x = instr.split()
            name = x[1]
            args = x[2:]
            tmp.append(':%s' % name)
            tmp.append('MAKE_CONTEXT')
            # arguments are pushed first one first by native python compiler,
            # and they have to be popped in reverse order
            for arg in reversed(args):
                tmp.append('STORE_FAST %s' % arg)
        elif instr.startswith('RETURN_VALUE'):
            tmp.append('POP_CONTEXT')
            tmp.append(instr)
        else:
            tmp.append(instr)
    newcode = tmp

    # rename jump opcodes as all are absolute
    tmp = []
    for instr in newcode:
        if re.match('^JUMP_ABSOLUTE|JUMP_FORWARD', instr):
            x = instr.split()
            label = x[1]
            tmp.append('JUMP ' + label)
        else:
            tmp.append(instr)
    newcode = tmp

    # replace INPLACE_* with BINARY_ equivalent
    tmp = []
    for instr in newcode:
        instr = re.sub('^INPLACE_', 'BINARY_', instr)
        tmp.append(instr)
    newcode = tmp

    # clean long int representation (python2)
    tmp = []
    for instr in newcode:
        if re.match('LOAD_CONST +\d+L', instr):
            tmp.append(instr[:-1])
        else:
            tmp.append(instr)
    newcode = tmp

    # # TODO: ici ?
    # list_macros = ('BINARY_ADD', 'BINARY_SUBTRACT', 'BINARY_MULTIPLY')
    # newcode3 = normalize('\n'.join(newcode3), macros=list_macros)
    # newcode3 = newcode3.splitlines()

    # trace if requested
    if trace:
        for instr in newcode:
            print instr

    # return list of instructions
    return newcode


label_counter = 0
def new_label():
    global label_counter
    r = 'return%d' % label_counter
    label_counter += 1
    return r


# -- Reading opcode ----------------------------------------------------------


def read_opcode_module(source, trace=False):
    with open(source) as f:<commit_msg>Remove no more useful function.<commit_after>"""
Generating and interpreting numsed opcodes.
"""


import sys
import re
import dis
from StringIO import StringIO  # Python2
#from io import StringIO  # Python3
import transformer


# -- Disassemble -------------------------------------------------------------


def disassemble(source, trace=False):

     # compile
    with open(source) as f:
        script = f.read()

    code = compile(script, source, "exec")

    # disassemble
    old_stdout = sys.stdout
    result = StringIO()
    sys.stdout = result
    dis.dis(code)
    sys.stdout = old_stdout
    code = result.getvalue().splitlines()

    # trace if requested
    if trace:
        for instr in code:
            print instr

    # return list of instructions
    return code


# -- Disassemble to numsed opcodes -------------------------------------------


def make_opcode_module(source, trace=False):

    if 1 == 1:
        global BINARY_ADD, BINARY_MULTIPLY
        def BINARY_ADD(): return 'BINARY_ADD'
        def BINARY_MULTIPLY(): return 'BINARY_MULTIPLY'

    # transform to positive form
    transformer.transform(source, '~.py')

    # disassemble
    dis_code = disassemble('~.py', trace=False)

    # simplify dis code
    dis_code = prepared_dis_code(dis_code)

    # convert dis codes to numsed codes
    newcode3 = opcodes(dis_code, trace)

    # return list of instructions
    return newcode3


def opcodes(dis_code, trace=False):
    newcode = []
    newcode.append('STARTUP')

    # add dummy context to be removed by final RETURN_VALUE
    newcode.append('MAKE_CONTEXT')
    # add dummy pointer address to be taken by final RETURN_VALUE
    newcode.append('LOAD_CONST end_of_script')

    # normalize disassembly labels and opcode arguments
    newcode.extend(dis_code)

    # inline helper functions (is_positive, negative, divide_by_ten)
    newcode = inline_helper_opcodes(newcode)

    # handle function arguments and context
    tmp = []
    for instr in newcode:
        if instr.startswith('FUNCTION'):
            x = instr.split()
            name = x[1]
            args = x[2:]
            tmp.append(':%s' % name)
            tmp.append('MAKE_CONTEXT')
            # arguments are pushed first one first by native python compiler,
            # and they have to be popped in reverse order
            for arg in reversed(args):
                tmp.append('STORE_FAST %s' % arg)
        elif instr.startswith('RETURN_VALUE'):
            tmp.append('POP_CONTEXT')
            tmp.append(instr)
        else:
            tmp.append(instr)
    newcode = tmp

    # rename jump opcodes as all are absolute
    tmp = []
    for instr in newcode:
        if re.match('^JUMP_ABSOLUTE|JUMP_FORWARD', instr):
            x = instr.split()
            label = x[1]
            tmp.append('JUMP ' + label)
        else:
            tmp.append(instr)
    newcode = tmp

    # replace INPLACE_* with BINARY_ equivalent
    tmp = []
    for instr in newcode:
        instr = re.sub('^INPLACE_', 'BINARY_', instr)
        tmp.append(instr)
    newcode = tmp

    # clean long int representation (python2)
    tmp = []
    for instr in newcode:
        if re.match('LOAD_CONST +\d+L', instr):
            tmp.append(instr[:-1])
        else:
            tmp.append(instr)
    newcode = tmp

    # # TODO: ici ?
    # list_macros = ('BINARY_ADD', 'BINARY_SUBTRACT', 'BINARY_MULTIPLY')
    # newcode3 = normalize('\n'.join(newcode3), macros=list_macros)
    # newcode3 = newcode3.splitlines()

    # trace if requested
    if trace:
        for instr in newcode:
            print instr

    # return list of instructions
    return newcode


# -- Reading opcode ----------------------------------------------------------


def read_opcode_module(source, trace=False):
    with open(source) as f:
{'fraction_non_alphanumeric': 0.13219945883262466, 'fraction_numerical': 0.004123180002576988, 'mean_word_length': 2.0764962346413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6830074', 'n_tokens_mistral': 2337, 'n_tokens_neox': 2231, 'n_words': 644}