<commit_before>import tempfile
import pickle
import os
import sys
import time
import zipfile
import inspect
import binaryninja as binja
from binaryninja.binaryview import BinaryViewType, BinaryView
from binaryninja.filemetadata import FileMetadata, SaveSettings
from binaryninja.datarender import DataRenderer
from binaryninja.function import InstructionTextToken, DisassemblyTextLine
from binaryninja.enums import InstructionTextTokenType, SaveOption
import subprocess
import re


# Dear people from the future: If you're adding tests or debuging an
#  issue where python2 and python3 are producing different output
#  for the same function and it's a issue of `longs`, run the output
#  through this function.  If it's a unicode/bytes issue, fix it in
#  api/python/
def fixOutput(outputList):
    # Apply regular expression to detect python2 longs
    splitList = []
    for elem in outputList:
        if isinstance(elem, str):
            splitList.append(re.split(r"((?<=[\[ ])0x[\da-f]+L|[\d]+L)", elem))
        else:
            splitList.append(elem)

    # Resolve application of regular expression
    result = []
    for elem in splitList:
        if isinstance(elem, list):
            newElem = []
            for item in elem:
                if len(item) > 1 and item[-1] == 'L':
                    newElem.append(item[:-1])
                else:
                    newElem.append(item)
            result.append(''.join(newElem))
        else:
            result.append(elem)
    return result


# Alright so this one is here for Binja functions that output <in set([blah, blah, blah])>
def fixSet(string):
    # Apply regular expression
    splitList = (re.split(r"((?<=<in set\(\[).*(?=\]\)>))", string))
    if len(splitList) > 1:
        return splitList[0] + ', '.join(sorted(splitList[1].split(', '))) + splitList[2]
    else:
        return string


def fixStrRepr(string):
    # Python 2 and Python 3 represent Unicode character reprs differently
    return string.replace(b"\xe2\x80\xa6".decode("utf8"), "\\xe2\\x80\\xa6")

def get_file_list(test_store_rel):
    test_store = os.path.join(os.path.dirname(__file__), test_store_rel)
    all_files = []
    for root, _, files in os.walk(test_store):
        for file in files:
            all_files.append(os.path.join(root, file))
    return all_files

def remove_low_confidence(type_string):
    low_confidence_types = ["int32_t", "void"]
    for lct in low_confidence_types:
        type_string = type_string.replace(lct + " ", '')  # done to resolve confidence ties
    return type_string

class Builder(object):
    def __init__(self, test_store):
        self.test_store = test_store
        # binja.log.log_to_stdout(binja.LogLevel.DebugLog)  # Uncomment for more info

    def methods(self):
        methodnames = []
        for methodname, _ in inspect.getmembers(self, predicate=inspect.ismethod):
            if methodname.startswith("test_"):
                methodnames.append(methodname)
        return methodnames

    def unpackage_file(self, filename):
        path = os.path.join(os.path.dirname(__file__), self.test_store, filename)
        if not os.path.exists(path):
            with zipfile.ZipFile(path + ".zip", "r") as zf:
                zf.extractall(path = os.path.dirname(__file__))
        assert os.path.exists(path)
        return os.path.relpath(path)

    def delete_package(self, filename):
        path = os.path.join(os.path.dirname(__file__), self.test_store, filename)
        os.unlink(path)

class BinaryViewTestBuilder(Builder):
    """ The BinaryViewTestBuilder is for test that are verified against a binary.
        The tests are first run on your dev machine to base line then run again
        on the build machine to verify they are correct.

         - Function that are tests should start with 'test_'
         - Function doc string used as 'on error' message
         - Should return: list of strings
    """
    def __init__(self, filename, options=None):
        self.filename = os.path.join(os.path.dirname(__file__), filename)
        if options is None:
            self.bv = BinaryViewType.get_view_of_file(self.filename)
        else:
            self.bv = BinaryViewType.get_view_of_file_with_options(self.filename, options=options)
        if self.bv is None:
            print("%s is not an executable format" % filename)
            return

    def test_available_types(self):
        """Available types don't match"""
        return ["Available Type: " + x.name for x in BinaryView(FileMetadata()).open(self.filename).available_view_types]

    def test_function_starts(self):
        """Function starts list doesnt match"""
        result = []
        for x in self.bv.functions:
            result.append("Function start: " + hex(x.start))
        return fixOutput(result)

    def test_function_symbol_names(self):
        """Function.symbol.name list doesnt match"""
        result = []
        for x in self.bv.functions:
            result.append("Symbol: " + x.symbol.name + ' ' + str(x.symbol.type) + ' ' + hex(x.symbol.address) + ' ' + str(x.symbol.namespace))
        return fixOutput(result)

    def test_function_can_return(self):
        """Function.can_return list doesnt match"""
        result = []
        for x in self.bv.functions:
            result.append("function name: " + x.symbol.name + ' type: ' + str(x.symbol.type) + ' address: ' + hex(x.symbol.address) + ' can_return: ' + str(bool(x.can_return)))
        return fixOutput(result)

    def test_function_basic_blocks(self):
        """Function basic_block list doesnt match (start, end, has_undetermined_outgoing_edges)"""
        bblist = []
        for func in self.bv.functions:
            for bb in func.basic_blocks:
                bblist.append("basic block {} start: ".format(str(bb)) + hex(bb.start) + ' end: ' + hex(bb.end) + ' undetermined outgoing edges: ' + str(bb.has_undetermined_outgoing_edges) + ' incoming edges: ' + str(bb.incoming_edges) + ' outgoing edges: ' + str(bb.outgoing_edges))
                for anno in func.get_block_annotations(bb.start):
                    bblist.append("basic block {} function annotation: ".format(str(bb)) + str(anno))
                bblist.append("basic block {} test get self: ".format(str(bb)) + str(func.get_basic_block_at(bb.start)))
        return fixOutput(bblist)

    def test_function_low_il_basic_blocks(self):
        """Function low_il_basic_block list doesnt match"""
        ilbblist = []
        for func in self.bv.functions:
            for bb in func.low_level_il.basic_blocks:
                ilbblist.append("LLIL basic block {} start: ".format(str(bb)) + hex(bb.start) + ' end: ' + hex(bb.end) + ' outgoing edges: ' + str(len(bb.outgoing_edges)))
        return fixOutput(ilbblist)

    def test_function_med_il_basic_blocks(self):
        """Function med_il_basic_block list doesn't match"""
        ilbblist = []
        for func in self.bv.functions:
            for bb in func.mlil.basic_blocks:
                ilbblist.append("MLIL basic block {} start: ".format(str(bb)) + hex(bb.start) + ' end: ' + hex(bb.end) + ' outgoing_edges: ' + str(len(bb.outgoing_edges)))
        return fixOutput(ilbblist)

    def test_symbols(self):
        """Symbols list doesn't match"""
        return ["Symbol: " + str(i) for i in sorted(self.bv.symbols)]

    def test_symbol_namespaces(self):
        """Symbol namespaces don't match"""
        return self.bv.namespaces

    def test_internal_external_namespaces(self):
        """Symbol namespaces don't match"""
        return [BinaryView.internal_namespace(), BinaryView.external_namespace()]

    def test_strings(self):
        """Strings list doesn't match"""
        return fixOutput(["String: " + str(x.value) + ' type: ' + str(x.type) + ' at: ' + hex(x.start) for x in self.bv.strings])

    def test_low_il_instructions(self):
        """LLIL instructions produced different output"""
        retinfo = []
        for func in self.bv.functions:
            for bb in func.low_level_il.basic_blocks:
                for ins in bb:
                    retinfo.append("Function: {:x} Instruction: {:x} LLIL->MLIL: {}".format(func.start, ins.address, str(ins.mlil)))
                    retinfo.append("Function: {:x} Instruction: {:x} LLIL->MLILS: {}".format(func.start, ins.address, str(sorted(list(map(str, ins.mlils))))))
                    # TODO figure out the LLIL->HLIL instability
                    #retinfo.append("Function: {:x} Instruction: {:x} LLIL->HLIL: {}".format(func.start, ins.address, str(ins.hlil)))
                    retinfo.append("Function: {:x} Instruction: {:x} Mapped MLIL: {}".format(func.start, ins.address, str(ins.mapped_medium_level_il)))
                    retinfo.append("Function: {:x} Instruction: {:x} Value: {}".format(func.start, ins.address, str(ins.value)))
                    retinfo.append("Function: {:x} Instruction: {:x} Possible Values: {}".format(func.start, ins.address, str(ins.possible_values)))

                    prefixList = []
                    for i in ins.prefix_operands:<commit_msg>Add unit tests for 'get_llils_at' and 'get_lifted_ils_at' APIs.<commit_after>import tempfile
import pickle
import os
import sys
import time
import zipfile
import inspect
import binaryninja as binja
from binaryninja.binaryview import BinaryViewType, BinaryView
from binaryninja.filemetadata import FileMetadata, SaveSettings
from binaryninja.datarender import DataRenderer
from binaryninja.function import InstructionTextToken, DisassemblyTextLine
from binaryninja.enums import InstructionTextTokenType, SaveOption
import subprocess
import re


# Dear people from the future: If you're adding tests or debuging an
#  issue where python2 and python3 are producing different output
#  for the same function and it's a issue of `longs`, run the output
#  through this function.  If it's a unicode/bytes issue, fix it in
#  api/python/
def fixOutput(outputList):
    # Apply regular expression to detect python2 longs
    splitList = []
    for elem in outputList:
        if isinstance(elem, str):
            splitList.append(re.split(r"((?<=[\[ ])0x[\da-f]+L|[\d]+L)", elem))
        else:
            splitList.append(elem)

    # Resolve application of regular expression
    result = []
    for elem in splitList:
        if isinstance(elem, list):
            newElem = []
            for item in elem:
                if len(item) > 1 and item[-1] == 'L':
                    newElem.append(item[:-1])
                else:
                    newElem.append(item)
            result.append(''.join(newElem))
        else:
            result.append(elem)
    return result


# Alright so this one is here for Binja functions that output <in set([blah, blah, blah])>
def fixSet(string):
    # Apply regular expression
    splitList = (re.split(r"((?<=<in set\(\[).*(?=\]\)>))", string))
    if len(splitList) > 1:
        return splitList[0] + ', '.join(sorted(splitList[1].split(', '))) + splitList[2]
    else:
        return string


def fixStrRepr(string):
    # Python 2 and Python 3 represent Unicode character reprs differently
    return string.replace(b"\xe2\x80\xa6".decode("utf8"), "\\xe2\\x80\\xa6")

def get_file_list(test_store_rel):
    test_store = os.path.join(os.path.dirname(__file__), test_store_rel)
    all_files = []
    for root, _, files in os.walk(test_store):
        for file in files:
            all_files.append(os.path.join(root, file))
    return all_files

def remove_low_confidence(type_string):
    low_confidence_types = ["int32_t", "void"]
    for lct in low_confidence_types:
        type_string = type_string.replace(lct + " ", '')  # done to resolve confidence ties
    return type_string

class Builder(object):
    def __init__(self, test_store):
        self.test_store = test_store
        # binja.log.log_to_stdout(binja.LogLevel.DebugLog)  # Uncomment for more info

    def methods(self):
        methodnames = []
        for methodname, _ in inspect.getmembers(self, predicate=inspect.ismethod):
            if methodname.startswith("test_"):
                methodnames.append(methodname)
        return methodnames

    def unpackage_file(self, filename):
        path = os.path.join(os.path.dirname(__file__), self.test_store, filename)
        if not os.path.exists(path):
            with zipfile.ZipFile(path + ".zip", "r") as zf:
                zf.extractall(path = os.path.dirname(__file__))
        assert os.path.exists(path)
        return os.path.relpath(path)

    def delete_package(self, filename):
        path = os.path.join(os.path.dirname(__file__), self.test_store, filename)
        os.unlink(path)

class BinaryViewTestBuilder(Builder):
    """ The BinaryViewTestBuilder is for test that are verified against a binary.
        The tests are first run on your dev machine to base line then run again
        on the build machine to verify they are correct.

         - Function that are tests should start with 'test_'
         - Function doc string used as 'on error' message
         - Should return: list of strings
    """
    def __init__(self, filename, options=None):
        self.filename = os.path.join(os.path.dirname(__file__), filename)
        if options is None:
            self.bv = BinaryViewType.get_view_of_file(self.filename)
        else:
            self.bv = BinaryViewType.get_view_of_file_with_options(self.filename, options=options)
        if self.bv is None:
            print("%s is not an executable format" % filename)
            return

    def test_available_types(self):
        """Available types don't match"""
        return ["Available Type: " + x.name for x in BinaryView(FileMetadata()).open(self.filename).available_view_types]

    def test_function_starts(self):
        """Function starts list doesnt match"""
        result = []
        for x in self.bv.functions:
            result.append("Function start: " + hex(x.start))
        return fixOutput(result)

    def test_function_symbol_names(self):
        """Function.symbol.name list doesnt match"""
        result = []
        for x in self.bv.functions:
            result.append("Symbol: " + x.symbol.name + ' ' + str(x.symbol.type) + ' ' + hex(x.symbol.address) + ' ' + str(x.symbol.namespace))
        return fixOutput(result)

    def test_function_can_return(self):
        """Function.can_return list doesnt match"""
        result = []
        for x in self.bv.functions:
            result.append("function name: " + x.symbol.name + ' type: ' + str(x.symbol.type) + ' address: ' + hex(x.symbol.address) + ' can_return: ' + str(bool(x.can_return)))
        return fixOutput(result)

    def test_function_basic_blocks(self):
        """Function basic_block list doesnt match (start, end, has_undetermined_outgoing_edges)"""
        bblist = []
        for func in self.bv.functions:
            for bb in func.basic_blocks:
                bblist.append("basic block {} start: ".format(str(bb)) + hex(bb.start) + ' end: ' + hex(bb.end) + ' undetermined outgoing edges: ' + str(bb.has_undetermined_outgoing_edges) + ' incoming edges: ' + str(bb.incoming_edges) + ' outgoing edges: ' + str(bb.outgoing_edges))
                for anno in func.get_block_annotations(bb.start):
                    bblist.append("basic block {} function annotation: ".format(str(bb)) + str(anno))
                bblist.append("basic block {} test get self: ".format(str(bb)) + str(func.get_basic_block_at(bb.start)))
        return fixOutput(bblist)

    def test_function_low_il_basic_blocks(self):
        """Function low_il_basic_block list doesnt match"""
        ilbblist = []
        for func in self.bv.functions:
            for bb in func.low_level_il.basic_blocks:
                ilbblist.append("LLIL basic block {} start: ".format(str(bb)) + hex(bb.start) + ' end: ' + hex(bb.end) + ' outgoing edges: ' + str(len(bb.outgoing_edges)))
        return fixOutput(ilbblist)

    def test_function_med_il_basic_blocks(self):
        """Function med_il_basic_block list doesn't match"""
        ilbblist = []
        for func in self.bv.functions:
            for bb in func.mlil.basic_blocks:
                ilbblist.append("MLIL basic block {} start: ".format(str(bb)) + hex(bb.start) + ' end: ' + hex(bb.end) + ' outgoing_edges: ' + str(len(bb.outgoing_edges)))
        return fixOutput(ilbblist)

    def test_symbols(self):
        """Symbols list doesn't match"""
        return ["Symbol: " + str(i) for i in sorted(self.bv.symbols)]

    def test_symbol_namespaces(self):
        """Symbol namespaces don't match"""
        return self.bv.namespaces

    def test_internal_external_namespaces(self):
        """Symbol namespaces don't match"""
        return [BinaryView.internal_namespace(), BinaryView.external_namespace()]

    def test_strings(self):
        """Strings list doesn't match"""
        return fixOutput(["String: " + str(x.value) + ' type: ' + str(x.type) + ' at: ' + hex(x.start) for x in self.bv.strings])

    def test_low_il_instructions(self):
        """LLIL instructions produced different output"""
        retinfo = []
        for func in self.bv.functions:
            for bb in func.low_level_il.basic_blocks:
                for ins in bb:
                    retinfo.append("Function: {:x} Instruction: {:x} ADDR->LiftedILS: {}".format(func.start, ins.address, str(sorted(list(map(str, func.get_lifted_ils_at(ins.address)))))))
                    retinfo.append("Function: {:x} Instruction: {:x} ADDR->LLILS: {}".format(func.start, ins.address, str(sorted(list(map(str, func.get_llils_at(ins.address)))))))
                    retinfo.append("Function: {:x} Instruction: {:x} LLIL->MLIL: {}".format(func.start, ins.address, str(ins.mlil)))
                    retinfo.append("Function: {:x} Instruction: {:x} LLIL->MLILS: {}".format(func.start, ins.address, str(sorted(list(map(str, ins.mlils))))))
                    # TODO figure out the LLIL->HLIL instability
                    #retinfo.append("Function: {:x} Instruction: {:x} LLIL->HLIL: {}".format(func.start, ins.address, str(ins.hlil)))
                    retinfo.append("Function: {:x} Instruction: {:x} Mapped MLIL: {}".format(func.start, ins.address, str(ins.mapped_medium_level_il)))
                    retinfo.append("Function: {:x} Instruction: {:x} Value: {}".format(func.start, ins.address, str(ins.value)))
                    retinfo.append("Function: {:x} Instruction: {:x} Possible Values: {}".format(func.start, ins.address, str(ins.possible_values)))

                    prefixList = []
                    for i in ins.prefix_operands:
{'fraction_non_alphanumeric': 0.12395984323830998, 'fraction_numerical': 0.0025769044934772106, 'mean_word_length': 2.7962094966374567, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3906437', 'n_tokens_mistral': 5769, 'n_tokens_neox': 5473, 'n_words': 1396}