<commit_before>from __future__ import print_function
import sys
import os
import networkx
import itertools
from binascii import unhexlify
from heapq import heappush, heappop
from xml.sax.saxutils import escape as xmlEscape
from HTMLParser import HTMLParser
from mako.template import Template
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtWebKit import *
from PyQt5.QtWebKitWidgets import *
import main


BAD_ADDR = 0xffffffffffffffff


class MLStripper(HTMLParser):
    '''http://stackoverflow.com/questions/753052'''
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def stripHtmlTags(html):
    """strip HTML tags from a string (not secure)"""
    s = MLStripper()
    s.feed(html)
    return s.get_data()


class MyBlock(object):
    def __init__(self, core, ops, endOp):
        '''(endOp causes the end of the block, but might not be the
            last op due to delay slots)'''

        self.core = core
        self.ops = ops
        self.endOp = endOp

    @property
    def addr(self):
        return self.ops[0]['addr']

    @property
    def jump(self):
        return self.endOp.get('jump')

    @property
    def fail(self):
        return self.endOp.get('fail')

    @property
    def asmOps(self):
        for op in self.ops:
            addr = op['addr']
            asmOp = self.core.getAsmOp(addr)
            hexBytes = self.core.analyzeOp(addr)['bytes']
            yield (addr, hexBytes, asmOp)

    @property
    def labelName(self):
        return 'blk_{0:x}'.format(self.addr)

    @staticmethod
    def isEndBlockOp(op):
        return ('jmp' in op['type']
            or 'call' in op['type']
            or 'ret' in op['type']
            or 'trap' in op['type'])

    @staticmethod
    def _makeMyBlockAt(core, addr):
        ops = []
        endOp = None

        opsLeft = -1
        while opsLeft != 0:
            op = core.analyzeOp(addr)
            if not op:
                # error
                return None

            ops.append(op)

            if MyBlock.isEndBlockOp(op):
                endOp = op
                # +1 including this one
                opsLeft = op['delay'] + 1

            addr += op['size']
            if opsLeft > 0:
                opsLeft -= 1

        return MyBlock(core, ops, endOp)

    @staticmethod
    def _makeFuncBlocks(core, funcAddr):<commit_msg>Don't end block on call.<commit_after>from __future__ import print_function
import sys
import os
import networkx
import itertools
from binascii import unhexlify
from heapq import heappush, heappop
from xml.sax.saxutils import escape as xmlEscape
from HTMLParser import HTMLParser
from mako.template import Template
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtWebKit import *
from PyQt5.QtWebKitWidgets import *
import main


BAD_ADDR = 0xffffffffffffffff


class MLStripper(HTMLParser):
    '''http://stackoverflow.com/questions/753052'''
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def stripHtmlTags(html):
    """strip HTML tags from a string (not secure)"""
    s = MLStripper()
    s.feed(html)
    return s.get_data()


class MyBlock(object):
    def __init__(self, core, ops, endOp):
        '''(endOp causes the end of the block, but might not be the
            last op due to delay slots)'''

        self.core = core
        self.ops = ops
        self.endOp = endOp

    @property
    def addr(self):
        return self.ops[0]['addr']

    @property
    def jump(self):
        return self.endOp.get('jump')

    @property
    def fail(self):
        return self.endOp.get('fail')

    @property
    def asmOps(self):
        for op in self.ops:
            addr = op['addr']
            asmOp = self.core.getAsmOp(addr)
            hexBytes = self.core.analyzeOp(addr)['bytes']
            yield (addr, hexBytes, asmOp)

    @property
    def labelName(self):
        return 'blk_{0:x}'.format(self.addr)

    @staticmethod
    def isEndBlockOp(op):
        return ('jmp' in op['type']
            or 'ret' in op['type']
            or 'trap' in op['type'])

    @staticmethod
    def _makeMyBlockAt(core, addr):
        ops = []
        endOp = None

        opsLeft = -1
        while opsLeft != 0:
            op = core.analyzeOp(addr)
            if not op:
                # error
                return None

            ops.append(op)

            if MyBlock.isEndBlockOp(op):
                endOp = op
                # +1 including this one
                opsLeft = op['delay'] + 1

            addr += op['size']
            if opsLeft > 0:
                opsLeft -= 1

        return MyBlock(core, ops, endOp)

    @staticmethod
    def _makeFuncBlocks(core, funcAddr):
{'fraction_non_alphanumeric': 0.10406243746247748, 'fraction_numerical': 0.008004802881729037, 'mean_word_length': 2.051282051282051, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4576813', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1526, 'n_words': 435}