<commit_before>#!/usr/bin/env python

import idalink
import pyvex
import logging
import symbolic
l = logging.getLogger("angr_binary")
l.setLevel(logging.DEBUG)

def ondemand(f):
	name = f.__name__
	def func(self, *args, **kwargs):
		if hasattr(self, "_" + name):
			return getattr(self, "_" + name)

		a = f(self, *args, **kwargs)
		setattr(self, "_" + name, a)
		return a
	func.__name__ = f.__name__
	return func

class Function(object):
	def __init__(self, func_start, ida):
		self.start = func_start
		self.ida = ida
		self.name = "sub_%x" % func_start

	@ondemand
	def range(self):
		starts, ends = [ ], [ ]
		l.debug("Getting range from IDA")

		flowchart = self.ida.idaapi.FlowChart(self.ida.idaapi.get_func(self.start))
		for block in flowchart:
			start, end = (block.startEA, block.endEA)
			starts.append(start)
			ends.append(end)

		r = ( min(starts), max(ends) )
		l.debug("Got range %s." % str(r))
		return r

	@ondemand
	def ida_blocks(self):
		l.debug("Getting blocks from IDA")
		ida_blocks = { }

		flowchart = self.ida.idaapi.FlowChart(self.ida.idaapi.get_func(self.start))
		for block in flowchart:
			start, end = (block.startEA, block.endEA)
			block_bytes = self.ida.idaapi.get_many_bytes(start, end - start)

			if not block_bytes:
				l.warning("... empty block_bytes at %x" % start)
				continue

			ida_blocks[(start, end)] = block_bytes
		return ida_blocks

	@ondemand
	def bytes(self):
		start, end = self.range()
		return self.ida.idaapi.get_many_bytes(start, end - start)

	@ondemand
	def vex_blocks(self):
		#for (s, e), b in ida_blocks().iteritems():
		#	self.make_vex_blocks(s, e, b)
		blocks = { }
		total_size = 0
		for start,sirsb in symbolic.translate_bytes(self.start, self.bytes(), self.start):
			irsb = sirsb.irsb

			size = irsb.size()
			total_size += size
			blocks[start] = irsb
			l.debug("Block at %x of size %d" % (start, irsb.size()))

		l.debug("Total VEX IRSB size, in bytes: %d" % total_size)
		return blocks

class Binary(object):
	def __init__(self, filename):
		self.filename = filename
		self.ida = idalink.make_idalink(filename)

	@ondemand
	def functions(self):
		functions = { }
		for f in self.ida.idautils.Functions():
			functions[f] = Function(f, self.ida)
		return functions<commit_msg>uses pysex now<commit_after>#!/usr/bin/env python

import idalink
import pyvex
import pysex
import logging
l = logging.getLogger("angr_binary")
l.setLevel(logging.DEBUG)

def ondemand(f):
	name = f.__name__
	def func(self, *args, **kwargs):
		if hasattr(self, "_" + name):
			return getattr(self, "_" + name)

		a = f(self, *args, **kwargs)
		setattr(self, "_" + name, a)
		return a
	func.__name__ = f.__name__
	return func

class Function(object):
	def __init__(self, func_start, ida):
		self.start = func_start
		self.ida = ida
		self.name = "sub_%x" % func_start

	@ondemand
	def range(self):
		starts, ends = [ ], [ ]
		l.debug("Getting range from IDA")

		flowchart = self.ida.idaapi.FlowChart(self.ida.idaapi.get_func(self.start))
		for block in flowchart:
			start, end = (block.startEA, block.endEA)
			starts.append(start)
			ends.append(end)

		r = ( min(starts), max(ends) )
		l.debug("Got range %s." % str(r))
		return r

	@ondemand
	def ida_blocks(self):
		l.debug("Getting blocks from IDA")
		ida_blocks = { }

		flowchart = self.ida.idaapi.FlowChart(self.ida.idaapi.get_func(self.start))
		for block in flowchart:
			start, end = (block.startEA, block.endEA)
			block_bytes = self.ida.idaapi.get_many_bytes(start, end - start)

			if not block_bytes:
				l.warning("... empty block_bytes at %x" % start)
				continue

			ida_blocks[(start, end)] = block_bytes
		return ida_blocks

	@ondemand
	def bytes(self):
		start, end = self.range()
		return self.ida.idaapi.get_many_bytes(start, end - start)

	@ondemand
	def vex_blocks(self):
		#for (s, e), b in ida_blocks().iteritems():
		#	self.make_vex_blocks(s, e, b)
		blocks = { }
		total_size = 0
		for start,sirsb in pysex.translate_bytes(self.start, self.bytes(), self.start):
			irsb = sirsb.irsb

			size = irsb.size()
			total_size += size
			blocks[start] = irsb
			l.debug("Block at %x of size %d" % (start, irsb.size()))

		l.debug("Total VEX IRSB size, in bytes: %d" % total_size)
		return blocks

class Binary(object):
	def __init__(self, filename):
		self.filename = filename
		self.ida = idalink.make_idalink(filename)

	@ondemand
	def functions(self):
		functions = { }
		for f in self.ida.idautils.Functions():
			functions[f] = Function(f, self.ida)
		return functions
{'fraction_non_alphanumeric': 0.14142761841227486, 'fraction_numerical': 0.00044474093840338, 'mean_word_length': 3.80042689434365, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5744377', 'n_tokens_mistral': 1868, 'n_tokens_neox': 1683, 'n_words': 399}