<commit_before>from io import BytesIO
from .enums import RuntimePlatform
from .resources import get_resource, STRINGS_DAT
from .utils import BinaryReader


class TypeTree:
	NULL = "(null)"

	def __init__(self, format):
		self.children = []
		self.version = 0
		self.is_array = False
		self.size = 0
		self.index = 0
		self.flags = 0
		self.type = self.NULL
		self.name = self.NULL
		self.format = format

	def __repr__(self):
		return "<%s %s (size=%r, index=%r, is_array=%r, flags=%r)>" % (
			self.type, self.name, self.size, self.index, self.is_array, self.flags
		)

	@property
	def post_align(self):
		return bool(self.flags & 0x4000)

	def load(self, buf):
		if self.format == 10 or self.format >= 12:
			self.load_blob(buf)
		else:
			self.load_old(buf)

	def load_old(self, buf):
		self.type = buf.read_string()
		self.name = buf.read_string()
		self.size = buf.read_int()
		self.index = buf.read_int()
		self.is_array = bool(buf.read_int())
		self.version = buf.read_int()
		self.flags = buf.read_int()

		num_fields = buf.read_int()
		for i in range(num_fields):
			tree = TypeTree(self.format)
			tree.load(buf)
			self.children.append(tree)

	def load_blob(self, buf):
		num_nodes = buf.read_uint()
		self.buffer_bytes = buf.read_uint()
		node_data = BytesIO(buf.read(24 * num_nodes))
		self.data = buf.read(self.buffer_bytes)

		parents = [self]

		buf = BinaryReader(node_data)

		for i in range(num_nodes):
			version = buf.read_int16()
			depth = buf.read_byte()

			if depth == 0:
				curr = self
			else:
				while len(parents) > depth:
					parents.pop()
				curr = TypeTree(self.format)
				parents[-1].children.append(curr)
				parents.append(curr)

			curr.version = version
			curr.is_array = buf.read_byte()
			curr.type = self.get_string(buf.read_int())
			curr.name = self.get_string(buf.read_int())
			curr.size = buf.read_int()
			curr.index = buf.read_uint()<commit_msg>Ensure depth and num_fields are unsigned to prevent pop from empty list<commit_after>from io import BytesIO
from .enums import RuntimePlatform
from .resources import get_resource, STRINGS_DAT
from .utils import BinaryReader


class TypeTree:
	NULL = "(null)"

	def __init__(self, format):
		self.children = []
		self.version = 0
		self.is_array = False
		self.size = 0
		self.index = 0
		self.flags = 0
		self.type = self.NULL
		self.name = self.NULL
		self.format = format

	def __repr__(self):
		return "<%s %s (size=%r, index=%r, is_array=%r, flags=%r)>" % (
			self.type, self.name, self.size, self.index, self.is_array, self.flags
		)

	@property
	def post_align(self):
		return bool(self.flags & 0x4000)

	def load(self, buf):
		if self.format == 10 or self.format >= 12:
			self.load_blob(buf)
		else:
			self.load_old(buf)

	def load_old(self, buf):
		self.type = buf.read_string()
		self.name = buf.read_string()
		self.size = buf.read_int()
		self.index = buf.read_int()
		self.is_array = bool(buf.read_int())
		self.version = buf.read_int()
		self.flags = buf.read_int()

		num_fields = buf.read_uint()
		for i in range(num_fields):
			tree = TypeTree(self.format)
			tree.load(buf)
			self.children.append(tree)

	def load_blob(self, buf):
		num_nodes = buf.read_uint()
		self.buffer_bytes = buf.read_uint()
		node_data = BytesIO(buf.read(24 * num_nodes))
		self.data = buf.read(self.buffer_bytes)

		parents = [self]

		buf = BinaryReader(node_data)

		for i in range(num_nodes):
			version = buf.read_int16()
			depth = buf.read_ubyte()

			if depth == 0:
				curr = self
			else:
				while len(parents) > depth:
					parents.pop()
				curr = TypeTree(self.format)
				parents[-1].children.append(curr)
				parents.append(curr)

			curr.version = version
			curr.is_array = buf.read_byte()
			curr.type = self.get_string(buf.read_int())
			curr.name = self.get_string(buf.read_int())
			curr.size = buf.read_int()
			curr.index = buf.read_uint()
{'fraction_non_alphanumeric': 0.13582166925868325, 'fraction_numerical': 0.009849663037843442, 'mean_word_length': 3.508177570093458, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5786454', 'n_tokens_mistral': 1629, 'n_tokens_neox': 1407, 'n_words': 312}