<commit_before>from struct import Struct, error as StructError
from gzip import GzipFile
import zlib
from collections import MutableMapping, MutableSequence, Sequence
import os, io

TAG_END = 0
TAG_BYTE = 1
TAG_SHORT = 2
TAG_INT = 3
TAG_LONG = 4
TAG_FLOAT = 5
TAG_DOUBLE = 6
TAG_BYTE_ARRAY = 7
TAG_STRING = 8
TAG_LIST = 9
TAG_COMPOUND = 10
TAG_INT_ARRAY = 11

class MalformedFileError(Exception):
	"""Exception raised on parse error."""
	pass

class TAG(object):
	"""TAG, a variable with an intrinsic name."""
	id = None

	def __init__(self, value=None, name=None):
		self.name = name
		self.value = value

	#Parsers and Generators
	def _parse_buffer(self, buffer):
		raise NotImplementedError(self.__class__.__name__)

	def _render_buffer(self, buffer):
		raise NotImplementedError(self.__class__.__name__)

	#Printing and Formatting of tree
	def tag_info(self):
		return self.__class__.__name__ + \
				('(%r)' % self.name if self.name else "") + \
				": " + self.valuestr()
	def valuestr(self):
		return str(self.value)

	def pretty_tree(self, indent=0):
		return ("\t"*indent) + self.tag_info()
	
	def __str__(self):
		return str(self.value)
	# Unlike regular iterators, __repr__() is not recursive.
	# Use pretty_tree for recursive results.
	# iterators should use __repr__ or tag_info for each item, like regular iterators
	def __repr__(self):
		return "<%s(%r) at 0x%x>" % (self.__class__.__name__,self.name,id(self))

class _TAG_Numeric(TAG):
	"""_TAG_Numeric, comparable to int with an intrinsic name"""
	def __init__(self, value=None, name=None, buffer=None):
		super(_TAG_Numeric, self).__init__(value, name)
		if buffer:
			self._parse_buffer(buffer)

	#Parsers and Generators
	def _parse_buffer(self, buffer):
		self.value = self.fmt.unpack(buffer.read(self.fmt.size))[0]

	def _render_buffer(self, buffer):
		buffer.write(self.fmt.pack(self.value))

#== Value Tags ==#
class TAG_Byte(_TAG_Numeric):
	id = TAG_BYTE
	fmt = Struct(">b")

class TAG_Short(_TAG_Numeric):
	id = TAG_SHORT
	fmt = Struct(">h")
<commit_msg>Return unicode strings for valuestr(), and thus tag_info() and pretty_tree()<commit_after>"""Handle the NBT (Named Binary Tag) data format"""

from struct import Struct, error as StructError
from gzip import GzipFile
import zlib
from collections import MutableMapping, MutableSequence, Sequence
import os, io

TAG_END = 0
TAG_BYTE = 1
TAG_SHORT = 2
TAG_INT = 3
TAG_LONG = 4
TAG_FLOAT = 5
TAG_DOUBLE = 6
TAG_BYTE_ARRAY = 7
TAG_STRING = 8
TAG_LIST = 9
TAG_COMPOUND = 10
TAG_INT_ARRAY = 11

class MalformedFileError(Exception):
	"""Exception raised on parse error."""
	pass

class TAG(object):
	"""TAG, a variable with an intrinsic name."""
	id = None

	def __init__(self, value=None, name=None):
		self.name = name
		self.value = value

	#Parsers and Generators
	def _parse_buffer(self, buffer):
		raise NotImplementedError(self.__class__.__name__)

	def _render_buffer(self, buffer):
		raise NotImplementedError(self.__class__.__name__)

	#Printing and Formatting of tree
	def tag_info(self):
		"""Return Unicode string with class, name and unnested value"""
		return self.__class__.__name__ + \
				('(%r)' % self.name if self.name else "") + \
				": " + self.valuestr()
	def valuestr(self):
		"""Return Unicode string of unnested value. For iterators, this returns a summary."""
		return unicode(self.value)

	def pretty_tree(self, indent=0):
		"""Return formated Unicode string of self, where iterable items are recursively listed in detail."""
		return ("\t"*indent) + self.tag_info()
	
	def __str__(self):
		"""Return a string (ascii formated for Python 2, unicode for Python 3) with the result in human readable format. Unlike valuestr(), the result is recursive for iterators till at least one level deep."""
		return str(self.value)
	# Unlike regular iterators, __repr__() is not recursive.
	# Use pretty_tree for recursive results.
	# iterators should use __repr__ or tag_info for each item, like regular iterators
	def __repr__(self):
		"""Return a string (ascii formated for Python 2, unicode for Python 3) describing the class, name and id for debugging purposes."""
		return "<%s(%r) at 0x%x>" % (self.__class__.__name__,self.name,id(self))

class _TAG_Numeric(TAG):
	"""_TAG_Numeric, comparable to int with an intrinsic name"""
	def __init__(self, value=None, name=None, buffer=None):
		super(_TAG_Numeric, self).__init__(value, name)
		if buffer:
			self._parse_buffer(buffer)

	#Parsers and Generators
	def _parse_buffer(self, buffer):
		self.value = self.fmt.unpack(buffer.read(self.fmt.size))[0]

	def _render_buffer(self, buffer):
		buffer.write(self.fmt.pack(self.value))

#== Value Tags ==#
class TAG_Byte(_TAG_Numeric):
	id = TAG_BYTE
	fmt = Struct(">b")

class TAG_Short(_TAG_Numeric):
	id = TAG_SHORT
	fmt = Struct(">h")

{'fraction_non_alphanumeric': 0.11478843736908254, 'fraction_numerical': 0.007959782153330541, 'mean_word_length': 4.895061728395062, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2294890', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1568, 'n_words': 492}