<commit_before>import struct
from helpers import *

class MessageParser:
	"""
	Commonly used parser to encode and decode communication messages.
	"""

	def encode(self, type, params):
		"""
		Encodes a message.

		Args:
			type - the type of the message
			params - a dictonary of parameters that will be encoded separately

		Returns:
			Returns the hexadecimal encoded message size and the message itself as a utf-8 string.
		"""

		result = "type:%s;" % (type)
		for param, value in params.items():
			result = "%s%s:%s;" % (result, param, value)
		
		return struct.pack('>H', len(result)) +  b(result)

	def decode(self, message):
		"""
		Decodes a message.

		Args:
			message - a string representative of the message

		Returns:
			Returns the type of the message and the parameters as a dictonary.
		"""

		#size = struct.unpack('>h', message[:2])

		messageType = None
		params = {}<commit_msg>[common] Count bytes instead of chars in parser<commit_after>import struct
import sys
from helpers import *

class MessageParser:
	"""
	Commonly used parser to encode and decode communication messages.
	"""

	def encode(self, type, params):
		"""
		Encodes a message.

		Args:
			type - the type of the message
			params - a dictonary of parameters that will be encoded separately

		Returns:
			Returns the hexadecimal encoded message size and the message itself as a utf-8 string.
		"""

		result = "type:%s;" % (type)
		for param, value in params.items():
			result = "%s%s:%s;" % (result, param, value)

		return struct.pack('>H', sys.getsizeof(result)) +  b(result)

	def decode(self, message):
		"""
		Decodes a message.

		Args:
			message - a string representative of the message

		Returns:
			Returns the type of the message and the parameters as a dictonary.
		"""

		#size = struct.unpack('>h', message[:2])

		messageType = None
		params = {}
{'fraction_non_alphanumeric': 0.10931610123855681, 'fraction_numerical': 0.002154011847065159, 'mean_word_length': 3.4556354916067145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5795519', 'n_tokens_mistral': 629, 'n_tokens_neox': 548, 'n_words': 219}