<commit_before>from struct import pack, unpack

def varlenDecode(b):
	if b[0] == b'\xff':
		return (unpack('<Q', b[1:9])[0], b[9:])
	if b[0] == b'\xfe':
		return (unpack('<L', b[1:5])[0], b[5:])
	if b[0] == b'\xfd':
		return (unpack('<H', b[1:3])[0], b[3:])
	return (b[0], b[1:])

def varlenEncode(n):
	if n < 0xfd:
		return pack('<B', n)
	if n < 0xffff:
		return '\xfd' + pack('<H', n)
	if n < 0xffffffff:
		return '\xfe' + pack('<L', n)
	return pack('<Q', n)<commit_msg>Bugfix: varlenDecode/varlenEncode still epic failed; finally fix this bugger, and add unit tests to make sure<commit_after>from struct import pack, unpack

def varlenDecode(b):
	if b[0] == 0xff:
		return (unpack('<Q', b[1:9])[0], b[9:])
	if b[0] == 0xfe:
		return (unpack('<L', b[1:5])[0], b[5:])
	if b[0] == 0xfd:
		return (unpack('<H', b[1:3])[0], b[3:])
	return (b[0], b[1:])

def varlenEncode(n):
	if n < 0xfd:
		return pack('<B', n)
	if n <= 0xffff:
		return b'\xfd' + pack('<H', n)
	if n <= 0xffffffff:
		return b'\xfe' + pack('<L', n)
	return b'\xff' + pack('<Q', n)

# tests
def _test():
	assert b'\0' == varlenEncode(0)
	assert b'\xfc' == varlenEncode(0xfc)
	assert b'\xfd\xfd\0' == varlenEncode(0xfd)
	assert b'\xfd\xff\xff' == varlenEncode(0xffff)
	assert b'\xfe\0\0\1\0' == varlenEncode(0x10000)
	assert b'\xfe\xff\xff\xff\xff' == varlenEncode(0xffffffff)
	assert b'\xff\0\0\0\0\1\0\0\0' == varlenEncode(0x100000000)
	assert b'\xff\xff\xff\xff\xff\xff\xff\xff\xff' == varlenEncode(0xffffffffffffffff)

_test()
{'fraction_non_alphanumeric': 0.23643670462156732, 'fraction_numerical': 0.05291359678499665, 'mean_word_length': 4.335714285714285, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1824773', 'n_tokens_mistral': 716, 'n_tokens_neox': 676, 'n_words': 155}