<commit_before>import binascii
import glob
import natsort
import struct

data = list()
for a in natsort.natsorted(glob.glob('./packetdata/pdata*')):
	with open(a, 'rb') as f:
		data.append(f.read())

data = [x for x in data if len(x) == 1367]
telemetryData = list()

for tele in data:
	packString  = "HB"
	packString += "B"
	packString += "bb"
	packString += "BBbBB"
	packString += "B"
	packString += "21f"
	packString += "H"
	packString += "B"
	packString += "B"
	packString += "hHhHHBBBBBbffHHBBbB"
	packString += "22f"
	packString += "8B12f8B8f12B4h20H16f4H"
	packString += "2f"
	packString += "2B"
	packString += "bbBbbb"

	for participant in range(56):
		packString += "hhhHBBBBf"
	
	packString += "fBBB"

	telemetryData.append(struct.unpack(packString, tele))

with open('tele.csv', 'w') as f:
	for p in telemetryData:
		for x in p:
			if isinstance(x, list):
				for y in x:
					#f.write(",".join(str(binascii.hexlify(z)) for z in y)+",")
					f.write(",".join(str(z) for z in y)+",")
			else:
				#f.write(str(binascii.hexlify(x))+",")
				f.write(str(x)+",")
		f.write("\n")
<commit_msg>Added support for other packet types.<commit_after>import binascii
import glob
import natsort
import struct

data = list()
for a in natsort.natsorted(glob.glob('./packetdata/pdata*')):
	with open(a, 'rb') as f:
		data.append(f.read())

telemetryData = list()

for tele in data:
	if len(tele) == 1367:
		packString  = "HB"
		packString += "B"
		packString += "bb"
		packString += "BBbBB"
		packString += "B"
		packString += "21f"
		packString += "H"
		packString += "B"
		packString += "B"
		packString += "hHhHHBBBBBbffHHBBbB"
		packString += "22f"
		packString += "8B12f8B8f12B4h20H16f4H"
		packString += "2f"
		packString += "2B"
		packString += "bbBbbb"

		for participant in range(56):
			packString += "hhhHBBBBf"
		
		packString += "fBBB"

	elif len(tele) == 1028:
		packString  = "HBB"

		for participant in range(64):
			packString += "16s"
	
	elif len(tele) == 1347:
		packString  = "HB64s64s64s64s64s"

		for participant in range(64):
			packString += "16s"

	telemetryData.append(struct.unpack(packString, tele))

with open('tele.csv', 'w') as f:
	for p in telemetryData:
		f.write(",".join(str(x) for x in p)+"\n")

{'fraction_non_alphanumeric': 0.16681654676258992, 'fraction_numerical': 0.0341726618705036, 'mean_word_length': 4.011261261261262, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3771822', 'n_tokens_mistral': 1015, 'n_tokens_neox': 907, 'n_words': 219}