<commit_before>import argparse
import csv
import struct
import os

version = 1

def get_byte_count(strings):
	byte_count = 0

	for string in strings:
		byte_count += len(string)

	return byte_count

def main():
	keys = dict()
	strings = []
	string_offsets = []
	languages = []

	parser = argparse.ArgumentParser()
	parser.add_argument('--input_file', dest='input_file', required=True)
	parser.add_argument('--output_dir', dest='output_dir', required=True)
	args = parser.parse_args()

	print args.input_file

	with open(args.input_file) as f:
		reader = csv.DictReader(f)

		languages = reader.fieldnames[1:] # omit first column, should be empty

		for row in reader:
			string_indices = []

			for language in languages:
				strings_index = -1

				# get string index
				try:
					strings_index = strings.index(row[language])
				except:
					strings_index = len(strings)
					strings.append(row[language] + '\0')

				string_indices.append(strings_index)

			keys[row['']] = string_indices

	# create output directory
	try:
		os.makedirs(args.output_dir)
	except OSError:
		if not os.path.isdir(args.output_dir):
			raise

	output_file = os.path.join(args.output_dir, os.path.splitext(os.path.basename(args.input_file))[0] + '.strings')

	with open(output_file, 'wb') as f:
		# magic, version, language count, key count<commit_msg>added some comments to stringpack.py<commit_after>import argparse
import csv
import struct
import os

version = 1

def get_byte_count(strings):
	byte_count = 0

	for string in strings:
		byte_count += len(string)

	return byte_count

def main():
	keys = dict()
	strings = []
	string_offsets = []
	languages = []

	parser = argparse.ArgumentParser()
	parser.add_argument('--input_file', dest='input_file', required=True)
	parser.add_argument('--output_dir', dest='output_dir', required=True)
	args = parser.parse_args()

	print args.input_file

	with open(args.input_file) as f:
		reader = csv.DictReader(f)

		# omit first column, should be empty
		languages = reader.fieldnames[1:]

		# traverse all keys
		for row in reader:
			string_indices = []

			# for each language
			for language in languages:
				strings_index = -1

				try:
					# attempt to find identical string
					strings_index = strings.index(row[language])
				except:
					# string is unique, append to strings list
					strings_index = len(strings)
					strings.append(row[language] + '\0')

				# append string index to indices list
				string_indices.append(strings_index)

			keys[row['']] = string_indices

	# create output directory
	try:
		os.makedirs(args.output_dir)
	except OSError:
		if not os.path.isdir(args.output_dir):
			raise

	output_file = os.path.join(args.output_dir, os.path.splitext(os.path.basename(args.input_file))[0] + '.strings')

	with open(output_file, 'wb') as f:
		# magic, version, language count, key count
{'fraction_non_alphanumeric': 0.11528998242530755, 'fraction_numerical': 0.00421792618629174, 'mean_word_length': 4.028268551236749, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1506635', 'n_tokens_mistral': 1071, 'n_tokens_neox': 931, 'n_words': 236}