<commit_before>import logging
import string
import sys
import os

INPUT_FILE_NAME = "weka-output.txt"

def convertDifficulties():

	f = open(os.path.join(sys.path[0], INPUT_FILE_NAME))

	if not f:
		logging.error("Couldn't find file " + INPUT_FILE_NAME)
		return

	result = {}

	for line in f:
		line = string.lstrip(line)
		if len(line) == 0:
			continue
		if line[0] != "+" and line[0] != "-":
			#All lines we're looking for start with either + or -
			continue
		negative = line[0] == "-"
		line = string.lstrip(line[1:])
		parts = line.split(" * ")
		if len(parts) > 2:
			logging.error("Skipped line " + line + " because it was not shaped the way we expected.")
		
		if len(parts) == 1:
			name = "Constant"
		else:
			name = string.strip(parts[1])

		if negative:
			parts[0] = "-" + parts[0]

		result[name] = float(parts[0])

	print("//auto-generated by difficulty-convert.py DO NOT EDIT")
	print("package sudoku")
	print("func init() {")

	print("\tdifficultySignalWeights = map[string]float64{")

	for key,val in result.items():
		print("\t\t\"" + key + "\" : " + str(val) + ",")

	print("\t}")
	print("}")



convertDifficulties()<commit_msg>difficulty-convert.py now outputs directly to the file, since go generate doesn't like pipes<commit_after>import logging
import string
import sys
import os

INPUT_FILE_NAME = "weka-output.txt"
OUTPUT_FILE_NAME = "../hs_difficulty_weights.go"

def convertDifficulties():

	f = open(os.path.join(sys.path[0], INPUT_FILE_NAME))

	if not f:
		logging.error("Couldn't find file " + INPUT_FILE_NAME)
		return

	result = {}

	for line in f:
		line = string.lstrip(line)
		if len(line) == 0:
			continue
		if line[0] != "+" and line[0] != "-":
			#All lines we're looking for start with either + or -
			continue
		negative = line[0] == "-"
		line = string.lstrip(line[1:])
		parts = line.split(" * ")
		if len(parts) > 2:
			logging.error("Skipped line " + line + " because it was not shaped the way we expected.")
		
		if len(parts) == 1:
			name = "Constant"
		else:
			name = string.strip(parts[1])

		if negative:
			parts[0] = "-" + parts[0]

		result[name] = float(parts[0])

	f.close()

	o = open(os.path.abspath(os.path.join(sys.path[0], OUTPUT_FILE_NAME)), "w")

	o.write("//auto-generated by difficulty-convert.py DO NOT EDIT\n\n")
	o.write("package sudoku\n\n")
	o.write("func init() {\n")

	o.write("\tdifficultySignalWeights = map[string]float64{\n")

	for key,val in result.items():
		o.write("\t\t\"" + key + "\" : " + str(val) + ",\n")

	o.write("\t}\n")
	o.write("}\n")

	o.close()

convertDifficulties()
{'fraction_non_alphanumeric': 0.17484423676012462, 'fraction_numerical': 0.011292834890965732, 'mean_word_length': 3.5711743772241995, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '894489', 'n_tokens_mistral': 1040, 'n_tokens_neox': 942, 'n_words': 231}