<commit_before># get symbol tokens
f = open("symbols.txt")
syms = f.read().split()

# write symbols.h
f = open("symbols.h", "w")
f.write("#ifndef _SYMBOLS_H_\n")
f.write("#define _SYMBOLS_H_\n\n")
f.write("char* symbols[] = {\n")
for i in range(len(syms)):
    sym = syms[i]
    f.write("\t\"%s\",\n" % sym)
f.write("};\n\n")
for i in range(len(syms)):
    sym = syms[i]
    f.write("#define SYM_%s\t\t%d\n" % (sym.upper(), i))
f.write("\n#endif\n");

# write Symbols.cs
f = open("Symbols.cs", "w")
f.write("namespace KowhaiSymbols\n{\n")
f.write("    public static class Symbols\n    {\n")
f.write("        public static string[] Strings =\n        {\n")
for sym in syms:
    f.write("            \"%s\",\n" % sym)
f.write("        };\n");
f.write("        public enum Constants\n        {\n")
for sym in syms:
    f.write("            %s,\n" % sym)
f.write("        };\n    }\n}\n");

<commit_msg>symbol generator script now strips duplicates<commit_after># get symbol tokens
f = open("symbols.txt")
syms = f.read().split()

# strip duplicates
def check_exists(sym, syms):
    for s in syms:
        if s.lower() == sym.lower():
            return True
syms2 = []
for sym in syms:
    if not check_exists(sym, syms2):
        syms2.append(sym)
syms = syms2

# write symbols.h
f = open("symbols.h", "w")
f.write("#ifndef _SYMBOLS_H_\n")
f.write("#define _SYMBOLS_H_\n\n")
f.write("char* symbols[] = {\n")
for i in range(len(syms)):
    sym = syms[i]
    f.write("\t\"%s\",\n" % sym)
f.write("};\n\n")
for i in range(len(syms)):
    sym = syms[i]
    f.write("#define SYM_%s\t\t%d\n" % (sym.upper(), i))
f.write("\n#endif\n");

# write Symbols.cs
f = open("Symbols.cs", "w")
f.write("namespace KowhaiSymbols\n{\n")
f.write("    public static class Symbols\n    {\n")
f.write("        public static string[] Strings =\n        {\n")
for sym in syms:
    f.write("            \"%s\",\n" % sym)
f.write("        };\n");
f.write("        public enum Constants\n        {\n")
for sym in syms:
    f.write("            %s,\n" % sym)
f.write("        };\n    }\n}\n");


{'fraction_non_alphanumeric': 0.21861366941347551, 'fraction_numerical': 0.0019389238972370335, 'mean_word_length': 2.564766839378238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6676718', 'n_tokens_mistral': 825, 'n_tokens_neox': 812, 'n_words': 199}