<commit_before>#!/usr/bin/env python3
import re
import sys
import requests
from bsdconv import Bsdconv

if len(sys.argv) != 2:
    print("Usage: {} outfile".format(sys.argv[0]))
    sys.exit(1)

outfile = sys.argv[1]

cv = Bsdconv("utf-8:split:bsdconv-keyword,bsdconv")

def bc(s):
    return cv.conv(s).strip(",")

url = "https://raw.githubusercontent.com/apache/lucene-solr/master/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java"
java = requests.get(url).text
tks = re.findall(r"(?:case '(.*?)':|output\[outputPos\+\+\] = '(.*?)';|(break);)", java)

m = {}
f = []
t = ""
for tk in tks:
    if tk[2]=="break":
        m[t.encode("utf-8").decode("unicode_escape")] = f
        f = []
        t = ""
    elif tk[1]:
        t = t + tk[1]
    elif tk[0]:
        f.append(tk[0].encode("utf-8").decode("unicode_escape"))
    else:
        print("Unexpected Error")

l = []
for f in m:
    t = m[f]
    for c in t:
        print("{}\t{}".format(c, f))<commit_msg>update tools/gen_asciifold.py: follow python3 api change<commit_after>#!/usr/bin/env python3
import re
import sys
import requests
from bsdconv import Bsdconv

if len(sys.argv) != 2:
    print("Usage: {} outfile".format(sys.argv[0]))
    sys.exit(1)

outfile = sys.argv[1]

cv = Bsdconv("utf-8:split:bsdconv-keyword,bsdconv")

def bc(s):
    return cv.uconv(s).strip(",")

url = "https://raw.githubusercontent.com/apache/lucene-solr/master/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java"
java = requests.get(url).text
tks = re.findall(r"(?:case '(.*?)':|output\[outputPos\+\+\] = '(.*?)';|(break);)", java)

m = {}
f = []
t = ""
for tk in tks:
    if tk[2]=="break":
        m[t.encode("utf-8").decode("unicode_escape")] = f
        f = []
        t = ""
    elif tk[1]:
        t = t + tk[1]
    elif tk[0]:
        f.append(tk[0].encode("utf-8").decode("unicode_escape"))
    else:
        print("Unexpected Error")

l = []
for f in m:
    t = m[f]
    for c in t:
        print("{}\t{}".format(c, f))
{'fraction_non_alphanumeric': 0.21019417475728155, 'fraction_numerical': 0.013106796116504855, 'mean_word_length': 3.155241935483871, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6682191', 'n_tokens_mistral': 829, 'n_tokens_neox': 806, 'n_words': 148}