<commit_before>from wordfreq_builder.word_counts import read_freqs, merge_freqs, write_wordlist
import argparse


def merge_lists(input_names, output_name, cutoff, lang):
    freq_dicts = []

    # Don't use Chinese tokenization while building wordlists, as that would
    # create a circular dependency.
    if lang == 'zh':
        lang = None

    for input_name in input_names:
        freq_dicts.append(read_freqs(input_name, cutoff=cutoff, lang=lang))
    merged = merge_freqs(freq_dicts)
    write_wordlist(merged, output_name)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-o', '--output', help='filename to write the output to', default='combined-freqs.csv')
    parser.add_argument('-c', '--cutoff', type=int, help='stop after seeing a count below this', default=2)
    parser.add_argument('-l', '--language', 'language code for which language the words are in', default=None)
    parser.add_argument('inputs', help='names of input files to merge', nargs='+')
    args = parser.parse_args()
    merge_lists(args.inputs, args.output, args.cutoff, args.language)
<commit_msg>fix '--language' option definition<commit_after>from wordfreq_builder.word_counts import read_freqs, merge_freqs, write_wordlist
import argparse


def merge_lists(input_names, output_name, cutoff, lang):
    freq_dicts = []

    # Don't use Chinese tokenization while building wordlists, as that would
    # create a circular dependency.
    if lang == 'zh':
        lang = None

    for input_name in input_names:
        freq_dicts.append(read_freqs(input_name, cutoff=cutoff, lang=lang))
    merged = merge_freqs(freq_dicts)
    write_wordlist(merged, output_name)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-o', '--output', help='filename to write the output to', default='combined-freqs.csv')
    parser.add_argument('-c', '--cutoff', type=int, help='stop after seeing a count below this', default=2)
    parser.add_argument('-l', '--language', help='language code for which language the words are in', default=None)
    parser.add_argument('inputs', help='names of input files to merge', nargs='+')
    args = parser.parse_args()
    merge_lists(args.inputs, args.output, args.cutoff, args.language)

{'fraction_non_alphanumeric': 0.1194225721784777, 'fraction_numerical': 0.0008748906386701663, 'mean_word_length': 4.445238095238095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3103267', 'n_tokens_mistral': 718, 'n_tokens_neox': 697, 'n_words': 193}