<commit_before>#!/usr/bin/env python3

"""Train detokenizers. Run from lex root directory."""

import subprocess

langs = ['de', 'es', 'fr']
raw = 'data/detok/detok_sample_1M_{}_raw.clean.gz'
tok = 'data/detok/detok_sample_1M_{}_raw.clean.tok.gz'
out = 'data/detok/detok_sample_1M_{}_raw.clean.detokenizer'
java = 'java -cp build/install/lex/lib/*'.split()
mem =  '-Xmx3g -XX:+UseG1GC -XX:+UseStringDeduplication'.split()
cls = ['org.oscii.detokenize.TrainDetokenizer']

if __name__ == '__main__':
    subprocess.check_call(['gradle', 'installDist'])
    for lang in langs:
        opt = ['-raw', raw.format(lang), '-tok', tok.format(lang), '-out', out.format(lang)]
        cmd = java + mem + cls + opt
        subprocess.check_call(cmd)
        
        <commit_msg>set training, memory, and regularization<commit_after>#!/usr/bin/env python3

"""Train detokenizers. Run from lex root directory."""

import subprocess

langs = ['de', 'es', 'fr']
raw = 'data/detok/detok_sample_1M_{}_raw.clean.gz'
tok = 'data/detok/detok_sample_1M_{}_raw.clean.tok.gz'
out = 'data/detok/detok_sample_1M_{}_raw.clean.detokenizer'
java = 'java -cp build/install/lex/lib/*'.split()
mem =  '-Xmx4g -XX:+UseG1GC -XX:+UseStringDeduplication'.split()
cls = ['org.oscii.detokenize.TrainDetokenizer']

if __name__ == '__main__':
    subprocess.check_call(['gradle', 'installDist'])
    for lang in langs:
        opt = ['-raw', raw.format(lang), '-tok', tok.format(lang), '-out', out.format(lang)]
        opt += ['-trainsize', '400000', '-regularization', '400']
        cmd = java + mem + cls + opt
        subprocess.check_call(cmd)
        
        
{'fraction_non_alphanumeric': 0.17986494782074894, 'fraction_numerical': 0.01289134438305709, 'mean_word_length': 3.808259587020649, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5190847', 'n_tokens_mistral': 614, 'n_tokens_neox': 597, 'n_words': 115}