<commit_before># experiment.meta: now with comments.

[CORPUS] multiple
get-corpus
	in: get-corpus-script
	out: raw-stem
	default-name: corpus/txt
	rerun-on-change: input-extension output-extension
	template: IN OUT $input-extension $output-extension
tokenize
	in: raw-stem
	out: tokenized-stem
	default-name: corpus/tok
	pass-unless: input-tokenizer output-tokenizer
	template-if: input-tokenizer IN.$input-extension OUT.$input-extension
	template-if: output-tokenizer IN.$output-extension OUT.$output-extension 
	parallelizable: yes
clean
	in: tokenized-stem
	out: clean-stem
	default-name: corpus/clean
	ignore-if: cleaner
	rerun-on-change: max-sentence-length $moses-script-dir/training/clean-corpus-n.perl clean-options
	template: $moses-script-dir/training/clean-corpus-n.perl IN $input-extension $output-extension OUT 1 $max-sentence-length OUT.lines-retained
	error: there is a blank factor
	error: is too long! at
custom-clean
	in: tokenized-stem
	out: clean-stem
	default-name: corpus/clean
	pass-unless: cleaner
	rerun-on-change: max-sentence-length cleaner
	template: $cleaner IN $input-extension $output-extension OUT 1 $max-sentence-length OUT.lines-retained
	error: there is a blank factor
	error: is too long! at
parse
	in: clean-stem
	out: parsed-stem
	default-name: corpus/parsed
	pass-unless: input-parser output-parser
	template-if: input-parser IN.$input-extension OUT.$input-extension
	template-if: output-parser IN.$output-extension OUT.$output-extension
	parallelizable: yes
post-parse-clean
	in: parsed-stem 
	out: clean-parsed-stem
	default-name: corpus/parsed-clean
	pass-unless: input-parser output-parser
	template: $moses-script-dir/training/clean-corpus-n.perl IN $input-extension $output-extension OUT 1 10000 OUT.lines-retained --ignore-xml<commit_msg>remove unused variable<commit_after># experiment.meta: now with comments.

[CORPUS] multiple
get-corpus
	in: get-corpus-script
	out: raw-stem
	default-name: corpus/txt
	rerun-on-change: input-extension output-extension
	template: IN OUT $input-extension $output-extension
tokenize
	in: raw-stem
	out: tokenized-stem
	default-name: corpus/tok
	pass-unless: input-tokenizer output-tokenizer
	template-if: input-tokenizer IN.$input-extension OUT.$input-extension
	template-if: output-tokenizer IN.$output-extension OUT.$output-extension 
	parallelizable: yes
clean
	in: tokenized-stem
	out: clean-stem
	default-name: corpus/clean
	ignore-if: cleaner
	rerun-on-change: max-sentence-length $moses-script-dir/training/clean-corpus-n.perl
	template: $moses-script-dir/training/clean-corpus-n.perl IN $input-extension $output-extension OUT 1 $max-sentence-length OUT.lines-retained
	error: there is a blank factor
	error: is too long! at
custom-clean
	in: tokenized-stem
	out: clean-stem
	default-name: corpus/clean
	pass-unless: cleaner
	rerun-on-change: max-sentence-length cleaner
	template: $cleaner IN $input-extension $output-extension OUT 1 $max-sentence-length OUT.lines-retained
	error: there is a blank factor
	error: is too long! at
parse
	in: clean-stem
	out: parsed-stem
	default-name: corpus/parsed
	pass-unless: input-parser output-parser
	template-if: input-parser IN.$input-extension OUT.$input-extension
	template-if: output-parser IN.$output-extension OUT.$output-extension
	parallelizable: yes
post-parse-clean
	in: parsed-stem 
	out: clean-parsed-stem
	default-name: corpus/parsed-clean
	pass-unless: input-parser output-parser
	template: $moses-script-dir/training/clean-corpus-n.perl IN $input-extension $output-extension OUT 1 10000 OUT.lines-retained --ignore-xml
{'fraction_non_alphanumeric': 0.10855539971949509, 'fraction_numerical': 0.004488078541374474, 'mean_word_length': 4.7056, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5740458', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1147, 'n_words': 294}