<commit_before>#!/usr/bin/perl

if ($#ARGV!=0) {
    print stderr "Usage: $0 <lang> where <lang> is the two letter suffix for the language\n";
    exit;
}

$lang=$ARGV[0];

link "30_text_$lang.cf", "orig_30_text_$lang.cf";

open IN, "30_text_$lang.cf";
while (<IN>) {
    next unless /^lang $lang describe\s([^\s]+)\s/;
    $rule{$1}=1;
}
close IN;

open IN, "cat *|grep '^describe'|";
open OUT, ">>30_text_$lang.cf";
while (<IN>) {
    next unless /^describe\s([^\s]+)\s/;
    next if $rule{$1};
    print OUT "lang $lang $_";
}<commit_msg>Bugzilla #388: updated language prep tool<commit_after>#!/usr/local/bin/perl

if ($#ARGV!=0) {
    print stderr "Usage: $0 <lang>
where <lang> is the two letters suffix for the language
";
    exit;
}

$lang=$ARGV[0];

unlink "orig_30_text_$lang.cf" if -f "orig_30_text_$lang.cf";
system "cp 30_text_$lang.cf orig_30_text_$lang.cf";

open IN, "30_text_$lang.cf";
while (<IN>) {
    next unless /^lang $lang describe\s([^\s]+)\s/;
    $rule{$1}=1;
}
close IN;

open IN, "cat *|grep '^describe'|";
open OUT, ">>30_text_$lang.cf";
while (<IN>) {
    next unless /^describe\s([^\s]+)\s/;
    $r=$1;
    next if $rule{$r};
    $rule{$r}=1;
    $expr=`cat * |grep $r|grep -v describe|grep -v score|grep -v -e '^test'|grep -v '\#'`;
    $expr=~s/^.*$r\s+//;
    chomp;
    # print "$_+++$expr";
    print OUT "lang $lang $_ $expr";
}
{'fraction_non_alphanumeric': 0.22896854425749816, 'fraction_numerical': 0.026335040234089245, 'mean_word_length': 3.5906040268456376, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '892096', 'n_tokens_mistral': 618, 'n_tokens_neox': 587, 'n_words': 141}