<commit_before>#!/usr/bin/perl -w
use strict;
use Getopt::Std;

my $usage = qq{run-mofessor.pl -m morfessor_path -p PPLTHRESH < vocablary_file
};

our($opt_m, $opt_p);
getopts('m:p:');
die $usage unless ($opt_m and $opt_p);
$opt_m =~ s/\/$//g;

my %words;
while (<>) {
  $_ =~ s/#/|PSN|/g;
  $_ =~ s/\\\*/|AST|/g;
  $_ =~ s/\\\//|SLH|/g;
  $_ =~ s/\+/|PLS|/g;
  $words{$_}++;
}


open(FP, "| gzip > $opt_m/train/wsj.words.clean.gz") or die $!;
while (my($word, $c) = each(%words)) {
  print FP $c." ".$word;
}
close(FP);

system("cd $opt_m/train; sed 's/^GZIPPEDINPUTDATA = mydata.gz/GZIPPEDINPUTDATA = wsj.words.clean.gz/g' Makefile | sed 's/^PPLTHRESH = 10/PPLTHRESH = $opt_p/g' > tmp; mv tmp Makefile; make");

open(FP, "zcat $opt_m/train/segmentation.final.gz |") or die $!;
while (<FP>) {
  print $_;
}
close(FP);

`cd $opt_m/train; make realclean`;<commit_msg>*** empty log message ***<commit_after>#!/usr/bin/perl -w
use strict;
use Getopt::Std;
use File::Temp qw/tempdir/;

my $usage = qq{run-mofessor.pl -m morfessor_path -p PPLTHRESH < vocablary_file
};

our($opt_m, $opt_p);
getopts('m:p:');
die $usage unless ($opt_m and $opt_p);
$opt_m =~ s/\/$//g;

my %words;
while (<>) {
  $_ =~ s/#/|PSN|/g;
  $_ =~ s/\\\*/|AST|/g;
  $_ =~ s/\\\//|SLH|/g;
  $_ =~ s/\+/|PLS|/g;
  $words{$_}++;
}


open(FP, "| gzip > $opt_m/train/wsj.words.clean.gz") or die $!;
while (my($word, $c) = each(%words)) {
  print FP $c." ".$word;
}
close(FP);

my $tmp = tempdir("$opt_m/train-XXXX", CLEANUP => 1);
system("cd $tmp; cp ../train/* .; sed 's/^GZIPPEDINPUTDATA = mydata.gz/GZIPPEDINPUTDATA = wsj.words.clean.gz/g' Makefile | sed 's/^PPLTHRESH = [0-9]\\+/PPLTHRESH = $opt_p/g' > tmp; mv tmp Makefile; make > /dev/null");

open(FP, "zcat $tmp/segmentation.final.gz |") or die $!;
while (<FP>) {
  print $_;
}
close(FP);

`cd $tmp; make realclean`;
{'fraction_non_alphanumeric': 0.25639629831246596, 'fraction_numerical': 0.0027218290691344584, 'mean_word_length': 3.5835411471321694, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7433756', 'n_tokens_mistral': 905, 'n_tokens_neox': 871, 'n_words': 166}