<commit_before>#!/usr/bin/perl -w
#
# Detokenization tests.
#

use strict;
# This is here to suppress (false) warnings about OLDOUT and OLDERR being used only once.  Maybe there is a less brutish way to suppress that, but I don't know it.
no warnings 'once';
use utf8;

use Cwd ('abs_path');
use File::Spec::Functions;
use File::Basename ('dirname');
use IPC::Run3;
use Getopt::Long;

use MosesRegressionTesting;

GetOptions("detokenizer=s" => \(my $detokenizer),
           "results-dir=s"=> \(my $results_dir)
          ) or exit 1;

unless (defined $results_dir) {
    print STDERR "Usage: run-test-detokenizer.t --results-dir <RESULTS-DIRECTORY> [--detokenizer <DETOKENIZER-SCRIPT>]\n";
    exit 1;
}

die "ERROR: Results directory ".$results_dir." doesn't exist or is not a writable directory. Dying" unless (-d $results_dir && -w $results_dir);

$detokenizer = catfile(dirname(dirname(abs_path($0))), "scripts", "tokenizer", "detokenizer.perl") unless $detokenizer;
die "ERROR: Detokenizer script ".$detokenizer." does not exist. Dying" unless -f $detokenizer;


use Test::More;

######################################
# Definitions of individual test cases
######################################

# A simple English test
&runDetokenizerTest("TEST_ENGLISH_EASY", "en",
<<'TOK',
This sentence is really simple , so it should not be hard to detokenize .
This one is no more difficult , but , hey , it is on a new line .
TOK
<<'EXP'
This sentence is really simple, so it should not be hard to detokenize.
This one is no more difficult, but, hey, it is on a new line.
EXP
);

# A simple French test
&runDetokenizerTest("TEST_FRENCH_EASY", "fr",
<<'TOK',
Ici une phrase simple .
TOK
<<'EXP'
Ici une phrase simple.
EXP
);

######################################
# end of individual test cases
######################################

done_testing();<commit_msg>a few more detokenization tests, including a TODO one that exposes a bug<commit_after>#!/usr/bin/perl -w
#
# Detokenization tests.
#

use strict;
# This is here to suppress (false) warnings about OLDOUT and OLDERR being used only once.  Maybe there is a less brutish way to suppress that, but I don't know it.
no warnings 'once';
use utf8;

use Cwd ('abs_path');
use File::Spec::Functions;
use File::Basename ('dirname');
use IPC::Run3;
use Getopt::Long;

use MosesRegressionTesting;

GetOptions("detokenizer=s" => \(my $detokenizer),
           "results-dir=s"=> \(my $results_dir)
          ) or exit 1;

unless (defined $results_dir) {
    print STDERR "Usage: run-test-detokenizer.t --results-dir <RESULTS-DIRECTORY> [--detokenizer <DETOKENIZER-SCRIPT>]\n";
    exit 1;
}

die "ERROR: Results directory ".$results_dir." doesn't exist or is not a writable directory. Dying" unless (-d $results_dir && -w $results_dir);

$detokenizer = catfile(dirname(dirname(abs_path($0))), "scripts", "tokenizer", "detokenizer.perl") unless $detokenizer;
die "ERROR: Detokenizer script ".$detokenizer." does not exist. Dying" unless -f $detokenizer;


use Test::More;

######################################
# Definitions of individual test cases
######################################

# A simple English test
&runDetokenizerTest("TEST_ENGLISH_EASY", "en",
<<'TOK',
This sentence is really simple , so it should not be hard to detokenize .
This one is no more difficult , but , hey , it is on a new line .
TOK
<<'EXP'
This sentence is really simple, so it should not be hard to detokenize.
This one is no more difficult, but, hey, it is on a new line.
EXP
);

# An English test involving double-quotes
&runDetokenizerTest("TEST_ENGLISH_DOUBLEQUOTES", "en",
<<'TOK',
This is a somewhat " less simple " test .
TOK
<<'EXP'
This is a somewhat "less simple" test.
EXP
);

# A simple French test
&runDetokenizerTest("TEST_FRENCH_EASY", "fr",
<<'TOK',
Voici une phrase simple .
TOK
<<'EXP'
Voici une phrase simple.
EXP
);

# A French test involving an apostrophe
&runDetokenizerTest("TEST_FRENCH_APOSTROPHE", "fr",
<<'TOK',
Moi , j' ai une apostrophe .
TOK
<<'EXP'
Moi, j'ai une apostrophe.
EXP
);

TODO: {
    local $TODO = "A bug is causing this to be detokenized wrong.";

# A French test involving an apostrophe on the second-last word
&runDetokenizerTest("TEST_FRENCH_APOSTROPHE_PENULTIMATE", "fr",
<<'TOK',
de musique rap issus de l' immigration
TOK
<<'EXP'
de musique rap issus de l'immigration
EXP
);

}

######################################
# end of individual test cases
######################################

done_testing();
{'fraction_non_alphanumeric': 0.18303571428571427, 'fraction_numerical': 0.002232142857142857, 'mean_word_length': 4.865183246073299, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3300871', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1439, 'n_words': 518}