<commit_before>#!/usr/bin/perl -w

# $Id$

use strict;
use FindBin qw($Bin);

my $script_dir; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; }
use Getopt::Long;

############################################################
my @tests = qw (
	extract.phrase-based
	extract-rules.hierarchical
  score.phrase-based
  score.phrase-based-inv
  score.phrase-based-with-alignment
  score.phrase-based-with-alignment-inv
  score.hierarchical
  score.hierarchical-inv
  mert.basic
  mert.pro
  mert.extractor-txt
  mert.extractor-bin
  chart.target-syntax
  chart.target-syntax.ondisk
  chart.hierarchical
  chart.hierarchical-withsrilm
  chart.hierarchical.ondisk
  phrase.basic-surface-only
  phrase.basic-surface-only-withirstlm
  phrase.basic-surface-only-withirstlm-binlm
  phrase.basic-lm-oov
  phrase.ptable-filtering
  phrase.multi-factor
  phrase.multi-factor-drop
  phrase.confusionNet-surface-only
  phrase.confusionNet-multi-factor
  phrase.basic-surface-binptable
  phrase.multi-factor-binptable
  phrase.nbest-multi-factor
  phrase.lattice-surface
  phrase.lattice-distortion
  phrase.lexicalized-reordering
  phrase.lexicalized-reordering-bin
  phrase.lexicalized-reordering-cn
  phrase.consensus-decoding-surface
  phrase.continue-partial-translation
  phrase.show-weights.lex-reorder
  phrase.show-weights
  phrase.xml-markup
);
  #phrase.basic-lm-oov-withkenlm
  #phrase.basic-surface-only-withkenlm
  #phrase.basic-surface-only-withkenlm.bin
  #chart.hierarchical-withkenlm

############################################################
use MosesRegressionTesting;
use File::Temp qw ( tempfile );
use POSIX qw ( strftime );

my $decoderPhrase = "$Bin/../moses-cmd/src/moses";
my $decoderChart = "$Bin/../moses-chart-cmd/src/moses_chart";
my $scoreExe = "$Bin/../scripts/training/phrase-extract/score";
my $extractorExe = "$Bin/../scripts/training/phrase-extract/extract";
my $extractorRulesExe = "$Bin/../scripts/training/phrase-extract/extract-rules";
my $kenlmBinarizer = "$Bin/../kenlm/build_binary";
my $test_dir;
my $BIN_TEST = $script_dir;
my $data_dir;

GetOptions(	"decoder-phrase=s" => \$decoderPhrase,
			"decoder-chart=s" => \$decoderChart,
           	"data-dir=s" => \$data_dir,
          ) or exit 1;


$data_dir = MosesRegressionTesting::find_data_directory($BIN_TEST, $data_dir);

my $test_run = "$BIN_TEST/run-single-test.pl --data-dir=$data_dir";
$test_dir = $script_dir . "/tests";
$test_run .= " --test-dir=$test_dir" if $test_dir;

print "Data directory: $data_dir\n";

die "Please specify the phrase-based decoder & the chart decoder to test with --decoder-phrase=[path] --decoder-chart=[path] \n" unless ($decoderPhrase and $decoderChart);

die "Cannot locate executable called $decoderPhrase\n" unless (-x $decoderPhrase);

print "Running tests: @tests\n\n";

print "TEST NAME               STATUS     PATH TO RESULTS\n";
my $lb = "---------------------------------------------------------------------------------------------------------\n";
print $lb;
<commit_msg>delete test directory from repository<commit_after>#!/usr/bin/perl -w

# $Id$

use strict;
use FindBin qw($Bin);

my $script_dir; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; }
use Getopt::Long;

############################################################
my @tests = qw (
	extract.phrase-based
	extract-rules.hierarchical
  score.phrase-based
  score.phrase-based-inv
  score.phrase-based-with-alignment
  score.phrase-based-with-alignment-inv
  score.hierarchical
  score.hierarchical-inv
  mert.basic
  mert.pro
  mert.extractor-txt
  mert.extractor-bin
  chart.target-syntax
  chart.target-syntax.ondisk
  chart.hierarchical
  chart.hierarchical-withsrilm
  chart.hierarchical.ondisk
  phrase.basic-surface-only
  phrase.basic-surface-only-withirstlm
  phrase.basic-surface-only-withirstlm-binlm
  phrase.basic-lm-oov
  phrase.ptable-filtering
  phrase.multi-factor
  phrase.multi-factor-drop
  phrase.confusionNet-surface-only
  phrase.confusionNet-multi-factor
  phrase.basic-surface-binptable
  phrase.multi-factor-binptable
  phrase.nbest-multi-factor
  phrase.lattice-surface
  phrase.lattice-distortion
  phrase.lexicalized-reordering
  phrase.lexicalized-reordering-bin
  phrase.lexicalized-reordering-cn
  phrase.consensus-decoding-surface
  phrase.continue-partial-translation
  phrase.show-weights.lex-reorder
  phrase.show-weights
  phrase.xml-markup
);
  #phrase.basic-lm-oov-withkenlm
  #phrase.basic-surface-only-withkenlm
  #phrase.basic-surface-only-withkenlm.bin
  #chart.hierarchical-withkenlm

############################################################
use MosesRegressionTesting;
use File::Temp qw ( tempfile );
use POSIX qw ( strftime );

my $decoderPhrase = "$Bin/../moses-cmd/src/moses";
my $decoderChart = "$Bin/../moses-chart-cmd/src/moses_chart";
my $scoreExe = "$Bin/../scripts/training/phrase-extract/score";
my $extractorExe = "$Bin/../scripts/training/phrase-extract/extract";
my $extractorRulesExe = "$Bin/../scripts/training/phrase-extract/extract-rules";
my $kenlmBinarizer = "$Bin/../kenlm/build_binary";
my $test_dir;
my $BIN_TEST = $script_dir;
my $data_dir;

GetOptions(	"decoder-phrase=s" => \$decoderPhrase,
			"decoder-chart=s" => \$decoderChart,
           	"data-dir=s" => \$data_dir,
          ) or exit 1;


$data_dir = MosesRegressionTesting::find_data_directory($BIN_TEST, $data_dir);

my $test_run = "$BIN_TEST/run-single-test.pl --data-dir=$data_dir";
$test_dir = "$data_dir/tests";
$test_run .= " --test-dir=$test_dir" if $test_dir;

print "Data directory: $data_dir\n";

die "Please specify the phrase-based decoder & the chart decoder to test with --decoder-phrase=[path] --decoder-chart=[path] \n" unless ($decoderPhrase and $decoderChart);

die "Cannot locate executable called $decoderPhrase\n" unless (-x $decoderPhrase);

print "Running tests: @tests\n\n";

print "TEST NAME               STATUS     PATH TO RESULTS\n";
my $lb = "---------------------------------------------------------------------------------------------------------\n";
print $lb;

{'fraction_non_alphanumeric': 0.2016273393002441, 'fraction_numerical': 0.0006509357200976403, 'mean_word_length': 3.6702127659574466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7472089', 'n_tokens_mistral': 2130, 'n_tokens_neox': 2039, 'n_words': 359}