<commit_before>#!/usr/bin/perl -w

use strict;
my $script_dir; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; }
my $main_dir=dirname(${script_dir});

use IrstLMRegressionTesting;
use Getopt::Long;
use File::Temp qw ( tempfile );
use POSIX qw ( strftime );
my @SIGS = qw ( SIGHUP SIGINT SIGQUIT SIGILL SIGTRAP SIGABRT SIGIOT SIGBUS SIGFPE SIGKILL SIGUSR1 SIGSEGV SIGUSR2 SIGPIPE SIGALRM SIGTERM SIGSTKFLT SIGCHLD SIGCONT SIGSTOP SIGTSTP SIGTTIN SIGTTOU SIGURG SIGXCPU SIGXFSZ SIGVTALRM SIGPROF SIGWINCH SIGIO SIGPWR SIGSYS SIGUNUSED SIGRTMIN );
my ($decoder, $test_name);

my $test_dir = "$main_dir/tests";
my $results_dir = "$main_dir/results";
my $data_dir = IrstLMRegressionTesting::find_data_directory($main_dir);

my $BIN_TEST = $script_dir;

GetOptions("test=s"    => \$test_name,
           "data-dir=s"=> \$data_dir,
           "test-dir=s"=> \$test_dir,
           "results-dir=s"=> \$results_dir,
          ) or exit 1;

die "Please specify a test to run with --test\n" unless $test_name;

die "Please specify the location of the data directory (absolute path) with --data-dir\n" unless $data_dir;
die "Please specifiy the absolute path of the data directory!\n" unless $data_dir=~/^\//;
#setting the path where LM are stored
$ENV{IRSTLM_LM_PATH} = "$data_dir/lm";


$test_dir .= "/$test_name";
print STDERR "test_dir:|${test_dir}|\n";
print STDERR "results_dir:|${results_dir}|\n";

die "Cannot locate test dir at $test_dir" unless (-d $test_dir);

#### get place to put results
if (!-d $results_dir) {
  print STDERR "[WARNING] Results directory not found.\n";
  mkdir ($results_dir) || die "Failed to create $results_dir";
}
$results_dir .= "/$test_name";

if (!-d $results_dir) {
  print STDERR "[WARNING] Results directory for test=$test_name could not be found.\n";
  mkdir ($results_dir) || die "Failed to create $results_dir";
}
##########

my $ts = get_timestamp("$test_dir/command");
my $results = "$results_dir/$ts";
mkdir($results) || die "Failed to create results directory: $results\n";

my $truth = "$test_dir/truth";
if (!-d $truth) {
  die "Could not find truth/ in $test_dir!\n";<commit_msg>minor change<commit_after>#!/usr/bin/perl -w

use strict;
my $script_dir; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; }
my $main_dir=dirname(${script_dir});

use IrstLMRegressionTesting;
use Getopt::Long;
use File::Temp qw ( tempfile );
use POSIX qw ( strftime );
my @SIGS = qw ( SIGHUP SIGINT SIGQUIT SIGILL SIGTRAP SIGABRT SIGIOT SIGBUS SIGFPE SIGKILL SIGUSR1 SIGSEGV SIGUSR2 SIGPIPE SIGALRM SIGTERM SIGSTKFLT SIGCHLD SIGCONT SIGSTOP SIGTSTP SIGTTIN SIGTTOU SIGURG SIGXCPU SIGXFSZ SIGVTALRM SIGPROF SIGWINCH SIGIO SIGPWR SIGSYS SIGUNUSED SIGRTMIN );
my ($decoder, $test_name);

my $test_dir = "$main_dir/tests";
my $results_dir = "$main_dir/results";
my $data_dir = IrstLMRegressionTesting::find_data_directory($main_dir);

my $BIN_TEST = $script_dir;

GetOptions("test=s"    => \$test_name,
           "data-dir=s"=> \$data_dir,
           "test-dir=s"=> \$test_dir,
           "results-dir=s"=> \$results_dir,
          ) or exit 1;

die "Please specify a test to run with --test\n" unless $test_name;

die "Please specify the location of the data directory (absolute path) with --data-dir\n" unless $data_dir;
die "Please specifiy the absolute path of the data directory!\n" unless $data_dir=~/^\//;
#setting the path where LM are stored
$ENV{IRSTLM_LM_PATH} = "$data_dir/lm";


$test_dir .= "/$test_name";
print STDERR "test_dir:|${test_dir}|\n";
print STDERR "results_dir:|${results_dir}|\n";

die "Cannot locate test dir at $test_dir" unless (-d $test_dir);

#### get place to put results
if (!-d $results_dir) {
  print STDERR "[WARNING] Results directory not found; hence created.\n";
  mkdir ($results_dir) || die "Failed to create $results_dir";
}
$results_dir .= "/$test_name";

if (!-d $results_dir) {
  print STDERR "[WARNING] Results directory for test=$test_name not found; hence created.\n";
  mkdir ($results_dir) || die "Failed to create $results_dir";
}
##########

my $ts = get_timestamp("$test_dir/command");
my $results = "$results_dir/$ts";
mkdir($results) || die "Failed to create results directory: $results\n";

my $truth = "$test_dir/truth";
if (!-d $truth) {
  die "Could not find truth/ in $test_dir!\n";
{'fraction_non_alphanumeric': 0.14676390154968094, 'fraction_numerical': 0.0018231540565177757, 'mean_word_length': 4.71484375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6509219', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1553, 'n_words': 468}