<commit_before>#!/usr/bin/perl -w

use strict;
my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; }
use Getopt::Long;

my $MAX_ITER_ATTEMPTS = 5; # number of times to retry a failed function evaluation
my $CWD=`pwd`; chomp $CWD;
my $BIN_DIR = $SCRIPT_DIR;
my $OPTIMIZER = "$BIN_DIR/mr_optimize_reduce";
my $DECODER = "$BIN_DIR/cdec";
my $COMBINER_CACHE_SIZE = 150;
my $PARALLEL = "/chomes/redpony/svn-trunk/sa-utils/parallelize.pl";
die "Can't find $OPTIMIZER" unless -f $OPTIMIZER;
die "Can't execute $OPTIMIZER" unless -x $OPTIMIZER;
my $restart = '';
if ($ARGV[0] && $ARGV[0] eq '--restart') { shift @ARGV; $restart = 1; }

my $pmem="2500mb";
my $nodes = 36;
my $max_iteration = 1000;
my $PRIOR_FLAG = "";
my $parallel = 1;
my $CFLAG = "-C 1";
my $LOCAL;
my $PRIOR;
my $OALG = "lbfgs";
my $sigsq = 1;
my $means_file;
GetOptions("decoder=s" => \$DECODER,
           "run_locally" => \$LOCAL,
           "gaussian_prior" => \$PRIOR,
           "sigma_squared=f" => \$sigsq,
           "means=s" => \$means_file,
           "optimizer=s" => \$OALG,<commit_msg>fix path<commit_after>#!/usr/bin/perl -w

use strict;
my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; }
use Getopt::Long;

my $MAX_ITER_ATTEMPTS = 5; # number of times to retry a failed function evaluation
my $CWD=`pwd`; chomp $CWD;
my $BIN_DIR = $SCRIPT_DIR;
my $OPTIMIZER = "$BIN_DIR/mr_optimize_reduce";
my $DECODER = "$BIN_DIR/../src/cdec";
my $COMBINER_CACHE_SIZE = 150;
# This is a hack to run this on a weird cluster,
# eventually, I'll provide Hadoop scripts.
my $PARALLEL = "/chomes/redpony/svn-trunk/sa-utils/parallelize.pl";
die "Can't find $OPTIMIZER" unless -f $OPTIMIZER;
die "Can't execute $OPTIMIZER" unless -x $OPTIMIZER;
my $restart = '';
if ($ARGV[0] && $ARGV[0] eq '--restart') { shift @ARGV; $restart = 1; }

my $pmem="2500mb";
my $nodes = 36;
my $max_iteration = 1000;
my $PRIOR_FLAG = "";
my $parallel = 1;
my $CFLAG = "-C 1";
my $LOCAL;
my $PRIOR;
my $OALG = "lbfgs";
my $sigsq = 1;
my $means_file;
GetOptions("decoder=s" => \$DECODER,
           "run_locally" => \$LOCAL,
           "gaussian_prior" => \$PRIOR,
           "sigma_squared=f" => \$sigsq,
           "means=s" => \$means_file,
           "optimizer=s" => \$OALG,
{'fraction_non_alphanumeric': 0.1780590717299578, 'fraction_numerical': 0.017721518987341773, 'mean_word_length': 3.8486707566462166, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2237246', 'n_tokens_mistral': 1062, 'n_tokens_neox': 971, 'n_words': 250}