<commit_before>#!/usr/bin/perl -w

package CRF::CrfUtils;

use strict;

use 5.14.0;

use Text::CSV;
use Class::CSV;
use Readonly;
use Data::Dumper;
use File::Temp qw/ tempfile tempdir /;
use Env qw(HOME);
use File::Path qw(make_path remove_tree);
use File::Spec;
use File::Basename;

my $class_path;

BEGIN
 {
    my $_dirname      = dirname( __FILE__ );
    my $_dirname_full = File::Spec->rel2abs( $_dirname );

    my $jar_dir = "$_dirname_full/jars";

    my $jars = [ 'mallet-deps.jar', 'mallet.jar' ];

    #Assumes Unix fix later.
    $class_path = scalar( join ':', ( map { "$jar_dir/$_" } @{ $jars } ) );

    say STDERR "classpath: $class_path";
 }

sub output_testing_and_training
{
    my ( $out_data, my $leave_out_part, my $parts ) = @_;

    my @test_data = @{ $out_data };

    my $part_size = my $parts_size = int( scalar( @test_data ) / $parts ) + 1;

    my @leave_out_data = splice @test_data, ( $part_size * $leave_out_part ), $part_size;

    my ( $leave_out_data_fh, $leave_out_data_file_name ) = tempfile( "/tmp/leave_out_tmpfileXXXXXX", SUFFIX => '.dat' );

    print $leave_out_data_fh @leave_out_data;

    close( $leave_out_data_fh );

    my ( $train_data_fh, $train_data_file_name ) = tempfile( "/tmp/train_tmpfileeXXXXXX", SUFFIX => '.dat' );

    print $train_data_fh @test_data;

    say STDERR $leave_out_data_file_name;
    say STDERR $train_data_file_name;

    close( $train_data_fh );

    return {
        leave_out_file  => $leave_out_data_file_name,
        train_data_file => $train_data_file_name
    };
}

sub generate_output_fhs
{
    my ( $output_dir ) = @_;

    my $ret                     = {};
    my $probabilities_file_name = "$output_dir/probabilities.txt";
    open my $probabilities_fh, '>', $probabilities_file_name or die "Failed to open file $@";

    $ret->{ probabilities_fh } = $probabilities_fh;

    my $predictions_file_name = "$output_dir/predictions.txt";
    open my $predictions_fh, '>', $predictions_file_name or die "Failed to open file $@";

    $ret->{ predictions_fh } = $predictions_fh;

    my $expected_results_file_name = "$output_dir/expected.txt";
    open my $expected_results_fh, '>', $expected_results_file_name or die "Failed to open file $@";

    $ret->{ expected_results_fh } = $expected_results_fh;

    return $ret;
}

my $gaussian = $ENV{ MAX_ENT_GUASSIAN };

sub create_model
{
    my ( $training_data_file, $iterations ) = @_;

    return create_model_inline_java( $training_data_file, $iterations );
}

sub create_model_inline_java
{
    my ( $training_data_file, $iterations ) = @_;

    say "Entering create_model_inline_java";

    use Inline (
        Java  => 'STUDY',
        STUDY => [
            qw ( cc.mallet.fst.SimpleTagger
              java.io.FileReader java.io.File )
        ],
        AUTOSTUDY => 1,
        CLASSPATH => $class_path,
        PACKAGE => 'main'
    );

    my $model_file_name = $training_data_file;

    $model_file_name =~ s/\.dat$/Model\.txt/;

    say "Model File: $model_file_name";

    my $foo = cc::mallet::fst::SimpleTagger->main(
        [ "--train", "true", "--iterations", $iterations, "--model-file", $model_file_name, $training_data_file ] );

    return;
}

my $crf;

sub run_model_inline_java_data_array
{
    my ( $model_file_name, $test_data_array ) = @_;

    if ( !defined( $crf ) )
    {
        $crf = model_runner->readModel( $model_file_name );
    }

    return run_model_on_array( $crf, $test_data_array );
}

sub run_model_on_array
{
    my ( $crf, $test_data_array ) = @_;

    my $test_data = join "\n", @{ $test_data_array };

    my $foo = model_runner->run_model_string( $test_data, $crf );

    return $foo;
}

sub run_model_inline_java
{
    my ( $model_file_name, $test_data_file, $output_fhs ) = @_;

    my $probabilities_fh = $output_fhs->{ probabilities_fh };

    my $predictions_fh = $output_fhs->{ predictions_fh };

    my $expected_results_fh = $output_fhs->{ expected_results_fh };

    my $create_model_script_path = "$HOME/Applications/mallet-2.0.7/run_simple_tagger.sh";

    say STDERR "generating predictions";

    say STDERR "classpath: $class_path";

    
    use Inline (
        Java  => 'STUDY',
        STUDY => [
            qw ( cc.mallet.fst.SimpleTagger
              java.io.FileReader java.io.File )
        ],
        AUTOSTUDY => 1,
        CLASSPATH => $class_path,
        PACKAGE => 'main'
    );

    open my $test_data_file_fh, '<', $test_data_file;

    my @test_data_array = <$test_data_file_fh>;

    my $foo = run_model_inline_java_data_array( $model_file_name, \@test_data_array );

    say join "\n", @{ $foo };

    exit();
}

sub run_model
{
    my ( $model_file_name, $test_data_file, $output_fhs ) = @_;

    return run_model_inline_java( $model_file_name, $test_data_file, $output_fhs );
}

sub train_and_test
{<commit_msg>Add run model with separate exec method<commit_after>#!/usr/bin/perl -w

package CRF::CrfUtils;

use strict;

use 5.14.0;

use Text::CSV;
use Class::CSV;
use Readonly;
use Data::Dumper;
use File::Temp qw/ tempfile tempdir /;
use Env qw(HOME);
use File::Path qw(make_path remove_tree);
use File::Spec;
use File::Basename;

my $class_path;

BEGIN
 {
    my $_dirname      = dirname( __FILE__ );
    my $_dirname_full = File::Spec->rel2abs( $_dirname );

    my $jar_dir = "$_dirname_full/jars";

    my $jars = [ 'mallet-deps.jar', 'mallet.jar' ];

    #Assumes Unix fix later.
    $class_path = scalar( join ':', ( map { "$jar_dir/$_" } @{ $jars } ) );

    say STDERR "classpath: $class_path";
 }

sub output_testing_and_training
{
    my ( $out_data, my $leave_out_part, my $parts ) = @_;

    my @test_data = @{ $out_data };

    my $part_size = my $parts_size = int( scalar( @test_data ) / $parts ) + 1;

    my @leave_out_data = splice @test_data, ( $part_size * $leave_out_part ), $part_size;

    my ( $leave_out_data_fh, $leave_out_data_file_name ) = tempfile( "/tmp/leave_out_tmpfileXXXXXX", SUFFIX => '.dat' );

    print $leave_out_data_fh @leave_out_data;

    close( $leave_out_data_fh );

    my ( $train_data_fh, $train_data_file_name ) = tempfile( "/tmp/train_tmpfileeXXXXXX", SUFFIX => '.dat' );

    print $train_data_fh @test_data;

    say STDERR $leave_out_data_file_name;
    say STDERR $train_data_file_name;

    close( $train_data_fh );

    return {
        leave_out_file  => $leave_out_data_file_name,
        train_data_file => $train_data_file_name
    };
}

sub generate_output_fhs
{
    my ( $output_dir ) = @_;

    my $ret                     = {};
    my $probabilities_file_name = "$output_dir/probabilities.txt";
    open my $probabilities_fh, '>', $probabilities_file_name or die "Failed to open file $@";

    $ret->{ probabilities_fh } = $probabilities_fh;

    my $predictions_file_name = "$output_dir/predictions.txt";
    open my $predictions_fh, '>', $predictions_file_name or die "Failed to open file $@";

    $ret->{ predictions_fh } = $predictions_fh;

    my $expected_results_file_name = "$output_dir/expected.txt";
    open my $expected_results_fh, '>', $expected_results_file_name or die "Failed to open file $@";

    $ret->{ expected_results_fh } = $expected_results_fh;

    return $ret;
}

my $gaussian = $ENV{ MAX_ENT_GUASSIAN };

sub create_model
{
    my ( $training_data_file, $iterations ) = @_;

    return create_model_inline_java( $training_data_file, $iterations );
}

sub create_model_inline_java
{
    my ( $training_data_file, $iterations ) = @_;

    say "Entering create_model_inline_java";

    use Inline (
        Java  => 'STUDY',
        STUDY => [
            qw ( cc.mallet.fst.SimpleTagger
              java.io.FileReader java.io.File )
        ],
        AUTOSTUDY => 1,
        CLASSPATH => $class_path,
        PACKAGE => 'main'
    );

    my $model_file_name = $training_data_file;

    $model_file_name =~ s/\.dat$/Model\.txt/;

    say "Model File: $model_file_name";

    my $foo = cc::mallet::fst::SimpleTagger->main(
        [ "--train", "true", "--iterations", $iterations, "--model-file", $model_file_name, $training_data_file ] );

    return;
}

my $crf;

sub run_model_inline_java_data_array
{
    my ( $model_file_name, $test_data_array ) = @_;

    undef($crf);

    if ( !defined( $crf ) )
    {
        say STDERR "Read model ";
        $crf = model_runner->readModel( $model_file_name );
    }

    return run_model_on_array( $crf, $test_data_array );
}

sub run_model_with_separate_exec
{
    my ( $model_file_name, $test_data_array ) = @_;

    my ( $test_data_fh, $test_data_file_name ) = tempfile( "/tmp/tested_arrayXXXXXX", SUFFIX => '.dat' );

    print $test_data_fh join "\n", @ { $test_data_array };

    close( $test_data_fh );

    my $output = `java -cp  $class_path cc.mallet.fst.SimpleTagger --model-file  $model_file_name $test_data_file_name `;

    return [ split "\n", $output];
}

sub run_model_on_array
{
    my ( $crf, $test_data_array ) = @_;

    my $test_data = join "\n", @{ $test_data_array };

    my $foo = model_runner->run_model_string( $test_data, $crf );

    return $foo;
}

sub run_model_inline_java
{
    my ( $model_file_name, $test_data_file, $output_fhs ) = @_;

    my $probabilities_fh = $output_fhs->{ probabilities_fh };

    my $predictions_fh = $output_fhs->{ predictions_fh };

    my $expected_results_fh = $output_fhs->{ expected_results_fh };

    my $create_model_script_path = "$HOME/Applications/mallet-2.0.7/run_simple_tagger.sh";

    say STDERR "generating predictions";

    say STDERR "classpath: $class_path";

    
    # use Inline (
    #     Java  => 'STUDY',
    #     STUDY => [
    #         qw ( cc.mallet.fst.SimpleTagger
    #           java.io.FileReader java.io.File )
    #     ],
    #     AUTOSTUDY => 1,
    #     CLASSPATH => $class_path,
    #     PACKAGE => 'main'
    # );

    open my $test_data_file_fh, '<', $test_data_file;

    my @test_data_array = <$test_data_file_fh>;

    my $foo = run_model_inline_java_data_array( $model_file_name, \@test_data_array );

    say join "\n", @{ $foo };

    exit();
}

sub run_model
{
    my ( $model_file_name, $test_data_file, $output_fhs ) = @_;

    return run_model_inline_java( $model_file_name, $test_data_file, $output_fhs );
}

sub train_and_test
{
{'fraction_non_alphanumeric': 0.138265556099951, 'fraction_numerical': 0.002155805977462028, 'mean_word_length': 3.2034596375617794, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '614723', 'n_tokens_mistral': 3949, 'n_tokens_neox': 3761, 'n_words': 735}