<commit_before>package MediaWords::Util::CrfExtractor;
use Modern::Perl "2013";
use MediaWords::CommonLibs;

use strict;

use Data::Dumper;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);

#use List::MoreUtils qw( :all);
use List::Compare::Functional qw (get_unique get_complement get_union_ref );
use IPC::Open2;
use Text::Trim;
use File::Spec;
use File::Basename;
use CRF::CrfUtils;

use Moose;

with 'MediaWords::Util::Extractor';

my $_model_file_name;

sub get_path_to_extractor_model()
{
    my $_dirname      = dirname( __FILE__ );
    my $_dirname_full = File::Spec->rel2abs( $_dirname );

    return "$_dirname_full/../../CRF/models/extractor_model";
}

BEGIN
{
    $_model_file_name = get_path_to_extractor_model();

    #say STDERR "model_file: $_model_file_name";
}

sub getScoresAndLines
{
    my ( $self, $line_info, $preprocessed_lines ) = @_;<commit_msg>New to-do<commit_after>package MediaWords::Util::CrfExtractor;
use Modern::Perl "2013";
use MediaWords::CommonLibs;

use strict;

use Data::Dumper;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);

#use List::MoreUtils qw( :all);
use List::Compare::Functional qw (get_unique get_complement get_union_ref );
use IPC::Open2;
use Text::Trim;
use File::Spec;
use File::Basename;
use CRF::CrfUtils;

use Moose;

with 'MediaWords::Util::Extractor';

my $_model_file_name;

# FIXME: make path to the extractor model configurable because that way Java
# wouldn't have to use hardcoded path to the extractor model
sub get_path_to_extractor_model()
{
    my $_dirname      = dirname( __FILE__ );
    my $_dirname_full = File::Spec->rel2abs( $_dirname );

    return "$_dirname_full/../../CRF/models/extractor_model";
}

BEGIN
{
    $_model_file_name = get_path_to_extractor_model();

    #say STDERR "model_file: $_model_file_name";
}

sub getScoresAndLines
{
    my ( $self, $line_info, $preprocessed_lines ) = @_;
{'fraction_non_alphanumeric': 0.12813807531380753, 'fraction_numerical': 0.006276150627615063, 'mean_word_length': 5.39799331103679, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4022783', 'n_tokens_mistral': 702, 'n_tokens_neox': 674, 'n_words': 167}