<commit_before>package CRF::CrfUtils::WebService;

#
# Package to access CRF model runner via web service
#

use strict;
use warnings;

use MediaWords::Util::Config;
use MediaWords::Util::CrfExtractor;
use CRF::CrfUtils;

use MediaWords::Util::Web;
use HTTP::Request;
use HTTP::Status qw(:constants);
use Encode;
use URI;

use Data::Dumper;

use constant DEFAULT_CRF_PORT => 8441;

#
# Helpers
#

sub _fatal_error($)
{
    my $error_message = shift;
    CRF::CrfUtils::_fatal_error( $error_message );
}

sub _crf_server_url()
{
    my $config         = MediaWords::Util::Config->get_config();
    my $crf_server_url = $config->{ crf_web_service }->{ server_url };

    unless ( $crf_server_url )
    {
        _fatal_error( "Unable to determine CRF model runner web service URL to use." );
    }

    # Validate URL
    my $uri;
    eval { $uri = URI->new( $crf_server_url )->canonical; };
    if ( $@ )
    {
        _fatal_error( "Invalid CRF model runner web service URI: $crf_server_url" );
    }

    # If someone forgot to explicitly set the port
    my $default_protocol_port = $uri->default_port;    # e.g. 80
    if ( $uri->port == $default_protocol_port and ( $crf_server_url !~ /:$default_protocol_port/ ) )
    {
        warn( "CRF model runner web service URL's port was not set, to I'm setting it to " . DEFAULT_CRF_PORT );
        $uri->port( DEFAULT_CRF_PORT );
    }

    return $uri->as_string;
}

my $_crf_server_url = _crf_server_url();
say STDERR "CRF model runner web service URL: $_crf_server_url";

#
# CRF::CrfUtils "implementation"
#

sub create_model($$$)
{
    my ( $class, $training_data_file, $iterations ) = @_;

    # Clients should use CRF::CrfUtils::InlineJava directly
    _fatal_error( "Not implemented in " . __PACKAGE__ );
}

sub run_model_inline_java_data_array($$$)
{
    my ( $class, $model_file_name, $test_data_array ) = @_;

    _validate_model_file_name( $model_file_name );

    my $test_data = join "\n", @{ $test_data_array };

    # If test data is empty
    unless ( $test_data )
    {
        return [];
    }

    my $test_data_encoded;<commit_msg>Temporarily disable CRF::Webservices since we have an yet implemented probability suppor there.<commit_after>package CRF::CrfUtils::WebService;

#
# Package to access CRF model runner via web service
#

use strict;
use warnings;

use MediaWords::Util::Config;
use MediaWords::Util::CrfExtractor;
use CRF::CrfUtils;

use MediaWords::Util::Web;
use HTTP::Request;
use HTTP::Status qw(:constants);
use Encode;
use URI;

use Data::Dumper;

use constant DEFAULT_CRF_PORT => 8441;

#
# Helpers
#

sub _fatal_error($)
{
    my $error_message = shift;
    CRF::CrfUtils::_fatal_error( $error_message );
}

sub _crf_server_url()
{
    my $config         = MediaWords::Util::Config->get_config();
    my $crf_server_url = $config->{ crf_web_service }->{ server_url };

    unless ( $crf_server_url )
    {
        _fatal_error( "Unable to determine CRF model runner web service URL to use." );
    }

    # Validate URL
    my $uri;
    eval { $uri = URI->new( $crf_server_url )->canonical; };
    if ( $@ )
    {
        _fatal_error( "Invalid CRF model runner web service URI: $crf_server_url" );
    }

    # If someone forgot to explicitly set the port
    my $default_protocol_port = $uri->default_port;    # e.g. 80
    if ( $uri->port == $default_protocol_port and ( $crf_server_url !~ /:$default_protocol_port/ ) )
    {
        warn( "CRF model runner web service URL's port was not set, to I'm setting it to " . DEFAULT_CRF_PORT );
        $uri->port( DEFAULT_CRF_PORT );
    }

    return $uri->as_string;
}

my $_crf_server_url = _crf_server_url();
say STDERR "CRF model runner web service URL: $_crf_server_url";

#
# CRF::CrfUtils "implementation"
#

sub create_model($$$)
{
    my ( $class, $training_data_file, $iterations ) = @_;

    # Clients should use CRF::CrfUtils::InlineJava directly
    _fatal_error( "Not implemented in " . __PACKAGE__ );
}

sub run_model_inline_java_data_array($$$)
{
    my ( $class, $model_file_name, $test_data_array ) = @_;

    _fatal_error(
        "CRF Webservice is disabled because it does not support probability results which are now required by CrfUtils." );

    _validate_model_file_name( $model_file_name );

    my $test_data = join "\n", @{ $test_data_array };

    # If test data is empty
    unless ( $test_data )
    {
        return [];
    }

    my $test_data_encoded;
{'fraction_non_alphanumeric': 0.12361581920903955, 'fraction_numerical': 0.002711864406779661, 'mean_word_length': 3.516326530612245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530596', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1528, 'n_words': 392}