<commit_before>package MediaWords::Tagger::Calais;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use strict;

# generate tags for text using calais

use Encode;
use HTTP::Request::Common;
use URI::Escape;
use URI::Split;
use XML::TreePP;
use RDF::Simple::Parser;
use Net::Calais;
use DateTime;

use MediaWords::Util::Config;

use List::Util qw (min max);

#specifies the maximum time that we will wait before retrying a request
use constant MAX_WAIT_SECONDS => 60 * 60 * 3;

# number of times to retry when we get an overlimit error
use constant MAX_OVERLIMIT_RETRIES => 25;

# number of times to retry when we get any non-overlimit error
use constant MAX_ERROR_RETRIES => 10;

# string in calais response that indicates an overlimit error
use constant CALAIS_OVERLIMIT_ERROR => "403 Forbidden";

# try to fetch the tags from calais once
sub fetch_tags
{
    my ( $content ) = @_;

    $content =~ s/[^[:ascii:]]/ /g;

    my $key = MediaWords::Util::Config::get_config->{ mediawords }->{ calais_key };

    #Uncomment this line to change the opencalais server.  We needed this in the 4.0 change over but probably don't now...
    #$Net::Calais::CALAIS_URL = 'http://api1.opencalais.com/enlighten/rest/';

    my $calais = Net::Calais->new( apikey => $key );

    my $calais_response;
    {
        no warnings;
        $calais_response = $calais->enlighten( $content, contentType => "text/txt", outputFormat => "xml/rdf" );
    }

    if ( !$calais_response )
    {
        if ( $calais->{ error } eq CALAIS_OVERLIMIT_ERROR )
        {
            return { error => "over calais request limit", overlimit => 1 };
        }
        else
        {
            return { error => "Calais error: $calais->{error}" };
        }
    }

    if ( index( $calais_response, '<Exception>' ) > 0 )
    {
        return { error => "Calais exception: $calais_response" };
    }

    my $begin_terms = index( $calais_response, '-->' ) + 7;
    my $end_terms = index( $calais_response, '-->', $begin_terms );
    my $terms = substr( $calais_response, $begin_terms, $end_terms - $begin_terms );
<commit_msg>output Calais error to STDOUT so as not to wait for the term extraction to finish<commit_after>package MediaWords::Tagger::Calais;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use strict;

# generate tags for text using calais

use Encode;
use HTTP::Request::Common;
use URI::Escape;
use URI::Split;
use XML::TreePP;
use RDF::Simple::Parser;
use Net::Calais;
use DateTime;

use MediaWords::Util::Config;

use List::Util qw (min max);

#specifies the maximum time that we will wait before retrying a request
use constant MAX_WAIT_SECONDS => 60 * 60 * 3;

# number of times to retry when we get an overlimit error
use constant MAX_OVERLIMIT_RETRIES => 25;

# number of times to retry when we get any non-overlimit error
use constant MAX_ERROR_RETRIES => 10;

# string in calais response that indicates an overlimit error
use constant CALAIS_OVERLIMIT_ERROR => "403 Forbidden";

# try to fetch the tags from calais once
sub fetch_tags
{
    my ( $content ) = @_;

    $content =~ s/[^[:ascii:]]/ /g;

    my $key = MediaWords::Util::Config::get_config->{ mediawords }->{ calais_key };

    #Uncomment this line to change the opencalais server.  We needed this in the 4.0 change over but probably don't now...
    #$Net::Calais::CALAIS_URL = 'http://api1.opencalais.com/enlighten/rest/';

    my $calais = Net::Calais->new( apikey => $key );

    my $calais_response;
    {
        no warnings;
        $calais_response = $calais->enlighten( $content, contentType => "text/txt", outputFormat => "xml/rdf" );
    }

    if ( !$calais_response )
    {
        if ( $calais->{ error } eq CALAIS_OVERLIMIT_ERROR )
        {
            return { error => "over calais request limit", overlimit => 1 };
        }
        else
        {
            return { error => "Calais error: $calais->{error}" };
        }
    }

    if ( index( $calais_response, '<Exception>' ) > 0 )
    {
        say STDERR "Calais exception: $calais_response";
        return { error => "Calais exception: $calais_response" };
    }

    my $begin_terms = index( $calais_response, '-->' ) + 7;
    my $end_terms = index( $calais_response, '-->', $begin_terms );
    my $terms = substr( $calais_response, $begin_terms, $end_terms - $begin_terms );

{'fraction_non_alphanumeric': 0.12890534598472575, 'fraction_numerical': 0.010182828049062717, 'mean_word_length': 3.291956305858987, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6864321', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1399, 'n_words': 432}