<commit_before>package MediaWords::Controller::Api::V2::Sentences;
use Modern::Perl "2015";
use MediaWords::CommonLibs;

use strict;
use warnings;
use base 'Catalyst::Controller';

use Date::Calc;
use JSON;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use Moose;
use namespace::autoclean;
use List::Compare;

use MediaWords::Solr;
use MediaWords::Solr::SentenceFieldCounts;

=head1 NAME

MediaWords::Controller::Media - Catalyst Controller

=head1 DESCRIPTION

Catalyst Controller.

=head1 METHODS

=cut

=head2 index

=cut

BEGIN { extends 'MediaWords::Controller::Api::V2::MC_REST_SimpleObject' }

__PACKAGE__->config(
    action => {
        single_GET      => { Does => [ qw( ~NonPublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        list_GET        => { Does => [ qw( ~NonPublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        put_tags_PUT    => { Does => [ qw( ~NonPublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        count_GET       => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        field_count_GET => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
    }
);

sub get_table_name
{
    return "story_sentences";
}

sub list : Local : ActionClass('MC_REST')
{
    #TRACE "starting Sentences/list";
}

# fill stories_ids temporary table with stories_ids from the given sentences
# and return the temp table name
sub _get_stories_ids_temporary_table
{
    my ( $db, $sentences ) = @_;

    $db->query( "create temporary table _stories_ids ( stories_id bigint )" );

    eval { $db->dbh->do( "copy _stories_ids from STDIN" ) };
    die( " Error on copy for _stories_ids: $@" ) if ( $@ );

    for my $ss ( @{ $sentences } )
    {
        eval { $db->dbh->pg_putcopydata( "$ss->{ stories_id }\n" ); };
        die( " Error on pg_putcopydata for _stories_ids: $@" ) if ( $@ );
    }

    eval { $db->dbh->pg_putcopyend(); };

    die( " Error on pg_putcopyend for _stories_ids: $@" ) if ( $@ );

    return '_stories_ids';
}

# attach the following fields to each sentence: sentence_number, media_id, publish_date, url, medium_name
sub _attach_data_to_sentences
{
    my ( $db, $sentences ) = @_;

    return unless ( @{ $sentences } );

    my $temp_stories_ids = _get_stories_ids_temporary_table( $db, $sentences );

    my $stories = $db->query( <<END )->hashes;
select s.publish_date, s.stories_id, s.url, m.name medium_name, s.media_id, 0 as sentence_number
    from stories s
        join $temp_stories_ids q on ( s.stories_id = q.stories_id )
        join media m on ( s.media_id = m.media_id )
END

    $db->query( "drop table $temp_stories_ids" );

    my $story_lookup = {};
    map { $story_lookup->{ $_->{ stories_id } } = $_ } @{ $stories };

    for my $sentence ( @{ $sentences } )
    {
        my $story = $story_lookup->{ $sentence->{ stories_id } };
        map { $sentence->{ $_ } = $story->{ $_ } } qw/sentence_number media_id publish_date url medium_name/;
    }
}

# return the solr sort param corresponding with the possible
# api params values of publish_date_asc, publish_date_desc, and random
sub _get_sort_param
{
    my ( $sort ) = @_;

    $sort //= 'publish_date_asc';

    $sort = lc( $sort );

    if ( $sort eq 'publish_date_asc' )
    {
        return 'publish_date asc';
    }
    elsif ( $sort eq 'publish_date_desc' )
    {
        return 'publish_date desc';
    }
    elsif ( $sort eq 'random' )
    {
        return 'random_1 asc';
    }
    else
    {
        die( "Unknown sort: $sort" );
    }
}

sub list_GET : Local
{
    my ( $self, $c ) = @_;

    # TRACE "starting list_GET";

    my $params = {};

    my $q  = $c->req->params->{ 'q' };
    my $fq = $c->req->params->{ 'fq' };

    my $start = $c->req->params->{ 'start' };
    my $rows  = $c->req->params->{ 'rows' };
    my $sort  = $c->req->params->{ 'sort' };

    $rows  //= 1000;
    $start //= 0;

    $params->{ q }     = $q;
    $params->{ fq }    = $fq;
    $params->{ start } = $start;
    $params->{ rows }  = $rows;

    $params->{ sort } = _get_sort_param( $sort ) if ( $rows );

    $rows = List::Util::min( $rows, 10000 );

    my $list = MediaWords::Solr::query( $c->dbis, $params, $c );

    #TRACE "Got List:\n" . Dumper( $list );
<commit_msg>remove blessed field returned by solr that is breaking json encoding<commit_after>package MediaWords::Controller::Api::V2::Sentences;
use Modern::Perl "2015";
use MediaWords::CommonLibs;

use strict;
use warnings;
use base 'Catalyst::Controller';

use Date::Calc;
use JSON;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use Moose;
use namespace::autoclean;
use List::Compare;

use MediaWords::Solr;
use MediaWords::Solr::SentenceFieldCounts;

=head1 NAME

MediaWords::Controller::Media - Catalyst Controller

=head1 DESCRIPTION

Catalyst Controller.

=head1 METHODS

=cut

=head2 index

=cut

BEGIN { extends 'MediaWords::Controller::Api::V2::MC_REST_SimpleObject' }

__PACKAGE__->config(
    action => {
        single_GET      => { Does => [ qw( ~NonPublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        list_GET        => { Does => [ qw( ~NonPublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        put_tags_PUT    => { Does => [ qw( ~NonPublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        count_GET       => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
        field_count_GET => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },
    }
);

sub get_table_name
{
    return "story_sentences";
}

sub list : Local : ActionClass('MC_REST')
{
    #TRACE "starting Sentences/list";
}

# fill stories_ids temporary table with stories_ids from the given sentences
# and return the temp table name
sub _get_stories_ids_temporary_table
{
    my ( $db, $sentences ) = @_;

    $db->query( "create temporary table _stories_ids ( stories_id bigint )" );

    eval { $db->dbh->do( "copy _stories_ids from STDIN" ) };
    die( " Error on copy for _stories_ids: $@" ) if ( $@ );

    for my $ss ( @{ $sentences } )
    {
        eval { $db->dbh->pg_putcopydata( "$ss->{ stories_id }\n" ); };
        die( " Error on pg_putcopydata for _stories_ids: $@" ) if ( $@ );
    }

    eval { $db->dbh->pg_putcopyend(); };

    die( " Error on pg_putcopyend for _stories_ids: $@" ) if ( $@ );

    return '_stories_ids';
}

# attach the following fields to each sentence: sentence_number, media_id, publish_date, url, medium_name
sub _attach_data_to_sentences
{
    my ( $db, $sentences ) = @_;

    return unless ( @{ $sentences } );

    my $temp_stories_ids = _get_stories_ids_temporary_table( $db, $sentences );

    my $stories = $db->query( <<END )->hashes;
select s.publish_date, s.stories_id, s.url, m.name medium_name, s.media_id, 0 as sentence_number
    from stories s
        join $temp_stories_ids q on ( s.stories_id = q.stories_id )
        join media m on ( s.media_id = m.media_id )
END

    $db->query( "drop table $temp_stories_ids" );

    my $story_lookup = {};
    map { $story_lookup->{ $_->{ stories_id } } = $_ } @{ $stories };

    for my $sentence ( @{ $sentences } )
    {
        my $story = $story_lookup->{ $sentence->{ stories_id } };
        map { $sentence->{ $_ } = $story->{ $_ } } qw/sentence_number media_id publish_date url medium_name/;
    }
}

# return the solr sort param corresponding with the possible
# api params values of publish_date_asc, publish_date_desc, and random
sub _get_sort_param
{
    my ( $sort ) = @_;

    $sort //= 'publish_date_asc';

    $sort = lc( $sort );

    if ( $sort eq 'publish_date_asc' )
    {
        return 'publish_date asc';
    }
    elsif ( $sort eq 'publish_date_desc' )
    {
        return 'publish_date desc';
    }
    elsif ( $sort eq 'random' )
    {
        return 'random_1 asc';
    }
    else
    {
        die( "Unknown sort: $sort" );
    }
}

sub list_GET : Local
{
    my ( $self, $c ) = @_;

    # TRACE "starting list_GET";

    my $params = {};

    my $q  = $c->req->params->{ 'q' };
    my $fq = $c->req->params->{ 'fq' };

    my $start = $c->req->params->{ 'start' };
    my $rows  = $c->req->params->{ 'rows' };
    my $sort  = $c->req->params->{ 'sort' };

    $rows  //= 1000;
    $start //= 0;

    $params->{ q }     = $q;
    $params->{ fq }    = $fq;
    $params->{ start } = $start;
    $params->{ rows }  = $rows;

    $params->{ sort } = _get_sort_param( $sort ) if ( $rows );

    $rows = List::Util::min( $rows, 10000 );

    my $list = MediaWords::Solr::query( $c->dbis, $params, $c );

    # quick hack to get rid of field that causes the json encoder to fail - hal
    delete( $list->{ responseHeader }->{ zkConnected } );

    #TRACE "Got List:\n" . Dumper( $list );

{'fraction_non_alphanumeric': 0.15471264367816093, 'fraction_numerical': 0.005057471264367816, 'mean_word_length': 2.9931161083065625, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '815903', 'n_tokens_mistral': 3224, 'n_tokens_neox': 3081, 'n_words': 769}