<commit_before>package MediaWords::Controller::Api::V2::ExtractLines;

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use MediaWords::DBI::StorySubsets;
use strict;
use warnings;
use base 'Catalyst::Controller::REST';
use JSON;
use Encode;
use utf8;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use Moose;
use namespace::autoclean;
use List::Compare;
use Carp;

=head1 NAME

MediaWords::Controller::Api::V2::ExtractLines - Catalyst Controller

=head1 DESCRIPTION

Catalyst Controller.

=head1 METHODS

=cut

=head2 index 

=cut

BEGIN { extends 'MediaWords::Controller::Api::V2::MC_Controller_REST' }

# Default authentication action roles
__PACKAGE__->config(    #
    action => {         #
        story_lines_GET => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },    #
      }    #
);         #

use constant ROWS_PER_PAGE => 20;

use MediaWords::Tagger;

sub story_lines : Local : ActionClass('REST')
{
}

sub story_lines_GET : Local
{
    my ( $self, $c ) = @_;
    my $body_html = $c->req->param( 'body_html' );
    $body_html //= ROWS_PER_PAGE;

    # say STDERR "rows $rows";

    my $lines = [ split( /[\n\r]+/, $body_html ) ];

    $lines = MediaWords::Crawler::Extractor::preprocess( $lines );

    $self->status_ok( $c, entity => $lines );
}

sub extract : Local : ActionClass('REST')    # action roles are to be set for each derivative sub-actions
{
}

sub extract_GET : Local
{
    my ( $self, $c ) = @_;

    my $temp               = $c->req->param( 'preprocessed_lines' );
    my $pp_lines           = encode_utf8( $temp );
    my $preprocessed_lines = decode_json( $pp_lines );
    my $story_title        = $c->req->param( 'story_title' );
    my $story_description  = $c->req->param( 'story_description' );

    my $extractor_method = $c->req->param( 'extractor_method' );

    if ( defined( $extractor_method ) )
    {
        die unless ( $extractor_method eq 'HeuristicExtractor' ) or ( $extractor_method eq 'CrfExtractor' );
    }

    die unless defined( $preprocessed_lines );

    my $extractor = MediaWords::Util::ExtractorFactory::createExtractor( $extractor_method );
<commit_msg>Convert story_lines API access to use PUT to avoid URL limits.<commit_after>package MediaWords::Controller::Api::V2::ExtractLines;

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use MediaWords::DBI::StorySubsets;
use strict;
use warnings;
use base 'Catalyst::Controller::REST';
use JSON;
use Encode;
use utf8;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use Moose;
use namespace::autoclean;
use List::Compare;
use Carp;

=head1 NAME

MediaWords::Controller::Api::V2::ExtractLines - Catalyst Controller

=head1 DESCRIPTION

Catalyst Controller.

=head1 METHODS

=cut

=head2 index 

=cut

BEGIN { extends 'MediaWords::Controller::Api::V2::MC_Controller_REST' }

# Default authentication action roles
__PACKAGE__->config(    #
    action => {         #
        story_lines_GET => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },    #
              # story_lines => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },    #
        story_lines_PUT => { Does => [ qw( ~PublicApiKeyAuthenticated ~Throttled ~Logged ) ] },    #
      }    #
);         #

use constant ROWS_PER_PAGE => 20;

use MediaWords::Tagger;

sub story_lines : Local : ActionClass('REST')
{
}

sub story_lines_GET : Local
{
}

sub story_lines_PUT : Local
{
    my ( $self, $c ) = @_;

    #say STDERR Dumper( $c->req->params );
    #say STDERR Dumper( $c->req->data );

    say STDERR "body_html";
    my $body_html = $c->req->data->{ body_html };

    say STDERR "body_html";
    say STDERR length( $body_html );

    my $lines = [ split( /[\n\r]+/, $body_html ) ];

    $lines = MediaWords::Crawler::Extractor::preprocess( $lines );

    $self->status_ok( $c, entity => $lines );
}

# sub story_lines_GET : Local
# {
#     my ( $self, $c ) = @_;

#     say STDERR "body_html";

#     my $body_html = $c->req->param( 'body_html' );

#     #$body_html //= ROWS_PER_PAGE;

#     say STDERR "body_html";
#     say STDERR length($body_html);

#     my $lines = [ split( /[\n\r]+/, $body_html ) ];

#     $lines = MediaWords::Crawler::Extractor::preprocess( $lines );

#     $self->status_ok( $c, entity => $lines );
# }

sub extract : Local : ActionClass('REST')    # action roles are to be set for each derivative sub-actions
{
}

sub extract_GET : Local
{
    my ( $self, $c ) = @_;

    my $temp               = $c->req->param( 'preprocessed_lines' );
    my $pp_lines           = encode_utf8( $temp );
    my $preprocessed_lines = decode_json( $pp_lines );
    my $story_title        = $c->req->param( 'story_title' );
    my $story_description  = $c->req->param( 'story_description' );

    my $extractor_method = $c->req->param( 'extractor_method' );

    if ( defined( $extractor_method ) )
    {
        die unless ( $extractor_method eq 'HeuristicExtractor' ) or ( $extractor_method eq 'CrfExtractor' );
    }

    die unless defined( $preprocessed_lines );

    my $extractor = MediaWords::Util::ExtractorFactory::createExtractor( $extractor_method );

{'fraction_non_alphanumeric': 0.14699029126213592, 'fraction_numerical': 0.005825242718446602, 'mean_word_length': 3.494764397905759, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3513883', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1758, 'n_words': 414}