<commit_before>package MediaWords::Job::ExtractAndVector;

#
# Extract a story
#
# Start this worker script by running:
#
# ./script/run_with_carton.sh local/bin/mjm_worker.pl lib/MediaWords/Job/ExtractAndVector.pm
#

use strict;
use warnings;

use Moose;
with 'MediaWords::AbstractJob';

BEGIN
{
    use FindBin;

    # "lib/" relative to "local/bin/mjm_worker.pl":
    use lib "$FindBin::Bin/../../lib";
}

use Modern::Perl "2015";
use MediaWords::CommonLibs;

use MediaWords::DB;
use MediaWords::DBI::Stories;
use MediaWords::DBI::Stories::ExtractorArguments;

# Extract, vector and process a story; LOGDIE() and / or return
# false on error.
#
# Arguments:
# * stories_id -- story ID to extract
# * (optional) disable_story_triggers -- disable triggers on "stories" table
#              (probably skips updating db_row_last_updated?)
# * (optional) skip_bitly_processing -- don't add extracted story to the Bit.ly
#              processing queue
# * (optional) skip_corenlp_annotation -- don't add extracted story to the
#              CoreNLP annotation queue
sub run($$)
{
    my ( $self, $args ) = @_;

    unless ( $args->{ stories_id } )
    {
        LOGDIE "stories_id is not set.";
    }

    my $stories_id = $args->{ stories_id };

    my $db = MediaWords::DB::connect_to_db();
    $db->set_autocommit( 0 );

    if ( exists $args->{ disable_story_triggers } and $args->{ disable_story_triggers } )
    {
        $db->query( "SELECT disable_story_triggers(); " );
        MediaWords::DB::disable_story_triggers();
    }
    else
    {
        $db->query( "SELECT enable_story_triggers(); " );
        MediaWords::DB::enable_story_triggers();
    }

    my $extractor_args = MediaWords::DBI::Stories::ExtractorArguments->new(
        {
            skip_bitly_processing   => $args->{ skip_bitly_processing },
            skip_corenlp_annotation => $args->{ skip_corenlp_annotation },
            use_cache               => $args->{ use_cache }
        }
    );

    eval {
        my $story = $db->find_by_id( 'stories', $stories_id );
        unless ( $story->{ stories_id } )
        {
            LOGDIE "Story with ID $stories_id was not found.";
        }

        MediaWords::DBI::Stories::extract_and_process_story( $db, $story, $extractor_args );

        # Enable story triggers in case the connection is reused due to connection pooling
        $db->query( "SELECT enable_story_triggers(); " );
    };
    if ( $@ )
    {
        LOGDIE "Extractor died: $@; job args: " . Dumper( $args );
    }

    return 1;
}

no Moose;    # gets rid of scaffolding

# Return package name instead of 1 or otherwise worker.pl won't know the name of the package it's loading
__PACKAGE__;<commit_msg>Remove useless set_autocommit(0), replace with transaction block<commit_after>package MediaWords::Job::ExtractAndVector;

#
# Extract a story
#
# Start this worker script by running:
#
# ./script/run_with_carton.sh local/bin/mjm_worker.pl lib/MediaWords/Job/ExtractAndVector.pm
#

use strict;
use warnings;

use Moose;
with 'MediaWords::AbstractJob';

BEGIN
{
    use FindBin;

    # "lib/" relative to "local/bin/mjm_worker.pl":
    use lib "$FindBin::Bin/../../lib";
}

use Modern::Perl "2015";
use MediaWords::CommonLibs;

use MediaWords::DB;
use MediaWords::DBI::Stories;
use MediaWords::DBI::Stories::ExtractorArguments;

# Extract, vector and process a story; LOGDIE() and / or return
# false on error.
#
# Arguments:
# * stories_id -- story ID to extract
# * (optional) disable_story_triggers -- disable triggers on "stories" table
#              (probably skips updating db_row_last_updated?)
# * (optional) skip_bitly_processing -- don't add extracted story to the Bit.ly
#              processing queue
# * (optional) skip_corenlp_annotation -- don't add extracted story to the
#              CoreNLP annotation queue
sub run($$)
{
    my ( $self, $args ) = @_;

    unless ( $args->{ stories_id } )
    {
        LOGDIE "stories_id is not set.";
    }

    my $stories_id = $args->{ stories_id };

    my $db = MediaWords::DB::connect_to_db();

    $db->begin;

    if ( exists $args->{ disable_story_triggers } and $args->{ disable_story_triggers } )
    {
        $db->query( "SELECT disable_story_triggers(); " );
        MediaWords::DB::disable_story_triggers();
    }
    else
    {
        $db->query( "SELECT enable_story_triggers(); " );
        MediaWords::DB::enable_story_triggers();
    }

    my $extractor_args = MediaWords::DBI::Stories::ExtractorArguments->new(
        {
            skip_bitly_processing   => $args->{ skip_bitly_processing },
            skip_corenlp_annotation => $args->{ skip_corenlp_annotation },
            use_cache               => $args->{ use_cache }
        }
    );

    eval {
        my $story = $db->find_by_id( 'stories', $stories_id );
        unless ( $story->{ stories_id } )
        {
            LOGDIE "Story with ID $stories_id was not found.";
        }

        MediaWords::DBI::Stories::extract_and_process_story( $db, $story, $extractor_args );

        # Enable story triggers in case the connection is reused due to connection pooling
        $db->query( "SELECT enable_story_triggers(); " );
    };
    if ( $@ )
    {
        LOGDIE "Extractor died: $@; job args: " . Dumper( $args );
    }

    $db->commit;

    return 1;
}

no Moose;    # gets rid of scaffolding

# Return package name instead of 1 or otherwise worker.pl won't know the name of the package it's loading
__PACKAGE__;
{'fraction_non_alphanumeric': 0.1285662033650329, 'fraction_numerical': 0.002560351133869788, 'mean_word_length': 2.8759744861800143, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3504325', 'n_tokens_mistral': 1820, 'n_tokens_neox': 1731, 'n_words': 439}