<commit_before>package MediaWords::GearmanFunction::Bitly::EnqueueStory;

#
# Enqueue a single story for processing via Bit.ly API based on its "publish_date"
#
# Start this worker script by running:
#
# ./script/run_with_carton.sh local/bin/gjs_worker.pl lib/MediaWords/GearmanFunction/Bitly/EnqueueStory.pm
#

use strict;
use warnings;

use Moose;

# Don't log each and every extraction job into the database
with 'Gearman::JobScheduler::AbstractFunction';

BEGIN
{
    use FindBin;

    # "lib/" relative to "local/bin/gjs_worker.pl":
    use lib "$FindBin::Bin/../../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use MediaWords::DB;
use MediaWords::Util::GearmanJobSchedulerConfiguration;
use MediaWords::Util::Bitly;
use MediaWords::Util::DateTime;
use MediaWords::Util::SQL;
use MediaWords::Util::URL;
use MediaWords::GearmanFunction::Bitly::FetchStoryURLStats;
use Readonly;
use DateTime;

# Having a global database object should be safe because
# Gearman::JobScheduler's workers don't support fork()s anymore
my $db = undef;

# Don't fetch data for stories older than this date
sub _publish_timestamp_lower_bound()
{
    return DateTime->new( year => 2008, month => 01, day => 01 )->epoch;
}

# Don't fetch data for stories newer than this date
sub _publish_timestamp_upper_bound()
{
    return DateTime->now()->epoch;
}

# Run job
sub run($;$)
{
    my ( $self, $args ) = @_;

    unless ( MediaWords::Util::Bitly::bitly_processing_is_enabled() )
    {
        die "Bit.ly processing is not enabled.";
    }

    # Postpone connecting to the database so that compile test doesn't do that
    $db ||= MediaWords::DB::connect_to_db();

    my $stories_id = $args->{ stories_id } or die "'stories_id' is not set.";

    say STDERR "Will enqueue story $stories_id for Bit.ly processing.";

    my $story = $db->find_by_id( 'stories', $stories_id );
    unless ( $story )
    {
        die "Story $stories_id was not found";
    }

    my $stories_url          = $story->{ url };
    my $stories_publish_date = $story->{ publish_date };

    unless ( $stories_url )
    {
        die "URL is unset for story $stories_id";
    }
    unless ( MediaWords::Util::URL::is_http_url( $stories_url ) )
    {
        die "URL '$stories_url' is not a HTTP(S) URL for story $stories_id";
    }

    unless ( $stories_publish_date )
    {
        die "Publish date is unset for story $stories_id";
    }

    my $publish_timestamp = MediaWords::Util::SQL::get_epoch_from_sql_date( $stories_publish_date );
    if ( $publish_timestamp <= _publish_timestamp_lower_bound() )
    {
        die "Publish timestamp is lower than the lower bound for story $stories_id";
    }
    if ( $publish_timestamp >= _publish_timestamp_upper_bound() )
    {
        die "Publish timestamp is bigger than the upper bound for story $stories_id";
    }

    # Round timestamp to the nearest day because that's what Bitly.pm does
    my $publish_datetime = gmt_datetime_from_timestamp( $publish_timestamp );
    $publish_datetime->set( hour => 0, minute => 0, second => 0 );
    $publish_timestamp = $publish_datetime->epoch;

    # Span across ~300 days
    my $start_timestamp = $publish_timestamp - ( 60 * 60 * 24 * 150 );
    my $end_timestamp   = $publish_timestamp + ( 60 * 60 * 24 * 150 );

    say STDERR "Enqueueing story $stories_id for Bit.ly processing (start TS: $start_timestamp, end TS: $end_timestamp)...";

    MediaWords::GearmanFunction::Bitly::FetchStoryURLStats->enqueue_on_gearman(
        {
            stories_id      => $stories_id,
            start_timestamp => $start_timestamp,
            end_timestamp   => $end_timestamp
        }
    );

    say STDERR "Done enqueueing story $stories_id for Bit.ly processing.";
}

# write a single log because there are a lot of Bit.ly processing jobs so it's
# impractical to log each job into a separate file
sub unify_logs()
{
    return 1;
}
<commit_msg>Fetch stats for 2 days in the past, 30 days in the future from "publish_date"<commit_after>package MediaWords::GearmanFunction::Bitly::EnqueueStory;

#
# Enqueue a single story for processing via Bit.ly API based on its "publish_date"
#
# Start this worker script by running:
#
# ./script/run_with_carton.sh local/bin/gjs_worker.pl lib/MediaWords/GearmanFunction/Bitly/EnqueueStory.pm
#

use strict;
use warnings;

use Moose;

# Don't log each and every extraction job into the database
with 'Gearman::JobScheduler::AbstractFunction';

BEGIN
{
    use FindBin;

    # "lib/" relative to "local/bin/gjs_worker.pl":
    use lib "$FindBin::Bin/../../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use MediaWords::DB;
use MediaWords::Util::GearmanJobSchedulerConfiguration;
use MediaWords::Util::Bitly;
use MediaWords::Util::DateTime;
use MediaWords::Util::SQL;
use MediaWords::Util::URL;
use MediaWords::GearmanFunction::Bitly::FetchStoryURLStats;
use Readonly;
use DateTime;

# Having a global database object should be safe because
# Gearman::JobScheduler's workers don't support fork()s anymore
my $db = undef;

# Don't fetch data for stories older than this date
sub _publish_timestamp_lower_bound()
{
    return DateTime->new( year => 2008, month => 01, day => 01 )->epoch;
}

# Don't fetch data for stories newer than this date
sub _publish_timestamp_upper_bound()
{
    return DateTime->now()->epoch;
}

# Run job
sub run($;$)
{
    my ( $self, $args ) = @_;

    unless ( MediaWords::Util::Bitly::bitly_processing_is_enabled() )
    {
        die "Bit.ly processing is not enabled.";
    }

    # Postpone connecting to the database so that compile test doesn't do that
    $db ||= MediaWords::DB::connect_to_db();

    my $stories_id = $args->{ stories_id } or die "'stories_id' is not set.";

    say STDERR "Will enqueue story $stories_id for Bit.ly processing.";

    my $story = $db->find_by_id( 'stories', $stories_id );
    unless ( $story )
    {
        die "Story $stories_id was not found";
    }

    my $stories_url          = $story->{ url };
    my $stories_publish_date = $story->{ publish_date };

    unless ( $stories_url )
    {
        die "URL is unset for story $stories_id";
    }
    unless ( MediaWords::Util::URL::is_http_url( $stories_url ) )
    {
        die "URL '$stories_url' is not a HTTP(S) URL for story $stories_id";
    }

    unless ( $stories_publish_date )
    {
        die "Publish date is unset for story $stories_id";
    }

    my $publish_timestamp = MediaWords::Util::SQL::get_epoch_from_sql_date( $stories_publish_date );
    if ( $publish_timestamp <= _publish_timestamp_lower_bound() )
    {
        die "Publish timestamp is lower than the lower bound for story $stories_id";
    }
    if ( $publish_timestamp >= _publish_timestamp_upper_bound() )
    {
        die "Publish timestamp is bigger than the upper bound for story $stories_id";
    }

    # Round timestamp to the nearest day because that's what Bitly.pm does
    my $publish_datetime = gmt_datetime_from_timestamp( $publish_timestamp );
    $publish_datetime->set( hour => 0, minute => 0, second => 0 );
    $publish_timestamp = $publish_datetime->epoch;

    # Span -2 days to the past (to account for TZ conversion errors), 30 days to the future
    my $start_timestamp = $publish_timestamp - ( 60 * 60 * 24 * 2 );
    my $end_timestamp   = $publish_timestamp + ( 60 * 60 * 24 * 30 );
    if ( $end_timestamp > _publish_timestamp_upper_bound() )
    {
        warn "End timestamp is in the future, so truncating to current timestamp; " .
          "consider fetching Bit.ly stats after a longer delay since 'publish_date'";
        $end_timestamp = DateTime->now()->epoch;
    }

    say STDERR "Enqueueing story $stories_id for Bit.ly processing (start TS: $start_timestamp, end TS: $end_timestamp)...";

    MediaWords::GearmanFunction::Bitly::FetchStoryURLStats->enqueue_on_gearman(
        {
            stories_id      => $stories_id,
            start_timestamp => $start_timestamp,
            end_timestamp   => $end_timestamp
        }
    );

    say STDERR "Done enqueueing story $stories_id for Bit.ly processing.";
}

# write a single log because there are a lot of Bit.ly processing jobs so it's
# impractical to log each job into a separate file
sub unify_logs()
{
    return 1;
}

{'fraction_non_alphanumeric': 0.11225603103406473, 'fraction_numerical': 0.008970784337495454, 'mean_word_length': 3.527991218441273, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1883793', 'n_tokens_mistral': 2734, 'n_tokens_neox': 2573, 'n_words': 778}