<commit_before>#!/usr/bin/env perl
#
# Enqueue stories that have their Bit.ly stats fetched but not yet aggregated

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;
use MediaWords::DB;
use MediaWords::Util::Bitly;
use MediaWords::CM;
use MediaWords::GearmanFunction::Bitly::AggregateStoryStats;

use Getopt::Long;
use Readonly;

sub main
{
    my ( $controversy_opt );

    binmode( STDOUT, 'utf8' );
    binmode( STDERR, 'utf8' );

    Readonly my $usage => "usage: $0 --controversy < controversy id or pattern >";

    Getopt::Long::GetOptions( "controversy=s" => \$controversy_opt, ) or die $usage;

    die $usage unless ( $controversy_opt );

    my $db = MediaWords::DB::connect_to_db;

    my $controversies = MediaWords::CM::require_controversies_by_opt( $db, $controversy_opt );

    for my $controversy ( @{ $controversies } )
    {
        my $controversies_id = $controversy->{ controversies_id };

        if ( MediaWords::Util::Bitly::num_controversy_stories_without_bitly_statistics( $db, $controversies_id ) == 0 )
        {
            say STDERR "All controversy's $controversies_id stories are processed against Bit.ly.";
            next;
        }

        my $unprocessed_stories = $db->query(
            <<EOF,<commit_msg>In "enqueue controversy for Bit.ly stats aggregation" helper script, check if Bit.ly processing is enabled and whether each controversy is enabled for Bit.ly processing before continuing<commit_after>#!/usr/bin/env perl
#
# Enqueue stories that have their Bit.ly stats fetched but not yet aggregated

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;
use MediaWords::DB;
use MediaWords::Util::Bitly;
use MediaWords::CM;
use MediaWords::GearmanFunction::Bitly::AggregateStoryStats;

use Getopt::Long;
use Readonly;

sub main
{
    my ( $controversy_opt );

    binmode( STDOUT, 'utf8' );
    binmode( STDERR, 'utf8' );

    Readonly my $usage => "usage: $0 --controversy < controversy id or pattern >";

    Getopt::Long::GetOptions( "controversy=s" => \$controversy_opt, ) or die $usage;

    die $usage unless ( $controversy_opt );

    my $db = MediaWords::DB::connect_to_db;

    unless ( MediaWords::Util::Bitly::bitly_processing_is_enabled() )
    {
        die "Bit.ly processing is not enabled.";
    }

    my $controversies = MediaWords::CM::require_controversies_by_opt( $db, $controversy_opt );

    for my $controversy ( @{ $controversies } )
    {
        my $controversies_id = $controversy->{ controversies_id };

        unless ( $controversy->{ process_with_bitly } )
        {
            say STDERR "Controversy $controversies_id is not set up for Bit.ly, skipping.";
            next;
        }

        if ( MediaWords::Util::Bitly::num_controversy_stories_without_bitly_statistics( $db, $controversies_id ) == 0 )
        {
            say STDERR "All controversy's $controversies_id stories are processed against Bit.ly, skipping.";
            next;
        }

        my $unprocessed_stories = $db->query(
            <<EOF,
{'fraction_non_alphanumeric': 0.12280150753768844, 'fraction_numerical': 0.005025125628140704, 'mean_word_length': 3.4174757281553396, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1085028', 'n_tokens_mistral': 1122, 'n_tokens_neox': 1029, 'n_words': 262}