<commit_before>#!/usr/bin/env perl

#
# fetch twitter and facebook statistics for all stories in a controversy
#

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use Getopt::Long;

use MediaWords::DB;
use MediaWords::CM;
use MediaWords::Util::Facebook;
use MediaWords::GearmanFunction;
use MediaWords::GearmanFunction::Twitter::FetchStoryStats;

sub main
{
    my ( $controversy_opt, $direct_job );

    binmode( STDOUT, 'utf8' );
    binmode( STDERR, 'utf8' );
    $| = 1;

    Readonly my $usage => <<EOF;
Usage: $0 --controversy < id > [--direct_job]
EOF

    Getopt::Long::GetOptions(
        "controversy=s" => \$controversy_opt,
        "direct_job!"   => \$direct_job
    ) or die $usage;
    die $usage unless ( $controversy_opt );

    unless ( $direct_job )
    {
        unless ( MediaWords::GearmanFunction::gearman_is_enabled() )
        {
            die "Gearman is disabled.";
        }
    }

    my $db = MediaWords::DB::connect_to_db;
    my $controversies = MediaWords::CM::require_controversies_by_opt( $db, $controversy_opt );
    unless ( $controversies )
    {
        die "Unable to find controversies for option '$controversy_opt'";
    }

    for my $controversy ( @{ $controversies } )
    {
        my $controversies_id = $controversy->{ controversies_id };

        my $stories = $db->query( <<END, $controversies_id )->hashes;
            SELECT stories_id
            FROM controversy_stories
            WHERE controversies_id = ?
END

        unless ( scalar @{ $stories } )
        {
            say STDERR "No stories found for controversy '$controversy->{ name }' ('$controversy_opt')";
        }

        for my $story ( @{ $stories } )
        {
            my $stories_id = $story->{ stories_id };
            my $args = { stories_id => $stories_id };

            if ( $direct_job )
            {
                say STDERR "Running local job for story $stories_id...";
                MediaWords::GearmanFunction::Twitter::FetchStoryStats->run_locally( $args );
            }
            else
            {
                say STDERR "Enqueueing Gearman job for story $stories_id...";
                MediaWords::GearmanFunction::Twitter::FetchStoryStats->enqueue_on_gearman( $args );
            }

            # if ( !$ss || $ss->{ facebook_share_count_error} || !defined( $ss->{ facebook_share_count } ) )
            # {
            #     my $count = MediaWords::Util::Facebook::get_and_store_share_count( $db, $story );
            #     say STDERR "facebook_share_count: $count";
            # }
        }
    }
}

main();<commit_msg>Don't enqueue stories that already have their stats fetched<commit_after>#!/usr/bin/env perl

#
# fetch twitter and facebook statistics for all stories in a controversy
#

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use Getopt::Long;

use MediaWords::DB;
use MediaWords::CM;
use MediaWords::Util::Facebook;
use MediaWords::GearmanFunction;
use MediaWords::GearmanFunction::Twitter::FetchStoryStats;

sub main
{
    my ( $controversy_opt, $direct_job );

    binmode( STDOUT, 'utf8' );
    binmode( STDERR, 'utf8' );
    $| = 1;

    Readonly my $usage => <<EOF;
Usage: $0 --controversy < id > [--direct_job]
EOF

    Getopt::Long::GetOptions(
        "controversy=s" => \$controversy_opt,
        "direct_job!"   => \$direct_job
    ) or die $usage;
    die $usage unless ( $controversy_opt );

    unless ( $direct_job )
    {
        unless ( MediaWords::GearmanFunction::gearman_is_enabled() )
        {
            die "Gearman is disabled.";
        }
    }

    my $db = MediaWords::DB::connect_to_db;
    my $controversies = MediaWords::CM::require_controversies_by_opt( $db, $controversy_opt );
    unless ( $controversies )
    {
        die "Unable to find controversies for option '$controversy_opt'";
    }

    for my $controversy ( @{ $controversies } )
    {
        my $controversies_id = $controversy->{ controversies_id };

        my $stories = $db->query( <<END, $controversies_id )->hashes;
            SELECT stories_id
            FROM controversy_stories
            WHERE controversies_id = ?
END

        unless ( scalar @{ $stories } )
        {
            say STDERR "No stories found for controversy '$controversy->{ name }' ('$controversy_opt')";
        }

        for my $story ( @{ $stories } )
        {
            my $stories_id = $story->{ stories_id };
            my $args = { stories_id => $stories_id };

            my $ss = $db->query( "select * from story_statistics where stories_id = ?", $stories_id )->hash;

            if ( !$ss or $ss->{ twitter_url_tweet_count_error } or !defined( $ss->{ twitter_url_tweet_count } ) )
            {
                if ( $direct_job )
                {
                    say STDERR "Running local job for story $stories_id...";
                    MediaWords::GearmanFunction::Twitter::FetchStoryStats->run_locally( $args );
                }
                else
                {
                    say STDERR "Enqueueing Gearman job for story $stories_id...";
                    MediaWords::GearmanFunction::Twitter::FetchStoryStats->enqueue_on_gearman( $args );
                }

                # if ( !$ss || $ss->{ facebook_share_count_error} || !defined( $ss->{ facebook_share_count } ) )
                # {
                #     my $count = MediaWords::Util::Facebook::get_and_store_share_count( $db, $story );
                #     say STDERR "facebook_share_count: $count";
                # }
            }
        }
    }
}

main();
{'fraction_non_alphanumeric': 0.12928621413575928, 'fraction_numerical': 0.0027991602519244225, 'mean_word_length': 2.2335972850678734, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4887857', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1782, 'n_words': 398}