<commit_before>#!/usr/bin/env perl
#
# Bit.ly click distribution histogram
#
# Usage:
# ./script/run_with_carton.sh ./script/mediawords_bitly_click_distribution.pl [--limit 200] > bitly_click_distrib.csv
#

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;
use MediaWords::DB;
use MediaWords::Util::Bitly;
use MediaWords::Util::DateTime;
use MediaWords::Util::SQL;

use Getopt::Long;
use Readonly;
use DateTime;

sub main
{
    binmode( STDOUT, 'utf8' );
    binmode( STDERR, 'utf8' );

    my $limit = 200;

    Readonly my $usage => <<"EOF";
Usage: $0 [--limit $limit]
EOF

    Getopt::Long::GetOptions( 'limit=i' => \$limit, ) or die $usage;

    my $db = MediaWords::DB::connect_to_db;

    my $publish_timestamp_lower_bound = DateTime->new( year => 2008, month => 01, day => 01 )->epoch;
    my $publish_timestamp_upper_bound = DateTime->now()->epoch;

    # Buckets for the histogram
    my $buckets = [
        { from => undef,       to => -7 * 24 },
        { from => -7 * 24 + 1, to => -6 * 24 },
        { from => -6 * 24 + 1, to => -5 * 24 },
        { from => -5 * 24 + 1, to => -4 * 24 },
        { from => -4 * 24 + 1, to => -3 * 24 },
        { from => -3 * 24 + 1, to => -2 * 24 },
        { from => -2 * 24 + 1, to => -1 * 24 },
        { from => -1 * 24 + 1, to => 0 },
        { from => 1,           to => 1 * 24 },
        { from => 1 * 24 + 1,  to => 2 * 24 },
        { from => 2 * 24 + 1,  to => 3 * 24 },
        { from => 3 * 24 + 1,  to => 4 * 24 },
        { from => 4 * 24 + 1,  to => 5 * 24 },
        { from => 5 * 24 + 1,  to => 6 * 24 },
        { from => 6 * 24 + 1,  to => 7 * 24 },
        { from => 7 * 24 + 1,  to => undef },
    ];

    say STDERR "Fetching (up to) $limit story IDs...";
    my $story_ids = $db->query(
        <<EOF,
        SELECT object_id AS stories_id
        FROM bitly_processing_results
        ORDER BY RANDOM()
        LIMIT ?
EOF
        $limit
    )->flat;

    my $min_publish_timestamp = undef;
    my $max_publish_timestamp = undef;

    foreach my $stories_id ( @{ $story_ids } )
    {
        my $story = $db->find_by_id( 'stories', $stories_id );
        unless ( $story )
        {
            say STDERR "Unable to find story with ID $stories_id";
            next;
        }

        my $publish_date = $story->{ publish_date };
        unless ( $publish_date )
        {
            say STDERR "Publish date is unset for story $stories_id";
            next;
        }

        my $publish_timestamp = MediaWords::Util::SQL::get_epoch_from_sql_date( $publish_date );
        if ( $publish_timestamp <= $publish_timestamp_lower_bound )
        {
            say STDERR "Publish timestamp is lower than the lower bound for story $stories_id";
            next;
        }
        if ( $publish_timestamp >= $publish_timestamp_upper_bound )
        {
            say STDERR "Publish timestamp is bigger than the upper bound for story $stories_id";
            next;
        }

        # Round timestamp to the nearest day because that's what Bitly.pm does
        my $publish_datetime = gmt_datetime_from_timestamp( $publish_timestamp );
        $publish_datetime->set( hour => 0, minute => 0, second => 0 );
        $publish_timestamp = $publish_datetime->epoch;

        $min_publish_timestamp = $publish_timestamp
          if !defined $min_publish_timestamp or $min_publish_timestamp > $publish_timestamp;
        $max_publish_timestamp = $publish_timestamp
          if !defined $max_publish_timestamp or $max_publish_timestamp < $publish_timestamp;

        my $story_stats = MediaWords::Util::Bitly::read_story_stats( $db, $stories_id );
        unless ( $story_stats )
        {
            die "Unable to fetch Bit.ly story stats for story $stories_id";
        }

        foreach my $bitly_hash ( keys %{ $story_stats->{ data } } )
        {
            foreach my $bitly_clicks ( @{ $story_stats->{ data }->{ $bitly_hash }->{ clicks } } )
            {
                foreach my $link_click ( @{ $bitly_clicks->{ link_clicks } } )
                {<commit_msg>Fetch click stats live from Bit.ly API based on stories "publish_date" and not on controversy's start / end dates in order to avoid possible bias<commit_after>#!/usr/bin/env perl
#
# Bit.ly click distribution histogram
#
# Usage:
# ./script/run_with_carton.sh ./script/mediawords_bitly_click_distribution.pl [--limit 200] > bitly_click_distrib.csv
#

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;
use MediaWords::DB;
use MediaWords::Util::Bitly;
use MediaWords::Util::DateTime;
use MediaWords::Util::SQL;

use Getopt::Long;
use Readonly;
use DateTime;

sub main
{
    binmode( STDOUT, 'utf8' );
    binmode( STDERR, 'utf8' );

    my $limit = 200;

    Readonly my $usage => <<"EOF";
Usage: $0 [--limit $limit]
EOF

    Getopt::Long::GetOptions( 'limit=i' => \$limit, ) or die $usage;

    my $db = MediaWords::DB::connect_to_db;

    my $publish_timestamp_lower_bound = DateTime->new( year => 2008, month => 01, day => 01 )->epoch;
    my $publish_timestamp_upper_bound = DateTime->now()->epoch;

    # Buckets for the histogram
    my $buckets = [
        { from => undef,       to => -7 * 24 },
        { from => -7 * 24 + 1, to => -6 * 24 },
        { from => -6 * 24 + 1, to => -5 * 24 },
        { from => -5 * 24 + 1, to => -4 * 24 },
        { from => -4 * 24 + 1, to => -3 * 24 },
        { from => -3 * 24 + 1, to => -2 * 24 },
        { from => -2 * 24 + 1, to => -1 * 24 },
        { from => -1 * 24 + 1, to => 0 },
        { from => 1,           to => 1 * 24 },
        { from => 1 * 24 + 1,  to => 2 * 24 },
        { from => 2 * 24 + 1,  to => 3 * 24 },
        { from => 3 * 24 + 1,  to => 4 * 24 },
        { from => 4 * 24 + 1,  to => 5 * 24 },
        { from => 5 * 24 + 1,  to => 6 * 24 },
        { from => 6 * 24 + 1,  to => 7 * 24 },
        { from => 7 * 24 + 1,  to => undef },
    ];

    say STDERR "Fetching (up to) $limit stories...";
    my $stories = $db->query(
        <<EOF,
        SELECT *
        FROM stories
        WHERE stories_id IN (
            SELECT stories_id
            FROM controversy_stories
                INNER JOIN controversies
                    ON controversy_stories.controversies_id = controversies.controversies_id
            WHERE controversies.process_with_bitly = 't'
            ORDER BY RANDOM()
            LIMIT ?
        )
        ORDER BY stories_id
EOF
        $limit
    )->hashes;

    my $min_publish_timestamp = undef;
    my $max_publish_timestamp = undef;

    foreach my $story ( @{ $stories } )
    {
        my $stories_id = $story->{ stories_id };

        my $publish_date = $story->{ publish_date };
        unless ( $publish_date )
        {
            say STDERR "Publish date is unset for story $stories_id";
            next;
        }

        my $publish_timestamp = MediaWords::Util::SQL::get_epoch_from_sql_date( $publish_date );
        if ( $publish_timestamp <= $publish_timestamp_lower_bound )
        {
            say STDERR "Publish timestamp is lower than the lower bound for story $stories_id";
            next;
        }
        if ( $publish_timestamp >= $publish_timestamp_upper_bound )
        {
            say STDERR "Publish timestamp is bigger than the upper bound for story $stories_id";
            next;
        }

        # Round timestamp to the nearest day because that's what Bitly.pm does
        my $publish_datetime = gmt_datetime_from_timestamp( $publish_timestamp );
        $publish_datetime->set( hour => 0, minute => 0, second => 0 );
        $publish_timestamp = $publish_datetime->epoch;

        $min_publish_timestamp = $publish_timestamp
          if !defined $min_publish_timestamp or $min_publish_timestamp > $publish_timestamp;
        $max_publish_timestamp = $publish_timestamp
          if !defined $max_publish_timestamp or $max_publish_timestamp < $publish_timestamp;

        # Span across ~300 days
        my $start_timestamp = $publish_timestamp - ( 60 * 60 * 24 * 150 );
        my $end_timestamp   = $publish_timestamp + ( 60 * 60 * 24 * 150 );

        # How many seconds to sleep between rate limiting errors
        Readonly my $BITLY_RATE_LIMIT_SECONDS_TO_WAIT => 60 * 10;    # every 10 minutes

        # How many times to try on rate limiting errors
        Readonly my $BITLY_RATE_LIMIT_TRIES => 7;                    # try fetching 7 times in total (70 minutes)

        # What stats to fetch for each story
        Readonly my $BITLY_FETCH_CATEGORIES => 0;
        Readonly my $BITLY_FETCH_CLICKS     => 1;
        Readonly my $BITLY_FETCH_REFERRERS  => 0;
        Readonly my $BITLY_FETCH_SHARES     => 0;
        Readonly my $stats_to_fetch         => MediaWords::Util::Bitly::StatsToFetch->new(
            $BITLY_FETCH_CATEGORIES,                                 # "/v3/link/category"
            $BITLY_FETCH_CLICKS,                                     # "/v3/link/clicks"
            $BITLY_FETCH_REFERRERS,                                  # "/v3/link/referrers"
            $BITLY_FETCH_SHARES                                      # "/v3/link/shares"
        );

        my $story_stats = undef;
        my $retry       = 0;
        my $error_message;
        do
        {
            say STDERR "Fetching story stats for story $stories_id" . ( $retry ? " (retry $retry)" : '' ) . "...";
            eval {
                $story_stats = MediaWords::Util::Bitly::fetch_stats_for_url( $db, $story->{ url },
                    $start_timestamp, $end_timestamp, $stats_to_fetch );
            };
            $error_message = $@;

            if ( $error_message )
            {
                if ( MediaWords::Util::Bitly::error_is_rate_limit_exceeded( $error_message ) )
                {
                    say STDERR "Rate limit exceeded while collecting story stats for story $stories_id";
                    say STDERR "Sleeping for $BITLY_RATE_LIMIT_SECONDS_TO_WAIT before retrying";
                    sleep( $BITLY_RATE_LIMIT_SECONDS_TO_WAIT + 0 );
                }
                else
                {
                    die "Error while collecting story stats for story $stories_id: $error_message";
                }
            }
            ++$retry;
        } until ( $retry > $BITLY_RATE_LIMIT_TRIES + 0 or ( !$error_message ) );

        unless ( $story_stats )
        {
            # No point die()ing and continuing with other jobs (didn't recover after rate limiting)
            die "Stats for story ID $stories_id is undef (after $retry retries).";
        }
        unless ( ref( $story_stats ) eq ref( {} ) )
        {
            # No point die()ing and continuing with other jobs (something wrong with fetch_stats_for_story())
            die "Stats for story ID $stories_id is not a hashref.";
        }
        say STDERR "Done fetching story stats for story $stories_id.";

        foreach my $bitly_hash ( keys %{ $story_stats->{ data } } )
        {
            foreach my $bitly_clicks ( @{ $story_stats->{ data }->{ $bitly_hash }->{ clicks } } )
            {
                foreach my $link_click ( @{ $bitly_clicks->{ link_clicks } } )
                {
{'fraction_non_alphanumeric': 0.12411473087818697, 'fraction_numerical': 0.025672804532577902, 'mean_word_length': 1.9465310380803338, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2131391', 'n_tokens_mistral': 3841, 'n_tokens_neox': 3485, 'n_words': 988}