<commit_before>package MediaWords::Util::Facebook;

#
# Facebook API helper
#

use strict;
use warnings;

use MediaWords::Util::JSON;
use MediaWords::Util::URL;
use MediaWords::Util::Web;

sub _get_single_url_share_comment_counts
{
    my ( $ua, $url ) = @_;

    # this is mostly to be able to generate an error for testing
    die( "invalid url: '$url'" ) if ( $url !~ /^http/i );

    my $api_uri = URI->new( 'https://graph.facebook.com/' );
    $api_uri->query_form( 'id' => $url );

    my $response = $ua->get( $api_uri->as_string );

    if ( !$response->is_success )
    {
        die( "error fetching for url '$url'" );
    }
    my $decoded_content = $response->decoded_content;

    my $data = MediaWords::Util::JSON::decode_json( $decoded_content );

    my $share_count   = $data->{ shares }   || 0;
    my $comment_count = $data->{ comments } || 0;

    return ( $share_count, $comment_count );
}

# use https://graph.facebook.com/?id= to get number of shares for the given url
# https://graph.facebook.com/?id=http://www.google.com/
sub get_url_share_comment_counts
{
    my ( $db, $url ) = @_;

    my $all_urls = [ MediaWords::Util::URL::all_url_variants( $db, $url ) ];

    my $ua = MediaWords::Util::Web::UserAgentDetermined();
    $ua->timing( '1,3,15,60,300,600' );

    my $url_share_counts   = {};
    my $url_comment_counts = {};
    for my $u ( @{ $all_urls } )
    {
        my ( $share_count, $comment_count ) = _get_single_url_share_comment_counts( $ua, $u );
        $url_share_counts->{ $share_count }     = $u;
        $url_comment_counts->{ $comment_count } = $u;
    }

    my $share_count_sum   = List::Util::sum( keys( %{ $url_share_counts } ) );
    my $comment_count_sum = List::Util::sum( keys( %{ $url_comment_counts } ) );

    return ( $share_count_sum, $comment_count_sum );
}

sub get_and_store_share_comment_counts
{
    my ( $db, $story ) = @_;

    my ( $share_count, $comment_count );
    eval { ( $share_count, $comment_count ) = get_url_share_comment_counts( $db, $story->{ url } ); };
    my $error = $@ ? $@ : undef;

    $share_count   ||= 0;
    $comment_count ||= 0;

    $db->query(
        <<END,<commit_msg>Print the URL variant for which we've fetched stats to get to know how many requests were made<commit_after>package MediaWords::Util::Facebook;

#
# Facebook API helper
#

use strict;
use warnings;

use MediaWords::Util::JSON;
use MediaWords::Util::URL;
use MediaWords::Util::Web;

sub _get_single_url_share_comment_counts
{
    my ( $ua, $url ) = @_;

    # this is mostly to be able to generate an error for testing
    die( "invalid url: '$url'" ) if ( $url !~ /^http/i );

    my $api_uri = URI->new( 'https://graph.facebook.com/' );
    $api_uri->query_form( 'id' => $url );

    my $response = $ua->get( $api_uri->as_string );

    if ( !$response->is_success )
    {
        die( "error fetching for url '$url'" );
    }
    my $decoded_content = $response->decoded_content;

    my $data = MediaWords::Util::JSON::decode_json( $decoded_content );

    my $share_count   = $data->{ shares }   || 0;
    my $comment_count = $data->{ comments } || 0;

    return ( $share_count, $comment_count );
}

# use https://graph.facebook.com/?id= to get number of shares for the given url
# https://graph.facebook.com/?id=http://www.google.com/
sub get_url_share_comment_counts
{
    my ( $db, $url ) = @_;

    my $all_urls = [ MediaWords::Util::URL::all_url_variants( $db, $url ) ];

    my $ua = MediaWords::Util::Web::UserAgentDetermined();
    $ua->timing( '1,3,15,60,300,600' );

    my $url_share_counts   = {};
    my $url_comment_counts = {};
    for my $u ( @{ $all_urls } )
    {
        my ( $share_count, $comment_count ) = _get_single_url_share_comment_counts( $ua, $u );

        say STDERR "* Share count: $share_count, comment count: $comment_count, URL variant: $u";

        $url_share_counts->{ $share_count }     = $u;
        $url_comment_counts->{ $comment_count } = $u;
    }

    my $share_count_sum   = List::Util::sum( keys( %{ $url_share_counts } ) );
    my $comment_count_sum = List::Util::sum( keys( %{ $url_comment_counts } ) );

    return ( $share_count_sum, $comment_count_sum );
}

sub get_and_store_share_comment_counts
{
    my ( $db, $story ) = @_;

    my ( $share_count, $comment_count );
    eval { ( $share_count, $comment_count ) = get_url_share_comment_counts( $db, $story->{ url } ); };
    my $error = $@ ? $@ : undef;

    $share_count   ||= 0;
    $comment_count ||= 0;

    $db->query(
        <<END,
{'fraction_non_alphanumeric': 0.1678120150742629, 'fraction_numerical': 0.007093770782531589, 'mean_word_length': 3.3053435114503817, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4645756', 'n_tokens_mistral': 1704, 'n_tokens_neox': 1622, 'n_words': 351}