<commit_before>package MediaWords::CM::Mine;

# Mine through stories found for the given controversy and find all the links in each story.
# Find each link, try to find whether it matches any given story.  If it doesn't, create a
# new story.  Add that story's links to the queue if it matches the pattern for the
# controversy.  Write the resulting stories and links to controversy_stories and controversy_links.

use strict;
use warnings;

use Carp;
use Data::Dumper;
use DateTime;
use Encode;
use Getopt::Long;
use HTML::LinkExtractor;
use URI;
use URI::Split;
use URI::Escape;

use MediaWords::CM::GuessDate;
use MediaWords::CM::GuessDate::Result;
use MediaWords::DBI::Activities;
use MediaWords::DBI::Media;
use MediaWords::DBI::Stories;
use MediaWords::Solr;
use MediaWords::Util::Tags;
use MediaWords::Util::URL;
use MediaWords::Util::Web;
use MediaWords::Util::Bitly;
use MediaWords::GearmanFunction::Bitly::EnqueueControversyStories;

# number of times to run through the recursive link weight process
use constant LINK_WEIGHT_ITERATIONS => 3;

# tag that will be associate with all controversy_stories at the end of the script
use constant ALL_TAG => 'all';

# ignore links that match this pattern
my $_ignore_link_pattern =
  '(www.addtoany.com)|(novostimira.com)|(ads\.pheedo)|(www.dailykos.com\/user)|' .
  '(livejournal.com\/(tag|profile))|(sfbayview.com\/tag)|(absoluteastronomy.com)|' .
  '(\/share.*http)|(digg.com\/submit)|(facebook.com.*mediacontentsharebutton)|' .
  '(feeds.wordpress.com\/.*\/go)|(sharetodiaspora.github.io\/|iconosquare.com|unz.com)';

# cache of media by media id
my $_media_cache = {};

# cache for spidered:spidered tag
my $_spidered_tag;

# cache of media by sanitized url
my $_media_url_lookup;

# fetch each link and add a { redirect_url } field if the
# { url } field redirects to another url
sub add_redirect_links
{
    my ( $db, $links ) = @_;

    my $urls = [ map { URI->new( $_->{ url } )->as_string } @{ $links } ];

    my $responses = MediaWords::Util::Web::ParallelGet( $urls );

    my $link_lookup = {};
    map { $link_lookup->{ URI->new( $_->{ url } )->as_string } = $_ } @{ $links };

    for my $response ( @{ $responses } )
    {
        my $original_url = MediaWords::Util::Web->get_original_request( $response )->uri->as_string;<commit_msg>add answers.com to skip list<commit_after>package MediaWords::CM::Mine;

# Mine through stories found for the given controversy and find all the links in each story.
# Find each link, try to find whether it matches any given story.  If it doesn't, create a
# new story.  Add that story's links to the queue if it matches the pattern for the
# controversy.  Write the resulting stories and links to controversy_stories and controversy_links.

use strict;
use warnings;

use Carp;
use Data::Dumper;
use DateTime;
use Encode;
use Getopt::Long;
use HTML::LinkExtractor;
use URI;
use URI::Split;
use URI::Escape;

use MediaWords::CM::GuessDate;
use MediaWords::CM::GuessDate::Result;
use MediaWords::DBI::Activities;
use MediaWords::DBI::Media;
use MediaWords::DBI::Stories;
use MediaWords::Solr;
use MediaWords::Util::Tags;
use MediaWords::Util::URL;
use MediaWords::Util::Web;
use MediaWords::Util::Bitly;
use MediaWords::GearmanFunction::Bitly::EnqueueControversyStories;

# number of times to run through the recursive link weight process
use constant LINK_WEIGHT_ITERATIONS => 3;

# tag that will be associate with all controversy_stories at the end of the script
use constant ALL_TAG => 'all';

# ignore links that match this pattern
my $_ignore_link_pattern =
  '(www.addtoany.com)|(novostimira.com)|(ads\.pheedo)|(www.dailykos.com\/user)|' .
  '(livejournal.com\/(tag|profile))|(sfbayview.com\/tag)|(absoluteastronomy.com)|' .
  '(\/share.*http)|(digg.com\/submit)|(facebook.com.*mediacontentsharebutton)|' .
  '(feeds.wordpress.com\/.*\/go)|(sharetodiaspora.github.io\/|iconosquare.com)|' . 'unz.com|answers.com';

# cache of media by media id
my $_media_cache = {};

# cache for spidered:spidered tag
my $_spidered_tag;

# cache of media by sanitized url
my $_media_url_lookup;

# fetch each link and add a { redirect_url } field if the
# { url } field redirects to another url
sub add_redirect_links
{
    my ( $db, $links ) = @_;

    my $urls = [ map { URI->new( $_->{ url } )->as_string } @{ $links } ];

    my $responses = MediaWords::Util::Web::ParallelGet( $urls );

    my $link_lookup = {};
    map { $link_lookup->{ URI->new( $_->{ url } )->as_string } = $_ } @{ $links };

    for my $response ( @{ $responses } )
    {
        my $original_url = MediaWords::Util::Web->get_original_request( $response )->uri->as_string;
{'fraction_non_alphanumeric': 0.13916415338216287, 'fraction_numerical': 0.00043084877208099956, 'mean_word_length': 4.80375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1748387', 'n_tokens_mistral': 1591, 'n_tokens_neox': 1518, 'n_words': 448}