<commit_before>#!/usr/bin/env perl

# query a random set of feeds and generate a list of feedly stories that would be added to media cloud if we were to run
# a feedly import for that feed

# usage: $0 < num_feeds >

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use Mediawords::CommonLibs;

use Data::Dumper;
use Encode;

use MediaWords::DB;
use MediaWords::ImportStories::Feedly;
use MediaWords::Util::CSV;

sub main
{
    my ( $num_feeds ) = @ARGV;

    my $db = MediaWords::DB::connect_to_db;

    my $feeds = $db->query( <<SQL, $num_feeds )->hashes;
select * from feeds where feed_status = 'active' order by random() limit ( ? * 10 )
SQL

    my $dates = [];

    my $validate_stories = [];

    my $feed_data = [];

    my $scraped_feeds = [];
    for my $feed ( @{ $feeds } )
    {
        say STDERR "feed: " . scalar( @{ $scraped_feeds } ) . " / $num_feeds";<commit_msg>import typo<commit_after>#!/usr/bin/env perl

# query a random set of feeds and generate a list of feedly stories that would be added to media cloud if we were to run
# a feedly import for that feed

# usage: $0 < num_feeds >

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use Data::Dumper;
use Encode;

use MediaWords::DB;
use MediaWords::ImportStories::Feedly;
use MediaWords::Util::CSV;

sub main
{
    my ( $num_feeds ) = @ARGV;

    my $db = MediaWords::DB::connect_to_db;

    my $feeds = $db->query( <<SQL, $num_feeds )->hashes;
select * from feeds where feed_status = 'active' order by random() limit ( ? * 10 )
SQL

    my $dates = [];

    my $validate_stories = [];

    my $feed_data = [];

    my $scraped_feeds = [];
    for my $feed ( @{ $feeds } )
    {
        say STDERR "feed: " . scalar( @{ $scraped_feeds } ) . " / $num_feeds";
{'fraction_non_alphanumeric': 0.14451476793248946, 'fraction_numerical': 0.007383966244725738, 'mean_word_length': 3.2533632286995515, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '9857', 'n_tokens_mistral': 724, 'n_tokens_neox': 684, 'n_words': 210}