<commit_before>use strict;
use warnings;

# basic sanity test of crawler functionality

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
    use lib $FindBin::Bin;
}

use Test::More;
use Test::Differences;
use Test::Deep;

use MediaWords::Crawler::Engine;
use MediaWords::DBI::DownloadTexts;
use MediaWords::DBI::MediaSets;
use MediaWords::DBI::Stories;
use MediaWords::Test::DB;
use MediaWords::Test::Data;
use MediaWords::Test::LocalServer;
use DBIx::Simple::MediaWords;
use MediaWords::StoryVectors;
use LWP::UserAgent;
use Perl6::Say;
use Data::Sorting qw( :basics :arrays :extras );

# add a test media source and feed to the database
sub add_test_feed
{
    my ( $db, $url_to_crawl ) = @_;

    my $test_medium = $db->query(
        "insert into media (name, url, moderated, feeds_added) values (?, ?, ?, ?) returning *",
        '_ Crawler Test',
        $url_to_crawl, 0, 0
    )->hash;
    my $feed = $db->query(
        "insert into feeds (media_id, name, url) values (?, ?, ?) returning *",
        $test_medium->{ media_id },
        '_ Crawler Test',
        "$url_to_crawl" . "gv/test.rss"
    )->hash;

    MediaWords::DBI::MediaSets::create_for_medium( $db, $test_medium );

    ok( $feed->{ feeds_id }, "test feed created" );

    return $feed;
}

# run the crawler for one minute, which should be enough time to gather all of the stories from the test feed
sub run_crawler
{

    my $crawler = MediaWords::Crawler::Engine->new();

    $crawler->processes( 1 );
    $crawler->throttle( 1 );
    $crawler->sleep_interval( 1 );
    $crawler->timeout( 60 );
    $crawler->pending_check_interval( 1 );

    $| = 1;

    print "running crawler for one minute ...\n";
    $crawler->crawl();

    print "crawler exiting ...\n";
}

sub extract_downloads
{
    my ( $db ) = @_;

    print "extracting downloads ...\n";

    my $downloads = $db->query( "select * from downloads" )->hashes;

    for my $download ( @{ $downloads } )
    {
        MediaWords::DBI::Downloads::process_download_for_extractor( $db, $download, 1 );
    }
}

# run extractor, tagger, and vector on all stories
sub process_stories
{
    my ( $db ) = @_;

    print "processing stories ...\n";

    my $stories = $db->query( "select * from stories" )->hashes;

    for my $story ( @{ $stories } )
    {
        print "adding default tags ...\n";
        MediaWords::DBI::Stories::add_default_tags( $db, $story );
    }

    print "processing stories done.\n";
}

# get stories from database, including content, text, tags, sentences, sentence_words, and story_sentence_words
sub get_expanded_stories
{
    my ( $db, $feed ) = @_;

    my $stories = $db->query(
        "select s.* from stories s, feeds_stories_map fsm " . "  where s.stories_id = fsm.stories_id and fsm.feeds_id = ?",
        $feed->{ feeds_id } )->hashes;

    for my $story ( @{ $stories } )
    {
        $story->{ content } = ${ MediaWords::DBI::Stories::fetch_content( $db, $story ) };
        $story->{ extracted_text } = MediaWords::DBI::Stories::get_text( $db, $story );
        $story->{ tags } = MediaWords::DBI::Stories::get_db_module_tags( $db, $story, 'NYTTopics' );

        $story->{ story_sentence_words } =
          $db->query( "select * from story_sentence_words where stories_id = ?", $story->{ stories_id } )->hashes;
    }

    return $stories;
}

# test various results of the crawler
sub test_stories
{
    my ( $db, $feed ) = @_;

    my $stories = get_expanded_stories( $db, $feed );

    is( @{ $stories }, 15, "story count" );

    my $test_stories = MediaWords::Test::Data::fetch_test_data( 'crawler_stories' );

    my $test_story_hash;
    map { $test_story_hash->{ $_->{ title } } = $_ } @{ $test_stories };

    for my $story ( @{ $stories } )
    {
        my $test_story = $test_story_hash->{ $story->{ title } };
        if ( ok( $test_story, "story match: " . $story->{ title } ) )
        {
            for my $field ( qw(publish_date description guid extracted_text) )
            {
                oldstyle_diff;

              TODO:
                {
                    my $fake_var;    #silence warnings
                     #eq_or_diff( $story->{ $field }, encode_utf8($test_story->{ $field }), "story $field match" , {context => 0});
                    is( $story->{ $field }, $test_story->{ $field }, "story $field match" );
                }
            }

            eq_or_diff( $story->{ content }, $test_story->{ content }, "story content matches" );

            is( scalar( @{ $story->{ tags } } ), scalar( @{ $test_story->{ tags } } ), "story tags count" );
          TODO:
            {
                my $fake_var;    #silence warnings

                my $test_story_sentence_words_count = scalar( @{ $test_story->{ story_sentence_words } } );
                my $story_sentence_words_count      = scalar( @{ $story->{ story_sentence_words } } );

                is( $story_sentence_words_count, $test_story_sentence_words_count, "story words count" );
            }
        }

        delete( $test_story_hash->{ $story->{ title } } );
    }

}

sub generate_aggregate_words
{
    my ( $db, $feed ) = @_;

    my ( $start_date ) = $db->query( "select date_trunc( 'day', min(publish_date) ) from stories" )->flat;
    my ( $end_date )   = $db->query( "select date_trunc( 'day', max(publish_date) ) from stories" )->flat;

    my $dashboard =
      $db->create( 'dashboards', { name => 'test_dashboard', start_date => $start_date, end_date => $end_date } );

    my $dashboard_topic = $db->create(
        'dashboard_topics',
        {
            dashboards_id => $dashboard->{ dashboards_id },
            name          => 'obama_topic',
            query         => 'obama',
            start_date    => $start_date,
            end_date      => $end_date
        }
    );

    MediaWords::StoryVectors::update_aggregate_words( $db, $start_date );
}

sub _process_top_500_weekly_words_for_testing
{
    my ( $hashes ) = @_;

    foreach my $hash ( @{ $hashes } )
    {
        delete( $hash->{ top_500_weekly_words_id } );
    }<commit_msg>Only aggregate words for the period in which have test data.<commit_after>use strict;
use warnings;

# basic sanity test of crawler functionality

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
    use lib $FindBin::Bin;
}

use Test::More;
use Test::Differences;
use Test::Deep;

use MediaWords::Crawler::Engine;
use MediaWords::DBI::DownloadTexts;
use MediaWords::DBI::MediaSets;
use MediaWords::DBI::Stories;
use MediaWords::Test::DB;
use MediaWords::Test::Data;
use MediaWords::Test::LocalServer;
use DBIx::Simple::MediaWords;
use MediaWords::StoryVectors;
use LWP::UserAgent;
use Perl6::Say;
use Data::Sorting qw( :basics :arrays :extras );

# add a test media source and feed to the database
sub add_test_feed
{
    my ( $db, $url_to_crawl ) = @_;

    my $test_medium = $db->query(
        "insert into media (name, url, moderated, feeds_added) values (?, ?, ?, ?) returning *",
        '_ Crawler Test',
        $url_to_crawl, 0, 0
    )->hash;
    my $feed = $db->query(
        "insert into feeds (media_id, name, url) values (?, ?, ?) returning *",
        $test_medium->{ media_id },
        '_ Crawler Test',
        "$url_to_crawl" . "gv/test.rss"
    )->hash;

    MediaWords::DBI::MediaSets::create_for_medium( $db, $test_medium );

    ok( $feed->{ feeds_id }, "test feed created" );

    return $feed;
}

# run the crawler for one minute, which should be enough time to gather all of the stories from the test feed
sub run_crawler
{

    my $crawler = MediaWords::Crawler::Engine->new();

    $crawler->processes( 1 );
    $crawler->throttle( 1 );
    $crawler->sleep_interval( 1 );
    $crawler->timeout( 60 );
    $crawler->pending_check_interval( 1 );

    $| = 1;

    print "running crawler for one minute ...\n";
    $crawler->crawl();

    print "crawler exiting ...\n";
}

sub extract_downloads
{
    my ( $db ) = @_;

    print "extracting downloads ...\n";

    my $downloads = $db->query( "select * from downloads" )->hashes;

    for my $download ( @{ $downloads } )
    {
        MediaWords::DBI::Downloads::process_download_for_extractor( $db, $download, 1 );
    }
}

# run extractor, tagger, and vector on all stories
sub process_stories
{
    my ( $db ) = @_;

    print "processing stories ...\n";

    my $stories = $db->query( "select * from stories" )->hashes;

    for my $story ( @{ $stories } )
    {
        print "adding default tags ...\n";
        MediaWords::DBI::Stories::add_default_tags( $db, $story );
    }

    print "processing stories done.\n";
}

# get stories from database, including content, text, tags, sentences, sentence_words, and story_sentence_words
sub get_expanded_stories
{
    my ( $db, $feed ) = @_;

    my $stories = $db->query(
        "select s.* from stories s, feeds_stories_map fsm " . "  where s.stories_id = fsm.stories_id and fsm.feeds_id = ?",
        $feed->{ feeds_id } )->hashes;

    for my $story ( @{ $stories } )
    {
        $story->{ content } = ${ MediaWords::DBI::Stories::fetch_content( $db, $story ) };
        $story->{ extracted_text } = MediaWords::DBI::Stories::get_text( $db, $story );
        $story->{ tags } = MediaWords::DBI::Stories::get_db_module_tags( $db, $story, 'NYTTopics' );

        $story->{ story_sentence_words } =
          $db->query( "select * from story_sentence_words where stories_id = ?", $story->{ stories_id } )->hashes;
    }

    return $stories;
}

# test various results of the crawler
sub test_stories
{
    my ( $db, $feed ) = @_;

    my $stories = get_expanded_stories( $db, $feed );

    is( @{ $stories }, 15, "story count" );

    my $test_stories = MediaWords::Test::Data::fetch_test_data( 'crawler_stories' );

    my $test_story_hash;
    map { $test_story_hash->{ $_->{ title } } = $_ } @{ $test_stories };

    for my $story ( @{ $stories } )
    {
        my $test_story = $test_story_hash->{ $story->{ title } };
        if ( ok( $test_story, "story match: " . $story->{ title } ) )
        {
            for my $field ( qw(publish_date description guid extracted_text) )
            {
                oldstyle_diff;

              TODO:
                {
                    my $fake_var;    #silence warnings
                     #eq_or_diff( $story->{ $field }, encode_utf8($test_story->{ $field }), "story $field match" , {context => 0});
                    is( $story->{ $field }, $test_story->{ $field }, "story $field match" );
                }
            }

            eq_or_diff( $story->{ content }, $test_story->{ content }, "story content matches" );

            is( scalar( @{ $story->{ tags } } ), scalar( @{ $test_story->{ tags } } ), "story tags count" );
          TODO:
            {
                my $fake_var;    #silence warnings

                my $test_story_sentence_words_count = scalar( @{ $test_story->{ story_sentence_words } } );
                my $story_sentence_words_count      = scalar( @{ $story->{ story_sentence_words } } );

                is( $story_sentence_words_count, $test_story_sentence_words_count, "story words count" );
            }
        }

        delete( $test_story_hash->{ $story->{ title } } );
    }

}

sub generate_aggregate_words
{
    my ( $db, $feed ) = @_;

    my ( $start_date ) = $db->query( "select date_trunc( 'day', min(publish_date) ) from stories" )->flat;
    #( $start_date ) = $db->query( "select date_trunc( 'day', min(publish_date)  - interval '1 week' ) from stories" )->flat;
    my ( $end_date )   = $db->query( "select date_trunc( 'day', max(publish_date) ) from stories" )->flat;

    #( $end_date )   = $db->query( "select date_trunc( 'day', max(publish_date) + interval '1 month' ) from stories" )->flat;

    my $dashboard =
      $db->create( 'dashboards', { name => 'test_dashboard', start_date => $start_date, end_date => $end_date } );

    my $dashboard_topic = $db->create(
        'dashboard_topics',
        {
            dashboards_id => $dashboard->{ dashboards_id },
            name          => 'obama_topic',
            query         => 'obama',
            start_date    => $start_date,
            end_date      => $end_date
        }
    );

    MediaWords::StoryVectors::update_aggregate_words( $db, $start_date, $end_date );
}

sub _process_top_500_weekly_words_for_testing
{
    my ( $hashes ) = @_;

    foreach my $hash ( @{ $hashes } )
    {
        delete( $hash->{ top_500_weekly_words_id } );
    }
{'fraction_non_alphanumeric': 0.1552625231165072, 'fraction_numerical': 0.0035378306665594596, 'mean_word_length': 2.62201514269074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5748304', 'n_tokens_mistral': 4375, 'n_tokens_neox': 4158, 'n_words': 993}