<commit_before>#!/usr/bin/perl

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use DBIx::Simple::MediaWords;
use MediaWords::DB;

# create a media source from a feed url
sub main
{
    my $db = DBIx::Simple::MediaWords->connect( MediaWords::DB::connect_info );

    my $res = $db->query(
"select * from media natural join (select media_id, max(similarity) as max_similarity, avg(similarity) as avg_similarity, min(similarity) as min_similarity,  avg(length(extracted_text)) as avg_extracted_length, avg(length(html_strip(title || description))) as avg_rss_length, avg(length(html_strip(description))) as avg_rss_discription, count(*) from (select *, similarity(extracted_text, html_strip(title || description) ) from stories, (select stories_id,        array_to_string(array_agg(download_text), ' ') as extracted_text        from (select * from downloads natural join stories natural join download_texts where publish_date > now() - interval ' 1 week'  order by downloads_id)            as downloads group by stories_id) as story_extracted_texts where stories.stories_id = story_extracted_texts.stories_id and publish_date > now() - interval '1 week' ) as media_extraction_text_similarity group by media_id order by media_id ) as foo"
    );

    print "Output:\n";
    print $res->flat;
    print "\n";
}

main();<commit_msg>Update indentation<commit_after>#!/usr/bin/perl

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use DBIx::Simple::MediaWords;
use MediaWords::DB;

# create a media source from a feed url
sub main
{
    my $db = DBIx::Simple::MediaWords->connect( MediaWords::DB::connect_info );

    my $res =
      $db->query( "select * from media natural join  " .
          "(select media_id, max(similarity) as max_similarity, avg(similarity) as avg_similarity, " .
          "   min(similarity) as min_similarity,  avg(length(extracted_text)) as avg_extracted_length," .
          "   avg(length(html_strip(title || description))) as avg_rss_length,  " .
          "   avg(length(html_strip(description))) as avg_rss_discription, count(*) from "
          "     (select *, similarity(extracted_text, html_strip(title || description) ) " .
          "       from stories, (select stories_id,  array_to_string(array_agg(download_text), ' ') as extracted_text   " .
"from (select * from downloads natural join stories natural join download_texts where publish_date > now() - interval ' 1 week'  order by downloads_id) "
          . "            as downloads group by stories_id) as story_extracted_texts where stories.stories_id = story_extracted_texts.stories_id and publish_date > now() - interval '1 week' ) as media_extraction_text_similarity group by media_id order by media_id ) as foo"
      );

    print "Output:\n";
    print $res->flat;
    print "\n";
}

main();
{'fraction_non_alphanumeric': 0.11381830838844413, 'fraction_numerical': 0.0013922728854855553, 'mean_word_length': 4.024475524475524, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6610071', 'n_tokens_mistral': 943, 'n_tokens_neox': 896, 'n_words': 246}