<commit_before>#!/usr/bin/perl

use strict;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Encode;
use MediaWords::DB;
use DBIx::Simple::MediaWords;
use MediaWords::Tagger;
use MediaWords::DBI::Downloads;
use List::Uniq ':all';

sub main
{

    my $dbs = DBIx::Simple::MediaWords->connect( MediaWords::DB::connect_info )
      || die DBIx::Simple::MediaWords->error;

    $dbs->query(
'update media set full_text_rss = true where media_id in (select media_id from media_rss_full_text_detection_data where  avg_similarity >= 0.97 and avg_rss_length > 1000) '
    );

}

<commit_msg>Don't change full_text_rss status on media that have previously been reviewed.<commit_after>#!/usr/bin/perl

use strict;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Encode;
use MediaWords::DB;
use DBIx::Simple::MediaWords;
use MediaWords::Tagger;
use MediaWords::DBI::Downloads;
use List::Uniq ':all';

sub main
{

    my $dbs = DBIx::Simple::MediaWords->connect( MediaWords::DB::connect_info )
      || die DBIx::Simple::MediaWords->error;

    $dbs->query(
'update media set full_text_rss = true where media_id in (select media_id from media_rss_full_text_detection_data where  avg_similarity >= 0.97 and avg_rss_length > 1000)  and full_text_rss is null'
    );

}


{'fraction_non_alphanumeric': 0.13261372397841173, 'fraction_numerical': 0.01079414032382421, 'mean_word_length': 4.643478260869565, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6737960', 'n_tokens_mistral': 504, 'n_tokens_neox': 457, 'n_words': 112}