<commit_before>#!/usr/bin/env perl

use strict;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2015";
use MediaWords::CommonLibs;

use Getopt::Long;
use HTML::Strip;
use DBIx::Simple::MediaWords;
use MediaWords::DB;

use MediaWords::DBI::Downloads;
use MediaWords::DBI::DownloadTexts;
use Readonly;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use XML::LibXML;
use Data::Dumper;

use Digest::SHA qw(sha1 sha1_hex sha1_base64);

#use XML::LibXML::CDATASection;
use Encode;
use MIME::Base64;

#use XML::LibXML::Enhanced;

my $_re_generate_cache = 0;

sub reextract_downloads
{

    my $downloads = shift;

    my @downloads = @{ $downloads };

    INFO "Starting reextract_downloads";

    @downloads = sort { $a->{ downloads_id } <=> $b->{ downloads_id } } @downloads;

    my $download_results = [];

    my $dbs = MediaWords::DB::connect_to_db;

    for my $download ( @downloads )
    {
        die "Non-content type download: $download->{ downloads_id } $download->{ type } "
          unless $download->{ type } eq 'content';

        INFO "Processing download $download->{downloads_id}";

        MediaWords::DBI::Downloads::process_download_for_extractor( $dbs, $download );
    }
}

# do a test run of the text extractor
sub main
{

    my $dbs = MediaWords::DB::connect_to_db;

    my $file;
    my @download_ids;

    GetOptions(
        'file|f=s'                  => \$file,
        'downloads|d=s'             => \@download_ids,
        'regenerate_database_cache' => \$_re_generate_cache,
    ) or die;

    unless ( $file || ( @download_ids ) )
    {
        die "no options given ";
    }

    my $downloads;

    DEBUG join( ', ', @download_ids );

    if ( @download_ids )
    {
        $downloads = $dbs->query( "SELECT * from downloads where downloads_id in (??)", @download_ids )->hashes;
    }
    elsif ( $file )
    {
        open( DOWNLOAD_ID_FILE, $file ) || die( "Could not open file: $file" );
        @download_ids = <DOWNLOAD_ID_FILE>;
        $downloads = $dbs->query( "SELECT * from downloads where downloads_id in (??)", @download_ids )->hashes;
    }
    else
    {
        die "must specify file or downloads id";
    }

    DEBUG Dumper( $downloads );

    die 'no downloads found ' unless scalar( @$downloads );

    DEBUG scalar( @$downloads ) . ' downloads';

    reextract_downloads( $downloads );

    INFO "completed extraction";
}
<commit_msg>update mediawords_reextract_downloads to queue downloads for extraction and to accept --query and --unextracted options<commit_after>#!/usr/bin/env perl

use strict;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2015";
use MediaWords::CommonLibs;

use Getopt::Long;
use HTML::Strip;
use DBIx::Simple::MediaWords;
use MediaWords::DB;

use MediaWords::DBI::Downloads;
use MediaWords::DBI::DownloadTexts;
use Readonly;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use XML::LibXML;
use Data::Dumper;

use Digest::SHA qw(sha1 sha1_hex sha1_base64);

#use XML::LibXML::CDATASection;
use Encode;
use MIME::Base64;

sub queue_extraction($$)
{
    my ( $db, $downloads_id ) = @_;

    my $args = {
        downloads_id            => $downloads_id,
        skip_bitly_processing   => 0,
        skip_corenlp_annotation => 0,
        use_cache               => 1
    };

    my $priority = $MediaCloud::JobManager::Job::MJM_JOB_PRIORITY_LOW;
    MediaWords::Job::ExtractAndVector->add_to_queue( $args, $priority );
}

# do a test run of the text extractor
sub main
{
    my ( $file, $download_ids, $unextracted, $query );

    $download_ids = [];

    GetOptions(
        'file|f=s'      => \$file,
        'downloads|d=s' => $download_ids,
        'query|q=s'     => \$query,
        'unextracted!'  => \$unextracted
    ) or die;

    unless ( $file || $download_ids || $query || $unextracted )
    {
        die( "must specify one of either --file, --download_ids, --query, or --unextracted" );
    }

    my $downloads_ids;

    my $db = MediaWords::DB::connect_to_db;

    if ( scalar( @{ $download_ids } ) )
    {
        my $ids = $db->get_temporary_ids_table( $download_ids );
        $downloads_ids =
          $db->query( "SELECT downloads_id from downloads where downloads_id in ( select id from $ids )" )->flat;
    }
    elsif ( $file )
    {
        open( DOWNLOAD_ID_FILE, $file ) || die( "Could not open file: $file" );
        $download_ids = [ map { chomp( $_ ); $_ } <DOWNLOAD_ID_FILE> ];
        my $ids = $db->get_temporary_ids_table( $download_ids );
        $downloads_ids =
          $db->query( "SELECT downloads_id from downloads where downloads_id in ( select id from $ids )" )->flat;
    }
    elsif ( $query )
    {
        $downloads_ids = $db->query( $query )->flat;
    }
    elsif ( $unextracted )
    {
        $downloads_ids = $db->query(
            "select downloads_id from downloads where state = 'success' and type = 'content' and extracted = false" )->flat;
    }
    else
    {
        die "must specify file or downloads id";
    }

    die 'no downloads found' unless scalar( @{ $downloads_ids } );

    DEBUG "queueing " . scalar( @{ $downloads_ids } ) . ' downloads for extraction ...';

    map { queue_extraction( $db, $_ ) } @{ $downloads_ids };
}

{'fraction_non_alphanumeric': 0.14164145234493192, 'fraction_numerical': 0.004916792738275341, 'mean_word_length': 2.894698085419735, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 85, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5615976', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1778, 'n_words': 446}