<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use MediaWords::DB;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use MediaWords::DBI::DownloadTexts;
use MediaWords::DBI::Stories;
use MediaWords::StoryVectors;
use MediaWords::Util::MC_Fork;

# extract, story, and tag downloaded text a slice of downloads.
# downloads are extracted by a total of num_total_jobs processings
# a total of num_total_processes, with a unique 1-indexed job_number
# for each job
sub extract_text
{
    my $db = MediaWords::DB::connect_to_db;

    $db->dbh->{ AutoCommit } = 0;

    my $last_story_sentences_id_processed =
      ( $db->query( " SELECT value from database_variables where name = 'LAST_STORY_SENTENCES_ID_PROCESSED' " )->flat() )
      [ 0 ];

    die unless defined( $last_story_sentences_id_processed );

    my $stop_story_sentences_id = ( $db->query( " SELECT max(story_sentences_id) from story_sentences" )->flat() )[ 0 ];

    $db->query(
" INSERT INTO processed_stories ( stories_id ) select distinct( stories_id ) from story_sentences where story_sentences_id >? and story_sentences_id <= ? order by stories_id",
        $last_story_sentences_id_processed, $stop_story_sentences_id
    );

    $db->query( "UPDATE database_variables SET value = ? where name =  'LAST_STORY_SENTENCES_ID_PROCESSED' ",
        $stop_story_sentences_id );

    $db->commit;

}

# fork of $num_processes
sub main
{
    extract_text();
}

# use Test::LeakTrace;
# leaktrace { main(); };

main();<commit_msg>Add a daemon mode and include additional debug output.<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use MediaWords::DB;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use MediaWords::DBI::DownloadTexts;
use MediaWords::DBI::Stories;
use MediaWords::StoryVectors;
use MediaWords::Util::MC_Fork;

sub update_processed_stories
{
    my $db = MediaWords::DB::connect_to_db;

    $db->dbh->{ AutoCommit } = 0;

    my $last_story_sentences_id_processed =
      ( $db->query( " SELECT value from database_variables where name = 'LAST_STORY_SENTENCES_ID_PROCESSED' " )->flat() )
      [ 0 ];

    die unless defined( $last_story_sentences_id_processed );

    my $stop_story_sentences_id = ( $db->query( " SELECT max(story_sentences_id) from story_sentences" )->flat() )[ 0 ];

    if ( $last_story_sentences_id_processed == $stop_story_sentences_id )
    {
        say STDERR "processed_stories is up to date. Stop story_sentences_id = $stop_story_sentences_id";
        return;
    }

    say STDERR "Updating processed stories from $last_story_sentences_id_processed to $stop_story_sentences_id";

    $db->query(
" INSERT INTO processed_stories ( stories_id ) select distinct( stories_id ) from story_sentences where story_sentences_id >? and story_sentences_id <= ? order by stories_id",
        $last_story_sentences_id_processed, $stop_story_sentences_id
    );

    $db->query( "UPDATE database_variables SET value = ? where name =  'LAST_STORY_SENTENCES_ID_PROCESSED' ",
        $stop_story_sentences_id );

    $db->commit;

}

sub main
{
    if ( ( scalar( @ARGV ) >= 1 ) && $ARGV[ 0 ] eq '-d' )
    {
        while ( 1 )
        {
            update_processed_stories();
            say STDERR "Sleeping...";
            sleep( 30 );
        }
    }
    else
    {
        update_processed_stories();

    }
}

main();
{'fraction_non_alphanumeric': 0.11659965537047674, 'fraction_numerical': 0.005743825387708214, 'mean_word_length': 3.8174273858921164, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7382590', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1179, 'n_words': 271}