<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use MediaWords::DB;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use MediaWords::DBI::DownloadTexts;
use MediaWords::DBI::Stories;
use MediaWords::StoryVectors;
use MediaWords::Util::MC_Fork;
use MediaWords::Util::XML;

use XML::LibXML;
use MIME::Base64;
use Encode;
use List::Util qw (min);
use Parallel::ForkManager;

sub set_relative_path_downloads
{
    my ( $start_downloads_id, $end_downloads_id, $batch_number, $max_downloads_id ) = @_;

    my $db = MediaWords::DB::connect_to_db;

    $db->dbh->{ AutoCommit } = 0;

    my $batch_information = '';

    if ( defined( $batch_number ) )
    {
        $batch_information = "Batch $batch_number";
    }

    my $max_downloads_id_message = '';
    if ( defined( $max_downloads_id ) )
    {
        $max_downloads_id_message = " max overall downloads_id $max_downloads_id";
    }

    say STDERR "$batch_information downloads_id $start_downloads_id -- $end_downloads_id  $max_downloads_id_message";

    my $download = $db->query(
"UPDATE downloads set relative_file_path = get_relative_file_path( path ) where downloads_id >= ?  and downloads_id <= ? ",
        $start_downloads_id, $end_downloads_id
    );

    return;
}

sub set_relative_path_all_downloads
{

    my $db = MediaWords::DB::connect_to_db;

    my ( $max_downloads_id ) =
      $db->query( " SELECT max( downloads_id) from downloads where type = 'feed' and state = 'success' " )->flat();

    my ( $min_downloads_id ) = $db->query( " SELECT min( downloads_id) from downloads " )->flat();

    #Make sure the file start and end ranges are multiples of 1000
    my $start_downloads_id = int( $min_downloads_id / 1000 ) * 1000;

    Readonly my $download_batch_size => 1000;

    my $batch_number = 0;

    my $pm = new Parallel::ForkManager( 15 );

    my $empty_download_check_frequency = 10;

    while ( $start_downloads_id <= $max_downloads_id )
    {
        unless ( $pm->start )
        {

            set_relative_path_downloads( $start_downloads_id, $start_downloads_id + $download_batch_size,
                $batch_number, $max_downloads_id );
            $pm->finish;
        }

        $start_downloads_id += $download_batch_size;
        $batch_number++;

        ## Skip over large ranges of empty downloads_id's
        if ( ( $batch_number % $empty_download_check_frequency ) == 0 )
        {
            ( $start_downloads_id ) =
              $db->query( " SELECT min( downloads_id) from downloads where downloads_id >= ? ", $start_downloads_id )
              ->flat();
            my $start_downloads_id = int( $start_downloads_id / 1000 ) * 1000;
        }

        #exit;
    }

    say "Waiting for children";

    $pm->wait_all_children;

}

# fork of $num_processes
sub main
{
    my ( $num_processes ) = @ARGV;

    binmode STDOUT, ":utf8";
    binmode STDERR, ":utf8";

    set_relative_path_all_downloads();
}

main();
<commit_msg>Update set relative path is only change downloads that have relative_file_path = 'tdb';<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use MediaWords::DB;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use MediaWords::DBI::DownloadTexts;
use MediaWords::DBI::Stories;
use MediaWords::StoryVectors;
use MediaWords::Util::MC_Fork;
use MediaWords::Util::XML;

use XML::LibXML;
use MIME::Base64;
use Encode;
use List::Util qw (min);
use Parallel::ForkManager;

sub set_relative_path_downloads
{
    my ( $start_downloads_id, $end_downloads_id, $batch_number, $max_downloads_id ) = @_;

    my $db = MediaWords::DB::connect_to_db;

    $db->dbh->{ AutoCommit } = 0;

    my $batch_information = '';

    if ( defined( $batch_number ) )
    {
        $batch_information = "Batch $batch_number";
    }

    my $max_downloads_id_message = '';
    if ( defined( $max_downloads_id ) )
    {
        $max_downloads_id_message = " max overall downloads_id $max_downloads_id";
    }

    say STDERR "$batch_information downloads_id $start_downloads_id -- $end_downloads_id  $max_downloads_id_message";

    my $download = $db->query(
"UPDATE downloads set relative_file_path = get_relative_file_path( path ) where relative_file_path = 'tbd' and downloads_id >= ?  and downloads_id <= ? ",
        $start_downloads_id, $end_downloads_id
    );

    return;
}

sub set_relative_path_all_downloads
{

    my $db = MediaWords::DB::connect_to_db;

    my ( $max_downloads_id ) =
      $db->query( " SELECT max( downloads_id) from downloads where state = 'success' " )->flat();

    my ( $min_downloads_id ) = $db->query( " SELECT min( downloads_id) from downloads " )->flat();

    #Make sure the file start and end ranges are multiples of 1000
    my $start_downloads_id = int( $min_downloads_id / 1000 ) * 1000;

    Readonly my $download_batch_size => 1000;

    my $batch_number = 0;

    my $pm = new Parallel::ForkManager( 15 );

    my $empty_download_check_frequency = 10;

    while ( $start_downloads_id <= $max_downloads_id )
    {
        unless ( $pm->start )
        {

            set_relative_path_downloads( $start_downloads_id, $start_downloads_id + $download_batch_size,
                $batch_number, $max_downloads_id );
            $pm->finish;
        }

        $start_downloads_id += $download_batch_size;
        $batch_number++;

        ## Skip over large ranges of empty downloads_id's
        # if ( ( $batch_number % $empty_download_check_frequency ) == 0 )
        # {
        #     ( $start_downloads_id ) =
        #       $db->query( " SELECT min( downloads_id) from downloads where downloads_id >= ? ", $start_downloads_id )
        #       ->flat();
        #     my $start_downloads_id = int( $start_downloads_id / 1000 ) * 1000;
        # }

        #exit;
    }

    say "Waiting for children";

    $pm->wait_all_children;

}

# fork of $num_processes
sub main
{
    my ( $num_processes ) = @ARGV;

    binmode STDOUT, ":utf8";
    binmode STDERR, ":utf8";

    set_relative_path_all_downloads();
}

main();

{'fraction_non_alphanumeric': 0.12183983037024955, 'fraction_numerical': 0.012722231283640516, 'mean_word_length': 3.1714285714285713, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 101, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '807701', 'n_tokens_mistral': 2245, 'n_tokens_neox': 2065, 'n_words': 455}