<commit_before>package Diversion::App::Command::refresh;
use v5.18;
use Diversion::App -command;

use IO::All;
use List::Util qw(shuffle);

use Diversion::FeedArchiver;
use Parallel::ForkManager;

sub execute {
    my ($self, $opt, $args) = @_;

    my $feed_url = $args->[0] or die "Missing URL in arg";

    my @feeds;
    if (-f $feed_url) {
        @feeds = grep { s/\s//g; $_ } io($feed_url)->chomp->getlines;
    } else {
        push @feeds, $feed_url;
    }

    my $forkman = Parallel::ForkManager->new(4);
    for (shuffle @feeds) {
        $forkman->start and next;
        my $feed_archiver = Diversion::FeedArchiver->new;
        say "[pid=$$] Processing $_";
        eval {
            $feed_archiver->fetch_then_archive( $_ );
            1;
        } or do {
            say STDERR $@;
        };
        $forkman->finish;
    }
    $forkman->wait_all_children;
}

1;
<commit_msg>process only http:// and https:// links.<commit_after>package Diversion::App::Command::refresh;
use v5.18;
use Diversion::App -command;

use IO::All;
use List::Util qw(shuffle);

use Diversion::FeedArchiver;
use Parallel::ForkManager;

sub execute {
    my ($self, $opt, $args) = @_;

    my $feed_url = $args->[0] or die "Missing URL in arg";

    my @feeds;
    if (-f $feed_url) {
        @feeds = grep { s/\s//g; $_ } io($feed_url)->chomp->getlines;
    } else {
        push @feeds, $feed_url;
    }

    my $forkman = Parallel::ForkManager->new(4);
    for (shuffle grep { /^https?:/ } @feeds) {
        $forkman->start and next;
        my $feed_archiver = Diversion::FeedArchiver->new;
        say "[pid=$$] Processing $_";
        eval {
            $feed_archiver->fetch_then_archive( $_ );
            1;
        } or do {
            say STDERR $@;
        };
        $forkman->finish;
    }
    $forkman->wait_all_children;
}

1;

{'fraction_non_alphanumeric': 0.1631321370309951, 'fraction_numerical': 0.0076128330614464385, 'mean_word_length': 2.363802559414991, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4506846', 'n_tokens_mistral': 683, 'n_tokens_neox': 659, 'n_words': 145}