<commit_before>package Diversion::App::Command::buildmail;
# ABSTRACT: build email containing recent entries
use v5.18;
use Moo;
with 'Diversion::Service';
use Diversion::App -command;

use IO::All;
use JSON::PP;
use DateTime;
use DateTime::Format::RSS;
use List::UtilsBy qw( rev_nsort_by uniq_by );
use List::Util qw( max );
use Text::Xslate;
use Text::WrapI18N qw(wrap);

use File::ShareDir;
use Log::Any qw($log);

use Diversion::FeedArchiver;
use Diversion::FeedArchiveIterator;

sub opt_spec {
    return (
        ["ago=i", "Include entries created up to this second ago.", { default => 86400 }],
    )
}

sub is_blank {
    my $str = $_[0];
    return !defined($str) || $str =~ m/^\s*$/;
}

sub execute {
    my ($self, $opt, $args) = @_;

    my $JSON = JSON::PP->new->utf8;

    my $blob_store = $self->blob_store;
    my $iter = Diversion::FeedArchiveIterator->new;
    my $now = DateTime->now;
    my $fmt = DateTime::Format::RSS->new;
    my %seen;
    my $tmpl_data = {};
    while (my $row = $iter->next()) {
	next unless $row->{uri} && $row->{sha1_digest};
	my $blob = $blob_store->get($row->{sha1_digest});
	my $data = $JSON->decode($blob);
	for my $entry (@{$data->{entry}}) {
	    next if !$entry->{pubDate} || !$entry->{link} || is_blank($entry->{title}) || $seen{$entry->{link}}++;
	    
            $entry->{_pubDate_as_DateTime} = $fmt->parse_datetime($entry->{pubDate});
	    if ($entry->{_pubDate_as_DateTime}) {
		next if ($now - $entry->{_pubDate_as_DateTime})->in_units("seconds") > $opt->{ago};
	    }
	    
	    if ($entry->{description} && length($entry->{description}) > 140) {
		$entry->{description_short} = substr($entry->{description}, 0, 137) . "...";
	    }
	    $entry->{description} //= "";
	    $entry->{description_short} //= "";
	    
	    push @{ $tmpl_data->{entries} }, $entry;
	}
    }

    my @l = (localtime)[5,4,3,2,1];
    $l[0]+=1900;
    $l[1]+=1;
    my $ts = sprintf("%4d%02d%02d%02d%02d",@l);
    my $output_dir = Diversion::App->config->{output}{directory} || "/tmp";

    my ($text_body, $html_body);
    if ($text_body = build_text_mail($tmpl_data)) {
	io->catfile($output_dir, "diversion-mail-$ts.txt")->utf8->print($text_body);
    }
    <commit_msg>buildmail: add "--limit" to constraint the entries in one email.<commit_after>package Diversion::App::Command::buildmail;
# ABSTRACT: build email containing recent entries
use v5.18;
use Moo;
with 'Diversion::Service';
use Diversion::App -command;

use IO::All;
use JSON::PP;
use DateTime;
use DateTime::Format::RSS;
use List::UtilsBy qw( rev_nsort_by uniq_by );
use List::Util qw( max );
use Text::Xslate;
use Text::WrapI18N qw(wrap);

use File::ShareDir;
use Log::Any qw($log);

use Diversion::FeedArchiver;
use Diversion::FeedArchiveIterator;

sub opt_spec {
    return (
        ["ago=i", "Include entries created up to this second ago.", { default => 86400 }],
        ["limit=i","Only include this number of entries..", { default => 0 }],
    )
}

sub is_blank {
    my $str = $_[0];
    return !defined($str) || $str =~ m/^\s*$/;
}

sub execute {
    my ($self, $opt, $args) = @_;

    my $JSON = JSON::PP->new->utf8;

    my $blob_store = $self->blob_store;
    my $iter = Diversion::FeedArchiveIterator->new( sql_order_clause => "created_at DESC" );
    my $now = DateTime->now;
    my $fmt = DateTime::Format::RSS->new;
    my %seen;
    my $tmpl_data = {};
    my $entry_count = 0;
    while (my $row = $iter->next()) {
	next unless $row->{uri} && $row->{sha1_digest};
	my $blob = $blob_store->get($row->{sha1_digest});
	my $data = $JSON->decode($blob);
	for my $entry (@{$data->{entry}}) {
	    next if !$entry->{pubDate} || !$entry->{link} || is_blank($entry->{title}) || $seen{$entry->{link}}++;
	    
            $entry->{_pubDate_as_DateTime} = $fmt->parse_datetime($entry->{pubDate});
	    
	    if ($entry->{_pubDate_as_DateTime}) {
		next if ($now - $entry->{_pubDate_as_DateTime})->in_units("seconds") > $opt->{ago};
	    } else {
		next;
	    }
	    
	    if ($entry->{description} && length($entry->{description}) > 140) {
		$entry->{description_short} = substr($entry->{description} =~ s/\p{Space}+/ /gr, 0, 137) . "...";
		$entry->{description_short} = substr($entry->{description}, 0, 137) . "...";
	    }
	    $entry->{description} //= "";
	    $entry->{description_short} //= "";
	    
	    push @{ $tmpl_data->{entries} }, $entry;

	    if ($opt->{limit}) {
		last if $entry_count++ > $opt->{limit};
	    }
	}
    }

    my @l = (localtime)[5,4,3,2,1];
    $l[0]+=1900;
    $l[1]+=1;
    my $ts = sprintf("%4d%02d%02d%02d%02d",@l);
    my $output_dir = Diversion::App->config->{output}{directory} || "/tmp";

    my ($text_body, $html_body);
    if ($text_body = build_text_mail($tmpl_data)) {
	io->catfile($output_dir, "diversion-mail-$ts.txt")->utf8->print($text_body);
    }
    
{'fraction_non_alphanumeric': 0.21369294605809128, 'fraction_numerical': 0.019087136929460582, 'mean_word_length': 3.608986615678776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 88, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3732379', 'n_tokens_mistral': 1898, 'n_tokens_neox': 1803, 'n_words': 361}