<commit_before>package Diversion::UrlArchiver {
    use v5.18;
    use Moo;
    with('Diversion::Db', 'Diversion::AppRole');

    use HTTP::Tiny;
    use Encode;
    use JSON;
    use DBI;
    use Digest::SHA1 'sha1_hex';
    use Data::Binary qw(is_binary);

    my $JSON = JSON->new->canonical->pretty;

    sub get_local {
        my ($self, $url) = @_;

        my $sha1_digest = $self->db_open(
            "url",
            sub {
                my ($dbh) = @_;
                my ($v) = @{$dbh->selectcol_arrayref(q{ SELECT sha1_digest FROM uri_archive WHERE uri = ? ORDER BY created_at DESC LIMIT 1},{}, $url)};
                return $v;
            }
        );
        return undef unless defined $sha1_digest;

        my $blob_store = $self->blob_store;
        if (my $response = $blob_store->get($sha1_digest)) {
            return $JSON->decode($response);
        }
        return undef;
    }

    sub get_remote {
        my ($self, $url) = @_;

        my $blob_store = $self->blob_store;

        my $response = HTTP::Tiny->new( timeout => 60 )->get($url);
        if (is_binary($response->{content})) {
            $response->{content} = { sha1_digest => $blob_store->put($response->{content}) };
        }
        my $response_digest = $blob_store->put("". $JSON->encode($response));

        $self->db_open(
            "url",
            sub {
                my ($dbh) = @_;
                my $sth_check = $dbh->prepare(q{ SELECT 1 FROM uri_archive WHERE uri = ? AND sha1_digest = ? LIMIT 1});
                $sth_check->execute($url, $response_digest);
                unless ($sth_check->fetchrow_array) {
                    my $sth_insert = $dbh->prepare(q{ INSERT INTO uri_archive(uri, created_at, sha1_digest) VALUES (?,?,?)});
                    $sth_insert->execute($url, 0+time, $response_digest);
                    $sth_insert->finish;
                }
                $sth_check->finish;
            }
        );

        return $response;
    }

    sub get {
        my ($self, $url) = @_;
        my $response = $self->get_local($url);
        return $response if defined $response;
        return $self->get_remote($url);
    }
};
1;
<commit_msg>Set the max_size of http response to be 400KB.<commit_after>package Diversion::UrlArchiver {
    use v5.18;
    use Moo;
    with('Diversion::Db', 'Diversion::AppRole');

    use HTTP::Tiny;
    use Encode;
    use JSON;
    use DBI;
    use Digest::SHA1 'sha1_hex';
    use Data::Binary qw(is_binary);

    my $JSON = JSON->new->canonical->pretty;

    sub get_local {
        my ($self, $url) = @_;

        my $sha1_digest = $self->db_open(
            "url",
            sub {
                my ($dbh) = @_;
                my ($v) = @{$dbh->selectcol_arrayref(q{ SELECT sha1_digest FROM uri_archive WHERE uri = ? ORDER BY created_at DESC LIMIT 1},{}, $url)};
                return $v;
            }
        );
        return undef unless defined $sha1_digest;

        my $blob_store = $self->blob_store;
        if (my $response = $blob_store->get($sha1_digest)) {
            return $JSON->decode($response);
        }
        return undef;
    }

    sub get_remote {
        my ($self, $url) = @_;

        my $blob_store = $self->blob_store;

        my $response = HTTP::Tiny->new( timeout => 60, max_size => 409600 )->get($url);
        if (is_binary($response->{content})) {
            $response->{content} = { sha1_digest => $blob_store->put($response->{content}) };
        }
        my $response_digest = $blob_store->put("". $JSON->encode($response));

        $self->db_open(
            "url",
            sub {
                my ($dbh) = @_;
                my $sth_check = $dbh->prepare(q{ SELECT 1 FROM uri_archive WHERE uri = ? AND sha1_digest = ? LIMIT 1});
                $sth_check->execute($url, $response_digest);
                unless ($sth_check->fetchrow_array) {
                    my $sth_insert = $dbh->prepare(q{ INSERT INTO uri_archive(uri, created_at, sha1_digest) VALUES (?,?,?)});
                    $sth_insert->execute($url, 0+time, $response_digest);
                    $sth_insert->finish;
                }
                $sth_check->finish;
            }
        );

        return $response;
    }

    sub get {
        my ($self, $url) = @_;
        my $response = $self->get_local($url);
        return $response if defined $response;
        return $self->get_remote($url);
    }
};
1;

{'fraction_non_alphanumeric': 0.14913268754223924, 'fraction_numerical': 0.010587970263572877, 'mean_word_length': 1.847979474021809, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '215198', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1386, 'n_words': 306}