<commit_before>=pod
GADS - Globally Accessible Data Store
Copyright (C) 2017 Ctrl O Ltd

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
=cut

package GADS::Filter;

use Data::Compare qw/Compare/;
use GADS::Config;
use Encode;
use JSON qw(decode_json encode_json);
use Log::Report 'linkspace';
use MIME::Base64;

use Moo;
use MooX::Types::MooseLike::Base qw(:all);

has as_json => (
    is      => 'rw',
    isa     => sub {
        decode_json_utf8($_[0]); # Will die on error
    },
    lazy    => 1,
    clearer => 1,
    coerce  => sub {
        # Ensure consistent format
        my $json = shift || '{}';
        my $hash = decode_json_utf8($json);
        decode("utf8", encode_json($hash));
    },
    builder => sub {
        my $self = shift;
        $self->_set_has_value(1);
        encode_json($self->as_hash);
    },
    predicate => 1,
    trigger => sub {
        my ($self, $new) = @_;
        # Need to compare as structures, as stringified JSON could be in different orders
        $self->_set_changed(1) if $self->has_value && !Compare($self->as_hash, decode_json_utf8($new));
        $self->clear_as_hash;
        # Force the hash to build with the new value, which will effectively
        # allow it to hold the old value, if we make any further changse (which
        # is then used to see if a change as happened)
        $self->as_hash;
        $self->_clear_lazy;
        $self->_set_has_value(1);
    },
);

has as_hash => (
    is      => 'rw',
    isa     => HashRef,
    lazy    => 1,
    clearer => 1,
    builder => sub {
        my $self = shift;
        $self->_set_has_value(1);
        return {} if !$self->has_as_json;
        decode_json_utf8($self->as_json);
    },
    trigger => sub {
        my ($self, $new) = @_;
        $self->_set_changed(1) if $self->has_value && !Compare(decode_json_utf8($self->as_json), $new);
        $self->clear_as_json;
        # Force the JSON to build with the new value, which will effectively
        # allow it to hold the old value, if we make any further changse (which
        # is then used to see if a change as happened)
        $self->as_json;
        $self->_clear_lazy;
        $self->_set_has_value(1);
    },
);

has has_value => (
    is  => 'rwp',
    isa => Bool,
);

has layout => (
    is => 'rw',
);

# Takes a JSON string that has wide characters, decodes it into utf8 bytes
# first, and then decodes the JSON. decode_json() expects utf8 binary
# characters, otherwise it borks. This module saves to the database and uses<commit_msg>Fix memory leak<commit_after>=pod
GADS - Globally Accessible Data Store
Copyright (C) 2017 Ctrl O Ltd

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
=cut

package GADS::Filter;

use Data::Compare qw/Compare/;
use GADS::Config;
use Encode;
use JSON qw(decode_json encode_json);
use Log::Report 'linkspace';
use MIME::Base64;

use Moo;
use MooX::Types::MooseLike::Base qw(:all);

has as_json => (
    is      => 'rw',
    isa     => sub {
        decode_json_utf8($_[0]); # Will die on error
    },
    lazy    => 1,
    clearer => 1,
    coerce  => sub {
        # Ensure consistent format
        my $json = shift || '{}';
        my $hash = decode_json_utf8($json);
        decode("utf8", encode_json($hash));
    },
    builder => sub {
        my $self = shift;
        $self->_set_has_value(1);
        encode_json($self->as_hash);
    },
    predicate => 1,
    trigger => sub {
        my ($self, $new) = @_;
        # Need to compare as structures, as stringified JSON could be in different orders
        $self->_set_changed(1) if $self->has_value && !Compare($self->as_hash, decode_json_utf8($new));
        $self->clear_as_hash;
        # Force the hash to build with the new value, which will effectively
        # allow it to hold the old value, if we make any further changse (which
        # is then used to see if a change as happened)
        $self->as_hash;
        $self->_clear_lazy;
        $self->_set_has_value(1);
    },
);

has as_hash => (
    is      => 'rw',
    isa     => HashRef,
    lazy    => 1,
    clearer => 1,
    builder => sub {
        my $self = shift;
        $self->_set_has_value(1);
        return {} if !$self->has_as_json;
        decode_json_utf8($self->as_json);
    },
    trigger => sub {
        my ($self, $new) = @_;
        $self->_set_changed(1) if $self->has_value && !Compare(decode_json_utf8($self->as_json), $new);
        $self->clear_as_json;
        # Force the JSON to build with the new value, which will effectively
        # allow it to hold the old value, if we make any further changse (which
        # is then used to see if a change as happened)
        $self->as_json;
        $self->_clear_lazy;
        $self->_set_has_value(1);
    },
);

has has_value => (
    is  => 'rwp',
    isa => Bool,
);

has layout => (
    is       => 'rw',
    # Make a weak ref so that when this object is created, it doesn't require
    # the reference to the layout to be destroyed to destroy this filter
    weak_ref => 1,
);

# Takes a JSON string that has wide characters, decodes it into utf8 bytes
# first, and then decodes the JSON. decode_json() expects utf8 binary
# characters, otherwise it borks. This module saves to the database and uses
{'fraction_non_alphanumeric': 0.10883822610868207, 'fraction_numerical': 0.008588382261086821, 'mean_word_length': 2.7238372093023258, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024271', 'n_tokens_mistral': 1984, 'n_tokens_neox': 1939, 'n_words': 735}