<commit_before>=pod
GADS - Globally Accessible Data Store
Copyright (C) 2014 Ctrl O Ltd

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
=cut

package GADS::Record;

use Dancer2 ':script';
use Dancer2::Plugin::DBIC qw(schema resultset rset);
use GADS::Record;
use GADS::Util         qw(:all);
use String::CamelCase qw(camelize);
use Ouch;
use Safe;
use DateTime;
use DateTime::Format::Strptime qw( );
use POSIX qw(ceil);
schema->storage->debug(1);

use GADS::Schema;

sub _safe_eval;
sub _search_construct;

sub current($$)
{   my ($class, $item) = @_;
    my $record_rs = rset('Record');
    my @ids;

    my $rows = $item->{rows};
    my $page = $item->{page};

    if($item->{record_id}) {
        push @ids, $item->{record_id};
    }
    else
    {
        my $search = $item->{current_id} ? { 'current_id' => $item->{current_id} } : {};
        $search->{approval} = 0;
        $search->{"me.record_id"} = undef;

        my $select =
        {
            'select' =>
            [
                { max => 'me.id',
                  -as => 'id',
                },
            ],
            'group_by' => 'current_id',
        };

        # Search for filtering results has to be done after initial selection,
        # otherwise "max" selection to get latest record version causes wrong
        # values to be selected
        my $count = $record_rs->search(
            $search, $select
        )->count;

        # Send page information back
        $item->{pages} = $rows ? ceil($count / $rows) : 1;

        $select->{rows} = $rows if $rows;
        $select->{page} = $page if $page;

        my $records = $record_rs->search(
            $search, $select
        );

        foreach my $r ($records->all) {
            push @ids, $r->id;
        }
    }

    my $search = {
        'me.id'  => \@ids,
        approval => 0
    };

    my @join;     # Used for conditionals, which may not be shown in the view of choice
    my @prefetch; # Used for any field that we display the value for
    
    if (my $view_id = $item->{view_id})
    {
        # So that we know how many times we've joined a table.
        # DBIC automatically suffixes additional joines _2 etc.
        my %joincount;
        # Keep a track of which we have joined at prefetch, so that we
        # add to this for the searches, otherwise use a join
        my %prefetch_used;

        # First, add all the columns in the view as a prefetch. During
        # this stage, we keep track of what we've added, so that we
        # can act accordingly during the filters
        my @columns = @{GADS::View->columns({ view_id => $view_id })};
        push @columns, @{$item->{additional}} if $item->{additional};
        foreach my $c (@columns)
        {
            # If it's a calculated/rag value, prefetch the columns in the calc
            if ($c->{type} eq 'rag' || $c->{type} eq 'calc')
            {
                push @columns, @{$c->{$c->{type}}->{columns}};
                next;
            }

            # Value of enums are always "value" (joined table)
            push @prefetch, $c->{join};          # Add to prefetch list
            $prefetch_used{$c->{sprefix}} = 1;   # Tag that we've prefetched this value
            $joincount{$c->{sprefix}}++;         # Increment join counter to track _2 suffixes
        }

        # Now add all the filters as joins (we don't need to prefetch this data). However,
        # the filter might also be a column in the view from before, in which case add
        # it to, or use, the prefetch. We use the tracking variables from above.
        foreach my $filter (@{GADS::View->filters($view_id)})
        {
            my $field = $filter->{column}->{field};
            my $fieldsearch;

            if ($filter->{column}->{type} eq "rag" || $filter->{column}->{type} eq "calc" || $filter->{column}->{type} eq "person")
            {
                next;
            }
            else
            {
                # As per the prefix, what we join depends on the type of field
                my $sprefix = $filter->{column}->{sprefix};
                my $joinv   = $filter->{column}->{join};
                if ($prefetch_used{$sprefix})
                {
                    push @prefetch, $joinv;
                }
                else {
                    push @join, $joinv;
                }
                $joincount{$sprefix}++;
                my $in = $joincount{$sprefix} == 1 ? '' : "_$joincount{$sprefix}"; # Join suffix
                $fieldsearch = "$sprefix$in.value";
            }<commit_msg>Prefetch all data for no view selected<commit_after>=pod
GADS - Globally Accessible Data Store
Copyright (C) 2014 Ctrl O Ltd

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
=cut

package GADS::Record;

use Dancer2 ':script';
use Dancer2::Plugin::DBIC qw(schema resultset rset);
use GADS::Record;
use GADS::Util         qw(:all);
use String::CamelCase qw(camelize);
use Ouch;
use Safe;
use DateTime;
use DateTime::Format::Strptime qw( );
use POSIX qw(ceil);
schema->storage->debug(1);

use GADS::Schema;

sub _safe_eval;
sub _search_construct;

sub current($$)
{   my ($class, $item) = @_;
    my $record_rs = rset('Record');
    my @ids;

    my $rows = $item->{rows};
    my $page = $item->{page};

    if($item->{record_id}) {
        push @ids, $item->{record_id};
    }
    else
    {
        my $search = $item->{current_id} ? { 'current_id' => $item->{current_id} } : {};
        $search->{approval} = 0;
        $search->{"me.record_id"} = undef;

        my $select =
        {
            'select' =>
            [
                { max => 'me.id',
                  -as => 'id',
                },
            ],
            'group_by' => 'current_id',
        };

        # Search for filtering results has to be done after initial selection,
        # otherwise "max" selection to get latest record version causes wrong
        # values to be selected
        my $count = $record_rs->search(
            $search, $select
        )->count;

        # Send page information back
        $item->{pages} = $rows ? ceil($count / $rows) : 1;

        $select->{rows} = $rows if $rows;
        $select->{page} = $page if $page;

        my $records = $record_rs->search(
            $search, $select
        );

        foreach my $r ($records->all) {
            push @ids, $r->id;
        }
    }

    my $search = {
        'me.id'  => \@ids,
        approval => 0
    };

    my @join;     # Used for conditionals, which may not be shown in the view of choice
    my @prefetch; # Used for any field that we display the value for
    
    # So that we know how many times we've joined a table.
    # DBIC automatically suffixes additional joines _2 etc.
    my %joincount;
    # Keep a track of which we have joined at prefetch, so that we
    # add to this for the searches, otherwise use a join
    my %prefetch_used;

    # First, add all the columns in the view as a prefetch. During
    # this stage, we keep track of what we've added, so that we
    # can act accordingly during the filters
    my @columns;
    if (my $view_id = $item->{view_id})
    {
        @columns = @{GADS::View->columns({ view_id => $view_id })};
    }
    else {
        @columns = @{GADS::View->columns};
    }
    push @columns, @{$item->{additional}} if $item->{additional};
    foreach my $c (@columns)
    {
        # If it's a calculated/rag value, prefetch the columns in the calc
        if ($c->{type} eq 'rag' || $c->{type} eq 'calc')
        {
            push @columns, @{$c->{$c->{type}}->{columns}};
            next;
        }

        # Value of enums are always "value" (joined table)
        push @prefetch, $c->{join};          # Add to prefetch list
        $prefetch_used{$c->{sprefix}} = 1;   # Tag that we've prefetched this value
        $joincount{$c->{sprefix}}++;         # Increment join counter to track _2 suffixes
    }

    # Now add all the filters as joins (we don't need to prefetch this data). However,
    # the filter might also be a column in the view from before, in which case add
    # it to, or use, the prefetch. We use the tracking variables from above.
    if (my $view_id = $item->{view_id})
    {
        foreach my $filter (@{GADS::View->filters($view_id)})
        {
            my $field = $filter->{column}->{field};
            my $fieldsearch;

            if ($filter->{column}->{type} eq "rag" || $filter->{column}->{type} eq "calc" || $filter->{column}->{type} eq "person")
            {
                next;
            }
            else
            {
                # As per the prefix, what we join depends on the type of field
                my $sprefix = $filter->{column}->{sprefix};
                my $joinv   = $filter->{column}->{join};
                if ($prefetch_used{$sprefix})
                {
                    push @prefetch, $joinv;
                }
                else {
                    push @join, $joinv;
                }
                $joincount{$sprefix}++;
                my $in = $joincount{$sprefix} == 1 ? '' : "_$joincount{$sprefix}"; # Join suffix
                $fieldsearch = "$sprefix$in.value";
            }
{'fraction_non_alphanumeric': 0.12467356610890802, 'fraction_numerical': 0.0029016345874842827, 'mean_word_length': 2.0014513788098696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7416981', 'n_tokens_mistral': 3145, 'n_tokens_neox': 3028, 'n_words': 1080}