<commit_before># --
# Copyright (C) 2001-2017 OTRS AG, http://otrs.com/
# --
# This software comes with ABSOLUTELY NO WARRANTY. For details, see
# the enclosed file COPYING for license information (AGPL). If you
# did not receive this file, see http://www.gnu.org/licenses/agpl.txt.
# --

package Kernel::System::Ticket::ArticleSearchIndex::DB;

use strict;
use warnings;

use Kernel::System::VariableCheck qw(:all);

our @ObjectDependencies = (
    'Kernel::Config',
    'Kernel::System::DB',
    'Kernel::System::Log',
    'Kernel::System::Ticket::Article',
);

=head1 NAME

Kernel::System::Ticket::ArticleSearchIndex::DB - DB based ticket article search index module

=head1 DESCRIPTION

This class provides functions to index articles for searching in the database.
The methods are currently documented in L<Kernel::System::Ticket::Article>.

=cut

sub new {
    my ( $Type, %Param ) = @_;

    my $Self = {};
    bless( $Self, $Type );

    return $Self;
}

sub ArticleSearchIndexBuild {
    my ( $Self, %Param ) = @_;

    for my $Needed (qw(TicketID ArticleID UserID)) {
        if ( !$Param{$Needed} ) {
            $Kernel::OM->Get('Kernel::System::Log')->Log(
                Priority => 'error',
                Message  => "Need $Needed!"
            );
            return;
        }
    }

    my $ArticleBackendObject = $Kernel::OM->Get('Kernel::System::Ticket::Article')->BackendForArticle(
        TicketID  => $Param{TicketID},
        ArticleID => $Param{ArticleID},
    );

    my %ArticleSearchableContent = $ArticleBackendObject->ArticleSearchableContentGet(
        TicketID  => $Param{TicketID},
        ArticleID => $Param{ArticleID},
        UserID    => $Param{UserID},
    );

    return 1 if !%ArticleSearchableContent;

    # clear old data from search index table
    my $Success = $Self->ArticleSearchIndexDelete(
        ArticleID => $Param{ArticleID},
        UserID    => $Param{UserID},
    );

    if ( !$Success ) {
        $Kernel::OM->Get('Kernel::System::Log')->Log(
            Priority => 'error',
            Message  => "Could not delete ArticleID '$Param{ArticleID}' from article search index!"
        );
        return;
    }

    my $FilterStopWords = $Kernel::OM->Get('Kernel::Config')->Get('Ticket::SearchIndex::FilterStopWords') // 1;

    my $DBObject = $Kernel::OM->Get('Kernel::System::DB');

    my @Bind;
    my @InsertValues;
    my $SQL = 'INSERT INTO article_search_index (ticket_id, article_id, article_key, article_value) VALUES ';

    for my $FieldKey ( sort keys %ArticleSearchableContent ) {

        if (
            $FilterStopWords
            && $ArticleSearchableContent{$FieldKey}->{Filterable}
            )
        {
            $ArticleSearchableContent{$FieldKey}->{String} = $Self->_ArticleSearchIndexString(
                %{ $ArticleSearchableContent{$FieldKey} }
            );
        }

        # Indexed content will be saved lowercase, even if it is not filterable to avoid
        # LOWER() statements on search time, which increases the search performance.
        # (this will be done automatically on filterable fields)
        else {
            $ArticleSearchableContent{$FieldKey}->{String} = lc $ArticleSearchableContent{$FieldKey}->{String};
        }

        # save insert values and bind variables
        push @InsertValues, '(?, ?, ?, ?)';
        push @Bind,         (
            \$Param{TicketID},
            \$Param{ArticleID},
            \$ArticleSearchableContent{$FieldKey}->{Key},
            \$ArticleSearchableContent{$FieldKey}->{String},
        );
    }

    $SQL .= join ',', @InsertValues;<commit_msg>Added separation between multiple inserts for oracle and other db types in ArticleSearchIndexBuild.<commit_after># --
# Copyright (C) 2001-2017 OTRS AG, http://otrs.com/
# --
# This software comes with ABSOLUTELY NO WARRANTY. For details, see
# the enclosed file COPYING for license information (AGPL). If you
# did not receive this file, see http://www.gnu.org/licenses/agpl.txt.
# --

package Kernel::System::Ticket::ArticleSearchIndex::DB;

use strict;
use warnings;

use Kernel::System::VariableCheck qw(:all);

our @ObjectDependencies = (
    'Kernel::Config',
    'Kernel::System::DB',
    'Kernel::System::Log',
    'Kernel::System::Ticket::Article',
);

=head1 NAME

Kernel::System::Ticket::ArticleSearchIndex::DB - DB based ticket article search index module

=head1 DESCRIPTION

This class provides functions to index articles for searching in the database.
The methods are currently documented in L<Kernel::System::Ticket::Article>.

=cut

sub new {
    my ( $Type, %Param ) = @_;

    my $Self = {};
    bless( $Self, $Type );

    return $Self;
}

sub ArticleSearchIndexBuild {
    my ( $Self, %Param ) = @_;

    for my $Needed (qw(TicketID ArticleID UserID)) {
        if ( !$Param{$Needed} ) {
            $Kernel::OM->Get('Kernel::System::Log')->Log(
                Priority => 'error',
                Message  => "Need $Needed!"
            );
            return;
        }
    }

    my $ArticleBackendObject = $Kernel::OM->Get('Kernel::System::Ticket::Article')->BackendForArticle(
        TicketID  => $Param{TicketID},
        ArticleID => $Param{ArticleID},
    );

    my %ArticleSearchableContent = $ArticleBackendObject->ArticleSearchableContentGet(
        TicketID  => $Param{TicketID},
        ArticleID => $Param{ArticleID},
        UserID    => $Param{UserID},
    );

    return 1 if !%ArticleSearchableContent;

    # clear old data from search index table
    my $Success = $Self->ArticleSearchIndexDelete(
        ArticleID => $Param{ArticleID},
        UserID    => $Param{UserID},
    );

    if ( !$Success ) {
        $Kernel::OM->Get('Kernel::System::Log')->Log(
            Priority => 'error',
            Message  => "Could not delete ArticleID '$Param{ArticleID}' from article search index!"
        );
        return;
    }

    my $FilterStopWords = $Kernel::OM->Get('Kernel::Config')->Get('Ticket::SearchIndex::FilterStopWords') // 1;

    my $DBObject = $Kernel::OM->Get('Kernel::System::DB');

    # determine the insert type to use
    my $DBType                 = $DBObject->GetDatabaseFunction('Type');
    my $AllowMultiple          = 1;
    my @SupportMultipleInserts = qw(mysql oracle postgresql);

    if ( !grep( /$DBType/, @SupportMultipleInserts ) ) {
        $AllowMultiple = 0;
    }

    my %MultiInsertStructure = (
        Oracle => {
            Start  => 'INSERT ALL ',
            Insert => '
                INTO article_search_index (
                    ticket_id, article_id, article_key, article_value
                )
                (?, ?, ?, ?)',
            End  => 'SELECT * FROM DUAL',
            Join => ' ',
        },
        Other => {
            Start  => 'INSERT INTO article_search_index (ticket_id, article_id, article_key, article_value) VALUES ',
            Insert => '(?, ?, ?, ?) ',
            End    => '',
            Join   => ', ',
            }
    );

    my $MultiInsertStructureKey = $DBType eq 'oracle' ? 'Oracle' : 'Other';

    my $SQL = $AllowMultiple ? $MultiInsertStructure{$MultiInsertStructureKey}->{Start} : '';
    my $FirstRun = 1;
    my @Bind;

    for my $FieldKey ( sort keys %ArticleSearchableContent ) {

        if (
            $FilterStopWords
            && $ArticleSearchableContent{$FieldKey}->{Filterable}
            )
        {
            $ArticleSearchableContent{$FieldKey}->{String} = $Self->_ArticleSearchIndexString(
                %{ $ArticleSearchableContent{$FieldKey} }
            );
        }

        # Indexed content will be saved lowercase, even if it is not filterable to avoid
        # LOWER() statements on search time, which increases the search performance.
        # (this will be done automatically on filterable fields)
        else {
            $ArticleSearchableContent{$FieldKey}->{String} = lc $ArticleSearchableContent{$FieldKey}->{String};
        }

        my @CurrentBind = (
            \$Param{TicketID},
            \$Param{ArticleID},
            \$ArticleSearchableContent{$FieldKey}->{Key},
            \$ArticleSearchableContent{$FieldKey}->{String},
        );

        $SQL .= $MultiInsertStructure{$MultiInsertStructureKey}->{Join} if !$FirstRun;
        $SQL .= $MultiInsertStructure{$MultiInsertStructureKey}->{Insert};

        # explicitly avoid multiple inserts in case the query size limit was reached
        $AllowMultiple = 0 if length $SQL > 3500 && $AllowMultiple;

        # Check the length of the SQL string on every cycle
        # (some databases only accept SQL strings up to 4k,
        # so we want to stay safe here with just 3500 bytes).
        if ( !$AllowMultiple ) {
            return if !$DBObject->Do(
                SQL => '
                    INSERT INTO article_search_index (
                        ticket_id, article_id, article_key, article_value
                    )
                    VALUES (?, ?, ?, ?)
                ',
                Bind => \@CurrentBind,
            );
        }
        else {
            push @Bind, @CurrentBind;
        }

        $FirstRun = 0 if $FirstRun;
    }

    return 1 if !$AllowMultiple;
{'fraction_non_alphanumeric': 0.13251828403012772, 'fraction_numerical': 0.004257177164065058, 'mean_word_length': 2.4072145779100036, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7239975', 'n_tokens_mistral': 2730, 'n_tokens_neox': 2549, 'n_words': 637}