<commit_before># --
# Kernel/System/Console/Command/Dev/Tools/TranslationsUpdate.pm - console command
# Copyright (C) 2001-2015 OTRS AG, http://otrs.com/
# --
# This software comes with ABSOLUTELY NO WARRANTY. For details, see
# the enclosed file COPYING for license information (AGPL). If you
# did not receive this file, see http://www.gnu.org/licenses/agpl.txt.
# --

package Kernel::System::Console::Command::Dev::Tools::TranslationsUpdate;

use strict;
use warnings;

use base qw(Kernel::System::Console::BaseCommand);

our @ObjectDependencies = (
    'Kernel::Config',
    'Kernel::System::Encode',
    'Kernel::System::Main',
    'Kernel::System::SysConfig',
);

sub Configure {
    my ( $Self, %Param ) = @_;

    $Self->Description('Update the OTRS translation files.');
    $Self->AddOption(
        Name        => 'language',
        Description => "Which language to use, omit to update all languages.",
        Required    => 0,
        HasValue    => 1,
        ValueRegex  => qr/.*/smx,
    );
    $Self->AddOption(
        Name        => 'module-directory',
        Description => "Translate the OTRS module in the given directory.",
        Required    => 0,
        HasValue    => 1,
        ValueRegex  => qr/.*/smx,
    );
    $Self->AddOption(
        Name        => 'generate-pot',
        Description => "Generate POT (translation template) file.",
        Required    => 0,
        HasValue    => 0,
    );
    $Self->AddOption(
        Name        => 'generate-po',
        Description => "Generate PO (translation content) files.",
        Required    => 0,
        HasValue    => 0,
    );

    my $Name = $Self->Name();

    $Self->AdditionalHelp(<<"EOF");

<yellow>Translating OTRS</yellow>

Make sure that you have a clean system with a current configuration. No modules may be installed or linked into the system!

    <green>otrs.Console.pl $Name --language ...</green>

Since OTRS 4 public translations are managed in transifex. Use the <yellow>--generate-pot</yellow> switch to update the .pot files and the <yellow>--generate-po</yellow> switch to update both .pot and .po files (usually not needed).

<yellow>Translating Extension Modules</yellow>

Make sure that you have a clean system with a current configuration. The module that needs to be translated has to be installed or linked into the system, but only this one!

    <green>otrs.Console.pl $Name --language ... --module-directory ...</green>
EOF

    return;
}

my $BreakLineAfterChars = 60;

sub Run {
    my ( $Self, %Param ) = @_;

    my @Languages;
    my $LanguageOption = $Self->GetOption('language');
    my $Home           = $Kernel::OM->Get('Kernel::Config')->Get('Home');

    # check params
    if ( !$LanguageOption ) {
        my %DefaultUsedLanguages = %{ $Kernel::OM->Get('Kernel::Config')->Get('DefaultUsedLanguages') };
        @Languages = sort keys %DefaultUsedLanguages;
        @Languages = grep { $_ ne 'en' } @Languages;    # ignore en*.pm files
    }
    else {
        push @Languages, $LanguageOption;
        if ( !-f "$Home/Kernel/Language/$Languages[0].pm" ) {
            $Self->PrintError("No core translation file: $Languages[0]!");
            return $Self->ExitCodeError();
        }
    }

    $Self->Print("<yellow>Starting...</yellow>\n\n");

    # Gather some statistics
    my %Stats;

    for my $Language (@Languages) {
        $Self->HandleLanguage(
            Language => $Language,
            Module   => $Self->GetOption('module-directory'),
            WritePOT => $Self->GetOption('generate-po') || $Self->GetOption('generate-pot'),
            WritePO  => $Self->GetOption('generate-po'),
            Stats    => \%Stats,
        );
    }

    my %Summary;
    for my $Language ( sort keys %Stats ) {
        $Summary{$Language}->{Translated} = scalar grep {$_} values %{ $Stats{$Language} };
        $Summary{$Language}->{Total} = scalar values %{ $Stats{$Language} };
    }

    $Self->Print("\n<yellow>Translation statistics:</yellow>\n");
    for my $Language (
        sort { $Summary{$b}->{Translated} <=> $Summary{$a}->{Translated} }
        keys %Stats
        )
    {
        my $Strings      = $Summary{$Language}->{Total};
        my $Translations = $Summary{$Language}->{Translated};
        $Self->Print( "\t" . sprintf( "%7s", $Language ) . ": " );
        $Self->Print( sprintf( "%02d", int( ( $Translations / $Strings ) * 100 ) ) );
        $Self->Print( sprintf( "%% (%4d/%4d)\n", $Translations, $Strings ) );

    }

    $Self->Print("\n<green>Done.</green>\n");

    return $Self->ExitCodeOk();
}

sub HandleLanguage {
    my ( $Self, %Param ) = @_;

    my $Language = $Param{Language};
    my $Module   = $Param{Module};

    my $ModuleDirectory = $Module;
    my $LanguageFile;
    my $TargetFile;
    my $TargetPOTFile;
    my $TargetPOFile;
    my $IsSubTranslation;

    my $DefaultTheme = $Kernel::OM->Get('Kernel::Config')->Get('DefaultTheme');

    # We need to map internal codes to the official ones used by Transifex
    my %TransifexLanguagesMap = (
        sr_Cyrl => 'sr',
        sr_Latn => 'sr@latin',
    );

    my $TransifexLanguage = $TransifexLanguagesMap{$Language} // $Language;
    my $Home = $Kernel::OM->Get('Kernel::Config')->Get('Home');

    if ( !$Module ) {
        $LanguageFile  = "$Home/Kernel/Language/$Language.pm";
        $TargetFile    = "$Home/Kernel/Language/$Language.pm";
        $TargetPOTFile = "$Home/i18n/otrs/otrs.pot";
        $TargetPOFile  = "$Home/i18n/otrs/otrs.$TransifexLanguage.po";
    }
    else {
        $IsSubTranslation = 1;

        # extract module name from module path
        $Module = basename $Module;

        # remove underscores and/or version numbers and following from module name
        # i.e. FAQ_2_0 or FAQ20
        $Module =~ s{ [_0-9]+ .+ \z }{}xms;

        # save module directory in target file
        $TargetFile = "$ModuleDirectory/Kernel/Language/${Language}_$Module.pm";

        $TargetPOTFile = "$ModuleDirectory/i18n/$Module/$Module.pot";
        $TargetPOFile  = "$ModuleDirectory/i18n/$Module/$Module.$TransifexLanguage.po";
    }

    if ( !-w $TargetFile ) {
        $Self->PrintError("Ignoring nonexisting file $TargetFile!");
        return;
    }

    if ($IsSubTranslation) {
        $Self->Print(
            "Processing language <yellow>$Language</yellow> template files from <yellow>$Module</yellow>, writing output to <yellow>$TargetFile</yellow>\n"
        );
    }
    else {
        $Self->Print(
            "Processing language <yellow>$Language</yellow> template files, writing output to <yellow>$TargetFile</yellow>\n"
        );
    }

    # Language file, which only contains the OTRS core translations
    my $LanguageCoreObject = Kernel::Language->new(
        UserLanguage    => $Language,
        TranslationFile => 1,
    );

    # Language file, which contains all translations
    my $LanguageObject = Kernel::Language->new(
        UserLanguage => $Language,
    );

    my %POTranslations;

    if ( $Param{WritePOT} || $Param{WritePO} ) {
        %POTranslations = $Self->LoadPOFile(
            TargetPOFile => $TargetPOFile,
        );
    }

    # open .tt files and write new translation file
    my %UsedWords;
    my $Directory = $IsSubTranslation
        ? "$ModuleDirectory/Kernel/Output/HTML/$DefaultTheme"
        : "$Home/Kernel/Output/HTML/$DefaultTheme";

    my @List = $Kernel::OM->Get('Kernel::System::Main')->DirectoryRead(
        Directory => $Directory,
        Filter    => '*.tt',
    );

    my @TranslationStrings;

    for my $File (@List) {

        my $ContentRef = $Kernel::OM->Get('Kernel::System::Main')->FileRead(
            Location => $File,
            Mode     => 'utf8',
        );

        if ( !ref $ContentRef ) {
            die "Can't open $File: $!";
        }

        my $Content = ${$ContentRef};

        $File =~ s!^.*/(.+?)\.tt!$1!;

        # do translation
        $Content =~ s{
            Translate\(
                \s*
                (["'])(.*?)(?<!\\)\1
                \s*
                (?:,[^\)]+)?
            \)
            (?:\s|[|])
        }
        {
            my $Word = $2 // '';

            # unescape any \" or \' signs
            $Word =~ s{\\"}{"}smxg;
            $Word =~ s{\\'}{'}smxg;

            if ($Word && !exists $UsedWords{$Word}) {

                # if we translate a module, we must handle also that possibly
                # there is already a translation in the core files
                if ($IsSubTranslation) {<commit_msg>Include missing translation strings.<commit_after># --
# Kernel/System/Console/Command/Dev/Tools/TranslationsUpdate.pm - console command
# Copyright (C) 2001-2015 OTRS AG, http://otrs.com/
# --
# This software comes with ABSOLUTELY NO WARRANTY. For details, see
# the enclosed file COPYING for license information (AGPL). If you
# did not receive this file, see http://www.gnu.org/licenses/agpl.txt.
# --

package Kernel::System::Console::Command::Dev::Tools::TranslationsUpdate;

use strict;
use warnings;

use base qw(Kernel::System::Console::BaseCommand);

our @ObjectDependencies = (
    'Kernel::Config',
    'Kernel::System::Encode',
    'Kernel::System::Main',
    'Kernel::System::SysConfig',
);

sub Configure {
    my ( $Self, %Param ) = @_;

    $Self->Description('Update the OTRS translation files.');
    $Self->AddOption(
        Name        => 'language',
        Description => "Which language to use, omit to update all languages.",
        Required    => 0,
        HasValue    => 1,
        ValueRegex  => qr/.*/smx,
    );
    $Self->AddOption(
        Name        => 'module-directory',
        Description => "Translate the OTRS module in the given directory.",
        Required    => 0,
        HasValue    => 1,
        ValueRegex  => qr/.*/smx,
    );
    $Self->AddOption(
        Name        => 'generate-pot',
        Description => "Generate POT (translation template) file.",
        Required    => 0,
        HasValue    => 0,
    );
    $Self->AddOption(
        Name        => 'generate-po',
        Description => "Generate PO (translation content) files.",
        Required    => 0,
        HasValue    => 0,
    );

    my $Name = $Self->Name();

    $Self->AdditionalHelp(<<"EOF");

<yellow>Translating OTRS</yellow>

Make sure that you have a clean system with a current configuration. No modules may be installed or linked into the system!

    <green>otrs.Console.pl $Name --language ...</green>

Since OTRS 4 public translations are managed in transifex. Use the <yellow>--generate-pot</yellow> switch to update the .pot files and the <yellow>--generate-po</yellow> switch to update both .pot and .po files (usually not needed).

<yellow>Translating Extension Modules</yellow>

Make sure that you have a clean system with a current configuration. The module that needs to be translated has to be installed or linked into the system, but only this one!

    <green>otrs.Console.pl $Name --language ... --module-directory ...</green>
EOF

    return;
}

my $BreakLineAfterChars = 60;

sub Run {
    my ( $Self, %Param ) = @_;

    my @Languages;
    my $LanguageOption = $Self->GetOption('language');
    my $Home           = $Kernel::OM->Get('Kernel::Config')->Get('Home');

    # check params
    if ( !$LanguageOption ) {
        my %DefaultUsedLanguages = %{ $Kernel::OM->Get('Kernel::Config')->Get('DefaultUsedLanguages') };
        @Languages = sort keys %DefaultUsedLanguages;
        @Languages = grep { $_ ne 'en' } @Languages;    # ignore en*.pm files
    }
    else {
        push @Languages, $LanguageOption;
        if ( !-f "$Home/Kernel/Language/$Languages[0].pm" ) {
            $Self->PrintError("No core translation file: $Languages[0]!");
            return $Self->ExitCodeError();
        }
    }

    $Self->Print("<yellow>Starting...</yellow>\n\n");

    # Gather some statistics
    my %Stats;

    for my $Language (@Languages) {
        $Self->HandleLanguage(
            Language => $Language,
            Module   => $Self->GetOption('module-directory'),
            WritePOT => $Self->GetOption('generate-po') || $Self->GetOption('generate-pot'),
            WritePO  => $Self->GetOption('generate-po'),
            Stats    => \%Stats,
        );
    }

    my %Summary;
    for my $Language ( sort keys %Stats ) {
        $Summary{$Language}->{Translated} = scalar grep {$_} values %{ $Stats{$Language} };
        $Summary{$Language}->{Total} = scalar values %{ $Stats{$Language} };
    }

    $Self->Print("\n<yellow>Translation statistics:</yellow>\n");
    for my $Language (
        sort { $Summary{$b}->{Translated} <=> $Summary{$a}->{Translated} }
        keys %Stats
        )
    {
        my $Strings      = $Summary{$Language}->{Total};
        my $Translations = $Summary{$Language}->{Translated};
        $Self->Print( "\t" . sprintf( "%7s", $Language ) . ": " );
        $Self->Print( sprintf( "%02d", int( ( $Translations / $Strings ) * 100 ) ) );
        $Self->Print( sprintf( "%% (%4d/%4d)\n", $Translations, $Strings ) );

    }

    $Self->Print("\n<green>Done.</green>\n");

    return $Self->ExitCodeOk();
}

sub HandleLanguage {
    my ( $Self, %Param ) = @_;

    my $Language = $Param{Language};
    my $Module   = $Param{Module};

    my $ModuleDirectory = $Module;
    my $LanguageFile;
    my $TargetFile;
    my $TargetPOTFile;
    my $TargetPOFile;
    my $IsSubTranslation;

    my $DefaultTheme = $Kernel::OM->Get('Kernel::Config')->Get('DefaultTheme');

    # We need to map internal codes to the official ones used by Transifex
    my %TransifexLanguagesMap = (
        sr_Cyrl => 'sr',
        sr_Latn => 'sr@latin',
    );

    my $TransifexLanguage = $TransifexLanguagesMap{$Language} // $Language;
    my $Home = $Kernel::OM->Get('Kernel::Config')->Get('Home');

    if ( !$Module ) {
        $LanguageFile  = "$Home/Kernel/Language/$Language.pm";
        $TargetFile    = "$Home/Kernel/Language/$Language.pm";
        $TargetPOTFile = "$Home/i18n/otrs/otrs.pot";
        $TargetPOFile  = "$Home/i18n/otrs/otrs.$TransifexLanguage.po";
    }
    else {
        $IsSubTranslation = 1;

        # extract module name from module path
        $Module = basename $Module;

        # remove underscores and/or version numbers and following from module name
        # i.e. FAQ_2_0 or FAQ20
        $Module =~ s{ [_0-9]+ .+ \z }{}xms;

        # save module directory in target file
        $TargetFile = "$ModuleDirectory/Kernel/Language/${Language}_$Module.pm";

        $TargetPOTFile = "$ModuleDirectory/i18n/$Module/$Module.pot";
        $TargetPOFile  = "$ModuleDirectory/i18n/$Module/$Module.$TransifexLanguage.po";
    }

    if ( !-w $TargetFile ) {
        $Self->PrintError("Ignoring nonexisting file $TargetFile!");
        return;
    }

    if ($IsSubTranslation) {
        $Self->Print(
            "Processing language <yellow>$Language</yellow> template files from <yellow>$Module</yellow>, writing output to <yellow>$TargetFile</yellow>\n"
        );
    }
    else {
        $Self->Print(
            "Processing language <yellow>$Language</yellow> template files, writing output to <yellow>$TargetFile</yellow>\n"
        );
    }

    # Language file, which only contains the OTRS core translations
    my $LanguageCoreObject = Kernel::Language->new(
        UserLanguage    => $Language,
        TranslationFile => 1,
    );

    # Language file, which contains all translations
    my $LanguageObject = Kernel::Language->new(
        UserLanguage => $Language,
    );

    my %POTranslations;

    if ( $Param{WritePOT} || $Param{WritePO} ) {
        %POTranslations = $Self->LoadPOFile(
            TargetPOFile => $TargetPOFile,
        );
    }

    # open .tt files and write new translation file
    my %UsedWords;
    my $Directory = $IsSubTranslation
        ? "$ModuleDirectory/Kernel/Output/HTML/$DefaultTheme"
        : "$Home/Kernel/Output/HTML/$DefaultTheme";

    my @List = $Kernel::OM->Get('Kernel::System::Main')->DirectoryRead(
        Directory => $Directory,
        Filter    => '*.tt',
    );

    my @TranslationStrings;

    for my $File (@List) {

        my $ContentRef = $Kernel::OM->Get('Kernel::System::Main')->FileRead(
            Location => $File,
            Mode     => 'utf8',
        );

        if ( !ref $ContentRef ) {
            die "Can't open $File: $!";
        }

        my $Content = ${$ContentRef};

        $File =~ s!^.*/(.+?)\.tt!$1!;

        # do translation
        $Content =~ s{
            Translate\(
                \s*
                (["'])(.*?)(?<!\\)\1
        }
        {
            my $Word = $2 // '';

            # unescape any \" or \' signs
            $Word =~ s{\\"}{"}smxg;
            $Word =~ s{\\'}{'}smxg;

            if ($Word && !exists $UsedWords{$Word}) {

                # if we translate a module, we must handle also that possibly
                # there is already a translation in the core files
                if ($IsSubTranslation) {
{'fraction_non_alphanumeric': 0.15638636631678263, 'fraction_numerical': 0.005778983370680505, 'mean_word_length': 2.3668850506253722, 'pattern_counts': {'":': 2, '<': 67, '<?xml version=': 0, '>': 219, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4822130', 'n_tokens_mistral': 5257, 'n_tokens_neox': 4978, 'n_words': 1247}