<commit_before>:
eval 'exec perl -wS $0 ${1+"$@"}'
    if 0;


#*************************************************************************
#
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# Copyright 2000, 2010 Oracle and/or its affiliates.
#
# OpenOffice.org - a multi-platform office productivity suite
#
# This file is part of OpenOffice.org.
#
# OpenOffice.org is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenOffice.org is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenOffice.org.  If not, see
# <http://www.openoffice.org/license.html>
# for a copy of the LGPLv3 License.
#
#*************************************************************************

use strict;
use Getopt::Long;
use IO::Handle;
use File::Find;
use File::Temp;
use File::Path;
use File::Copy;
use File::Glob qw(:glob csh_glob);
use Cwd;

my $CVS_BINARY = "/usr/bin/cvs";
# ver 1.1
#
#### module lookup
#use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");

#### module lookup
# OOo conform
my @lib_dirs;
BEGIN {
    if ( !defined($ENV{SOLARENV}) ) {
        die "No environment found (environment variable SOLARENV is undefined)";
    }
    push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
    push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
}
use lib (@lib_dirs);

#### globals ####
my $sdffile                 = '';
my $no_sort                 = '';
my $create_dirs             = '';
my $multi_localize_files    = '';
my $module_to_merge         = '';
my $sort_sdf_before         = '';
my $outputfile              = '';
my $no_gsicheck             = '';
my $mode                    = '';
my $bVerbose                = "0";
my $srcpath                 = '';
my $languages;
#my %sl_modules;     # Contains all modules where en-US and de is source language
my $use_default_date = '0';
my $force_ooo_module = '0';
my %is_ooo_module;
my %is_so_module;

         #         (                           leftpart                                                     )            (           rightpart                    )
         #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
# Always use this date to prevent cvs conflicts
my $default_date = "2002-02-02 02:02:02";
my @sdfparticles;

#### main ####
parse_options();

my $binpath = '';
if( defined $ENV{UPDMINOREXT} )
{
    $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin".$ENV{UPDMINOREXT}."/" ;
}
else
{
    $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin/" ;
}

#%sl_modules = fetch_sourcelanguage_dirlist();


if   ( $mode eq "merge"    )    {
    if ( ! $no_gsicheck ){
        merge_gsicheck();
    }
    splitfile( $sdffile );
    if ( ! $no_gsicheck ){
        unlink $sdffile;             # remove temp file!
    }
}
elsif( $mode eq "extract"  )    {
    collectfiles( $outputfile );
}
else                            {
    usage();
}

exit(0);

#########################################################
sub splitfile{

    my $lastFile        = '';
    my $currentFile     = '';
    my $cur_sdffile     = '';
    my $last_sdffile    = '';
    my $delim;
    my $badDelim;
    my $start           = 'TRUE';
    my %index  = ();
    my %block;

    STDOUT->autoflush( 1 );

    #print STDOUT "Open File $sdffile\n";
    open MYFILE , "< $sdffile"
    or die "Can't open '$sdffile'\n";

#    my %lang_hash;
    my %string_hash_ooo;
    my %string_hash_so;
    my %so_modules;
    $so_modules{ "extras_full" } = "TRUE";

    while( <MYFILE>){
         if( /$sdf_regex/ ){
            my $line           = defined $_ ? $_ : '';
            my $prj            = defined $3 ? $3 : '';
            my $file           = defined $4 ? $4 : '';
            my $type           = defined $6 ? $6 : '';
            my $gid            = defined $7 ? $7 : '';
            my $lid            = defined $8 ? $8 : '';
            my $lang           = defined $12 ? $12 : '';
            my $plattform      = defined $10 ? $10 : '';
            my $helpid         = defined $9 ? $9 : '';
            next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
            chomp( $line );

            if( $force_ooo_module )
            {
                $string_hash_ooo { $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
            }
            else
            {
                $string_hash_so{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
            }
        }
    }
    close( MYFILE );

    if( !defined $ENV{SOURCE_ROOT_DIR} ){
        print "Error, no SOURCE_ROOT_DIR in env found.\n";
        exit( -1 );
    }
    my $src_root = $ENV{SOURCE_ROOT_DIR};
    my $ooo_src_root = $src_root."/l10n/l10n";
    my $so_l10n_path  = $src_root."/sun/l10n_so/source";
    my $ooo_l10n_path = $ooo_src_root."/l10n/source";

    #print "$so_l10n_path\n";
    #print "$ooo_l10n_path\n";

    if( $force_ooo_module )
    {
        write_sdf( \%string_hash_ooo , $ooo_l10n_path );
    }
    else
    {
        write_sdf( \%string_hash_so , $so_l10n_path );
    }
}
<commit_msg>test123l10n04: path fixed<commit_after>:
eval 'exec perl -wS $0 ${1+"$@"}'
    if 0;


#*************************************************************************
#
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# Copyright 2000, 2010 Oracle and/or its affiliates.
#
# OpenOffice.org - a multi-platform office productivity suite
#
# This file is part of OpenOffice.org.
#
# OpenOffice.org is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenOffice.org is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenOffice.org.  If not, see
# <http://www.openoffice.org/license.html>
# for a copy of the LGPLv3 License.
#
#*************************************************************************

use strict;
use Getopt::Long;
use IO::Handle;
use File::Find;
use File::Temp;
use File::Path;
use File::Copy;
use File::Glob qw(:glob csh_glob);
use Cwd;

my $CVS_BINARY = "/usr/bin/cvs";
# ver 1.1
#
#### module lookup
#use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");

#### module lookup
# OOo conform
my @lib_dirs;
BEGIN {
    if ( !defined($ENV{SOLARENV}) ) {
        die "No environment found (environment variable SOLARENV is undefined)";
    }
    push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
    push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
}
use lib (@lib_dirs);

#### globals ####
my $sdffile                 = '';
my $no_sort                 = '';
my $create_dirs             = '';
my $multi_localize_files    = '';
my $module_to_merge         = '';
my $sort_sdf_before         = '';
my $outputfile              = '';
my $no_gsicheck             = '';
my $mode                    = '';
my $bVerbose                = "0";
my $srcpath                 = '';
my $languages;
#my %sl_modules;     # Contains all modules where en-US and de is source language
my $use_default_date = '0';
my $force_ooo_module = '0';
my %is_ooo_module;
my %is_so_module;

         #         (                           leftpart                                                     )            (           rightpart                    )
         #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
# Always use this date to prevent cvs conflicts
my $default_date = "2002-02-02 02:02:02";
my @sdfparticles;

#### main ####
parse_options();

my $binpath = '';
if( defined $ENV{UPDMINOREXT} )
{
    $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin".$ENV{UPDMINOREXT}."/" ;
}
else
{
    $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin/" ;
}

#%sl_modules = fetch_sourcelanguage_dirlist();


if   ( $mode eq "merge"    )    {
    if ( ! $no_gsicheck ){
        merge_gsicheck();
    }
    splitfile( $sdffile );
    if ( ! $no_gsicheck ){
        unlink $sdffile;             # remove temp file!
    }
}
elsif( $mode eq "extract"  )    {
    collectfiles( $outputfile );
}
else                            {
    usage();
}

exit(0);

#########################################################
sub splitfile{

    my $lastFile        = '';
    my $currentFile     = '';
    my $cur_sdffile     = '';
    my $last_sdffile    = '';
    my $delim;
    my $badDelim;
    my $start           = 'TRUE';
    my %index  = ();
    my %block;

    STDOUT->autoflush( 1 );

    #print STDOUT "Open File $sdffile\n";
    open MYFILE , "< $sdffile"
    or die "Can't open '$sdffile'\n";

#    my %lang_hash;
    my %string_hash_ooo;
    my %string_hash_so;
    my %so_modules;
    $so_modules{ "extras_full" } = "TRUE";

    while( <MYFILE>){
         if( /$sdf_regex/ ){
            my $line           = defined $_ ? $_ : '';
            my $prj            = defined $3 ? $3 : '';
            my $file           = defined $4 ? $4 : '';
            my $type           = defined $6 ? $6 : '';
            my $gid            = defined $7 ? $7 : '';
            my $lid            = defined $8 ? $8 : '';
            my $lang           = defined $12 ? $12 : '';
            my $plattform      = defined $10 ? $10 : '';
            my $helpid         = defined $9 ? $9 : '';
            next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
            chomp( $line );

            if( $force_ooo_module )
            {
                $string_hash_ooo { $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
            }
            else
            {
                $string_hash_so{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
            }
        }
    }
    close( MYFILE );

    if( !defined $ENV{SOURCE_ROOT_DIR} ){
        print "Error, no SOURCE_ROOT_DIR in env found.\n";
        exit( -1 );
    }
    my $src_root = $ENV{SOURCE_ROOT_DIR};
    my $ooo_src_root = $src_root."/l10n";
    my $so_l10n_path  = $src_root."/sun/l10n_so/source";
    my $ooo_l10n_path = $ooo_src_root."/l10n/source";

    #print "$so_l10n_path\n";
    #print "$ooo_l10n_path\n";

    if( $force_ooo_module )
    {
        write_sdf( \%string_hash_ooo , $ooo_l10n_path );
    }
    else
    {
        write_sdf( \%string_hash_so , $so_l10n_path );
    }
}

{'fraction_non_alphanumeric': 0.18630183261548855, 'fraction_numerical': 0.013427919939194325, 'mean_word_length': 2.176502145922747, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5691365', 'n_tokens_mistral': 4224, 'n_tokens_neox': 4004, 'n_words': 982}