<commit_before>#!/usr/bin/perl
use warnings;
use strict;
$|++;

# Test all tables with lou_checktable.
#
# Copyright (C) 2010 by Swiss Library for the Blind, Visually Impaired and Print Disabled
#
# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.

my $fail = 0;
# some tables are quite big and take some time to check, so keep the timeout reasonably long
my $timeout = 20; # seconds
my $tablesdir = $ENV{LOUIS_TABLEPATH};

# get all the tables from the tables directory
my @tables = glob("$tablesdir/*.[cu]tb $tablesdir/*.cti $tablesdir/*.dis");
# filter tables that only work when included inside others
@tables = grep(!/countries.cti|compress.ctb|corrections.ctb|core.[cu]tb/, @tables);

foreach my $table (@tables) {
    eval {
	local $SIG{ALRM} = sub { print STDERR "lou_checktable on $table stuck in endless loop\n"; die };
	alarm $timeout;
	unless (system("lou_checktable $table 2> /dev/null") == 0) {
	    print STDERR "lou_checktable on $table failed\n";
	    die;
	}
	alarm 0;
    };
    $fail = 1 if ($@);
}

exit $fail;<commit_msg>Increase the timeout so lou_checktable doesn't time out on large tables on slow architectures.<commit_after>#!/usr/bin/perl
use warnings;
use strict;
$|++;

# Test all tables with lou_checktable.
#
# Copyright (C) 2010 by Swiss Library for the Blind, Visually Impaired and Print Disabled
#
# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.

my $fail = 0;
# some tables are quite big and take some time to check, so keep the timeout reasonably long
my $timeout = 120; # seconds
my $tablesdir = $ENV{LOUIS_TABLEPATH};

# get all the tables from the tables directory
my @tables = glob("$tablesdir/*.[cu]tb $tablesdir/*.cti $tablesdir/*.dis");
# filter tables that only work when included inside others
@tables = grep(!/countries.cti|compress.ctb|corrections.ctb|core.[cu]tb/, @tables);

foreach my $table (@tables) {
    eval {
	local $SIG{ALRM} = sub { print STDERR "lou_checktable on $table stuck in endless loop\n"; die };
	alarm $timeout;
	unless (system("lou_checktable $table 2> /dev/null") == 0) {
	    print STDERR "lou_checktable on $table failed\n";
	    die;
	}
	alarm 0;
    };
    $fail = 1 if ($@);
}

exit $fail;
{'fraction_non_alphanumeric': 0.11541632316570487, 'fraction_numerical': 0.009480626545754328, 'mean_word_length': 4.264642082429501, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6480110', 'n_tokens_mistral': 803, 'n_tokens_neox': 757, 'n_words': 296}