<commit_before>use strict;
use warnings;
use utf8;

use Test::Builder::Tester;
use Test::More 0.95;
use Test::File;
use Test::utf8;

# Hello world! I am a string. Russian, courtesy of Google Translate
my $string1 = '<NAME>!';
my $string2 = 'Я строкой';
my $encoding = 'KOI8-R';

require 'setup_common';

my $file = '$file';
open my $fh, '>', $file or print "bail out! Could not write to $file: $!";
$fh->binmode(":encoding($encoding)");
$fh->print("$string1$/$/$/");
$fh->print("$string2$/");
$fh->close;

my $contents = do {
     open $fh, '<', $file;
     $fh->binmode(":encoding($encoding)");
     local $/;
     <$fh>;
};
$fh->close;
<commit_msg>I need the path to t/ to get setup_common<commit_after>use strict;
use warnings;
use utf8;

use Test::Builder::Tester;
use Test::More 0.95;
use Test::File;
use Test::utf8;

# Hello world! I am a string. Russian, courtesy of Google Translate
my $string1 = '<NAME>!';
my $string2 = 'Я строкой';
my $encoding = 'KOI8-R';

require 't/setup_common';

my $file = '$file';
open my $fh, '>', $file or print "bail out! Could not write to $file: $!";
$fh->binmode(":encoding($encoding)");
$fh->print("$string1$/$/$/");
$fh->print("$string2$/");
$fh->close;

my $contents = do {
     open $fh, '<', $file;
     $fh->binmode(":encoding($encoding)");
     local $/;
     <$fh>;
};
$fh->close;

{'fraction_non_alphanumeric': 0.22105263157894736, 'fraction_numerical': 0.015037593984962405, 'mean_word_length': 4.3453815261044175, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5598595', 'n_tokens_mistral': 530, 'n_tokens_neox': 513, 'n_words': 144}