<commit_before>#!/usr/bin/env perl

use v5.14; # 'say'
# need to read with utf8 same as Perl 6
use open IO => ':endoding(UTF-8)';

my $debug = 0;

if (!@ARGV) {
  print <<"HERE";
Usage: $0 <input file>
HERE

  exit;
}

my $ifil = shift @ARGV;

die "FATAL: file '$ifil' not found.\n"
    if !-f $ifil;

my $fsiz = -s $ifil;

say "  File '$ifil' size: $fsiz bytes" if $debug;

open my $fp, '<', $ifil
    or die "file: '$ifil': $!";

my $nlines = 0;
my $nchars = 0;
while (defined(my $line = <$fp>)) {
  chomp $line;
  ++$nlines;<commit_msg>use a more correct syntax<commit_after>#!/usr/bin/env perl

use v5.14; # 'say'

my $debug = 0;

if (!@ARGV) {
  print <<"HERE";
Usage: $0 <input file>
HERE

  exit;
}

my $ifil = shift @ARGV;

die "FATAL: file '$ifil' not found.\n"
    if !-f $ifil;

my $fsiz = -s $ifil;

say "  File '$ifil' size: $fsiz bytes" if $debug;

# need to read with utf8 same as Perl 6
open my $fp, '<:encoding(UTF-8)', $ifil
    or die "file: '$ifil': $!";

my $nlines = 0;
my $nchars = 0;
while (defined(my $line = <$fp>)) {
  chomp $line;
  ++$nlines;
{'fraction_non_alphanumeric': 0.20261437908496732, 'fraction_numerical': 0.018674136321195144, 'mean_word_length': 3.220472440944882, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6508435', 'n_tokens_mistral': 498, 'n_tokens_neox': 483, 'n_words': 154}