<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

my %files = ();

$files{300}      = shift;
$files{500}      = shift;
$files{800}      = shift;
$files{moleculo} = shift;

my @order = qw(300 500 800 moleculo);
my %flags = ();
for ( my $i = 0 ; $i < @order ; $i++ ) {
    $flags{ $order[$i] } = 2**$i;
}
<commit_msg>walking through the file set<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

my %files = ();

$files{300}      = shift;
$files{500}      = shift;
$files{800}      = shift;
$files{moleculo} = shift;

my @order = qw(300 500 800 moleculo);
my %flags = ();
for ( my $i = 0 ; $i < @order ; $i++ ) {
    $flags{ $order[$i] } = 2**$i;
}

for my $file (@order)
{
    open( my $fh, "<", $files{$file} )
      || die "Unable to open file '" . $files{$file} . "': $!";

    while (<$fh>)
    {
    }

    close( $fh )
      || die "Unable to close file '" . $files{$file} . "': $!";

}

{'fraction_non_alphanumeric': 0.22489082969432314, 'fraction_numerical': 0.043668122270742356, 'mean_word_length': 2.727642276422764, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1888015', 'n_tokens_mistral': 416, 'n_tokens_neox': 371, 'n_words': 96}