<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

my %files = ();

$files{300}      = shift;
$files{500}      = shift;
$files{800}      = shift;
$files{moleculo} = shift;

my @order = qw(300 500 800 moleculo);

foreach my $file (@order) {
    my $name = $files{$file};
    $files{$file} = { name => $name };
    $files{$file}{line} = undef;
    open( my $fh, "<", $files{$file}{name} )
      || die "Unable to open file '" . $files{$file}{name} . "': $!";
    $files{$file}{fh} = $fh;
}

while ( grep { !eof( $files{$_}{fh} ) } (@order) ) {

    # which file I have to read?
    foreach my $file (@order) {

       # read the next line from the file if not eof and the line value is undef
        unless ( defined $files{$file}{line} && eof( $files{$file}{fh} ) ) {
            my $fh = $files{$file}{fh};
            $files{$file}{line} = <$fh>;
            chomp( $files{$file}{line} );
            ( $files{$file}{kmer}, $files{$file}{count} ) =
              split( /\t/, $files{$file}{line} );
        }
    }

    # find the next (alphabetically first) kmer
    my ($next_kmer) = sort map { $files{$_}{kmer} } (@order);

    # extract the information from each input file
    my @values = ();
    foreach my $file (@order) {
        my $val = 0;
        if ( $files{$file}{kmer} eq $next_kmer ) {
            $val = $files{$file}{count};
            $files{$file}{line} = undef;
        }
        push( @values, $val );
    }

    print join( "\t", ( $next_kmer, @values ) ), "\n";

}

foreach my $file (@order) {
    close( $files{$file}{fh} )
      || die "Unable to close file '" . $files{$file}{name} . "': $!";
}<commit_msg>Added a flag for the kmer<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

my %files = ();

$files{300}      = shift;
$files{500}      = shift;
$files{800}      = shift;
$files{moleculo} = shift;

my @order = qw(300 500 800 moleculo);
my %flags = ();
for(my $i=0; $i<@order; $i++)
{
    $flags{$order[$i]} = 2**$i;
}

foreach my $file (@order) {
    my $name = $files{$file};
    $files{$file} = { name => $name };
    $files{$file}{line} = undef;
    open( my $fh, "<", $files{$file}{name} )
      || die "Unable to open file '" . $files{$file}{name} . "': $!";
    $files{$file}{fh} = $fh;
}

while ( grep { !eof( $files{$_}{fh} ) } (@order) ) {

    # which file I have to read?
    foreach my $file (@order) {

       # read the next line from the file if not eof and the line value is undef
        unless ( defined $files{$file}{line} && eof( $files{$file}{fh} ) ) {
            my $fh = $files{$file}{fh};
            $files{$file}{line} = <$fh>;
            chomp( $files{$file}{line} );
            ( $files{$file}{kmer}, $files{$file}{count} ) =
              split( /\t/, $files{$file}{line} );
        }
    }

    # find the next (alphabetically first) kmer
    my ($next_kmer) = sort map { $files{$_}{kmer} } (@order);

    # extract the information from each input file
    my @values = ();
    my $flag = 0;
    foreach my $file (@order) {
        my $val = 0;
        if ( $files{$file}{kmer} eq $next_kmer ) {
            $val = $files{$file}{count};
            $files{$file}{line} = undef;
	    $flag |= $flags{$file};
        }
        push( @values, $val );
    }

    print join( "\t", ( $next_kmer, @values, $flag ) ), "\n";

}

foreach my $file (@order) {
    close( $files{$file}{fh} )
      || die "Unable to close file '" . $files{$file}{name} . "': $!";
}
{'fraction_non_alphanumeric': 0.20905923344947736, 'fraction_numerical': 0.011904761904761904, 'mean_word_length': 2.4176587301587302, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3010165', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1213, 'n_words': 321}