<commit_before>#!/usr/bin/perl

use strict;
use warnings;
use utf8;
use Getopt::Long;
use List::Util qw(sum min max shuffle);
binmode STDIN, ":utf8";
binmode STDOUT, ":utf8";
binmode STDERR, ":utf8";

my $LANG = "en";
GetOptions(
"lang=s" => \$LANG,
);

if(@ARGV < 1) {
    print STDERR "Usage: $0 PREFIX [INTERLEAVE]\n";
    exit 1;
}
my $inter = $ARGV[1] ? $ARGV[1] : 1;

my @vals = qw(0 1 2 3 4 5 6 7 8 9 A B C D E F);
sub tohex {
    my $v = shift;
    $v = min(240, $v*500);
    my $str = $vals[int($v/16)].$vals[int($v)%16];
    return "$str$str$str";
}

my $id = 0;
my ($src, $trg, @context);
while(<STDIN>) {
    if(/^ll/) {
        foreach my $fid (0 .. $inter-1) {
            open FILE0, ">:utf8", "$ARGV[0]$id-$fid.html" or die "Couldn't open $ARGV[0]$id-$fid.html\n";
            print FILE0 "<html><body><table>";
            my @sa = split(/ +/, $src); push @sa, "&lt;s&gt;";
            print FILE0 "<tr><td>&nbsp;</td>"; for(@sa) { print FILE0 "<td>$_</td>"; } print FILE0 "</tr>";
            my @ta = split(/ +/, $trg); push @ta, "&lt;s&gt;";
            foreach my $i (0 .. $#ta) {
                print FILE0 "<tr><td>$ta[$i]</td>";
                for(split(/,/, $context[$i*$inter+$fid])) { print FILE0 "<td bgcolor=\"".tohex($_)."\">&nbsp;</td>"; }
                print FILE0 "<tr>";
            }
            print FILE0 "</table></body></html>";
            close FILE0;
        }
        @context = ();
        $id++;
    } elsif(/SentLL trg: (.*)/) {
        $trg = $1;
    } elsif(/SentLL src: (.*)/) {
        $src = $1;
    } elsif(/Alignments: \[(.*)\]/) {
        push @context, $1;
    }
}<commit_msg>Made attention printing script support ensembles<commit_after>#!/usr/bin/perl

use strict;
use warnings;
use utf8;
use Getopt::Long;
use List::Util qw(sum min max shuffle);
binmode STDIN, ":utf8";
binmode STDOUT, ":utf8";
binmode STDERR, ":utf8";

my $LANG = "en";
GetOptions(
"lang=s" => \$LANG,
);

if(@ARGV != 1) {
    print STDERR "Usage: $0 PREFIX\n";
    exit 1;
}

my @vals = qw(0 1 2 3 4 5 6 7 8 9 A B C D E F);
sub tohex {
    my $v = shift;
    $v = min(240, $v*500);
    my $str = $vals[int($v/16)].$vals[int($v)%16];
    return "$str$str$str";
}

sub create_context {
  my @ret;
  for(@_) {
    my @arr = split(/,/, $_);
    foreach my $i (0 .. $#arr) {
      $ret[$i] += $arr[$i];
    }
  }
  return map { $_/@_ } @ret;
}

my $id = 0;
my ($src, $trg, @context);
while(<STDIN>) {
    if(/^ll/) {
        open FILE0, ">:utf8", "$ARGV[0]$id.html" or die "Couldn't open $ARGV[0]$id.html\n";
        print FILE0 "<html><body><table>";
        my @sa = split(/ +/, $src); push @sa, "&lt;s&gt;";
        print FILE0 "<tr><td>&nbsp;</td>"; for(@sa) { print FILE0 "<td>$_</td>"; } print FILE0 "</tr>";
        my @ta = split(/ +/, $trg); push @ta, "&lt;s&gt;";
        if(@context % @ta != 0) { die "Context not divisible by ta: context=".scalar(@context).", ta=".scalar(@ta)."\n"; }
        my $num_ens = @context / @ta;
        foreach my $i (0 .. $#ta) {
            print FILE0 "<tr><td>$ta[$i]</td>";
            my @align = create_context($context[$i*$num_ens,($i+1)*$num_ens-1]);
            for(@align) { print FILE0 "<td bgcolor=\"".tohex($_)."\">&nbsp;</td>"; }
            print FILE0 "<tr>";
        }
        print FILE0 "</table></body></html>";
        close FILE0;
        @context = ();
        $id++;
    } elsif(/SentLL trg: (.*)/) {
        $trg = $1;
    } elsif(/SentLL src: (.*)/) {
        $src = $1;
    } elsif(/Alignments: \[(.*)\]/) {
        push @context, $1;
    }
}
{'fraction_non_alphanumeric': 0.242013005371784, 'fraction_numerical': 0.027989821882951654, 'mean_word_length': 2.2942271880819365, 'pattern_counts': {'":': 6, '<': 42, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2818873', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1396, 'n_words': 353}