<commit_before>package LogReporter::Util;
use strict;
use warnings;
use feature ':5.10';

use Exporter 'import';
our @EXPORT = (qw());
our @EXPORT_OK = (qw(canonical_ipv6_address SortIP unitize));

sub canonical_ipv6_address {
    my @a = split /:/, shift;
    my @b = qw(0 0 0 0 0 0 0 0);
    my $i = 0;
    # comparison is numeric, so we use hex function
    while (defined $a[0] and $a[0] ne '') {$b[$i++] = hex(shift @a);}
    @a = reverse @a;
    $i = 7;
    while (defined $a[0] and $a[0] ne '') {$b[$i--] = hex(shift @a);}
    @b;
}

sub SortIP {
    # $a & $b are in the caller's namespace.
    my $package = (caller)[0];
    no strict 'refs'; # Back off, man. I'm a scientist.
    my $A = $ {"${package}::a"};
    my $B = $ {"${package}::b"};
    $A =~ s/^::(ffff:)?(\d+\.\d+\.\d+\.\d+)$/$2/;
    $B =~ s/^::(ffff:)?(\d+\.\d+\.\d+\.\d+)$/$2/;
    use strict 'refs'; # We are a hedge. Please move along.
    if ($A =~ /:/ and $B =~ /:/) {
        my @a = canonical_ipv6_address($A);
        my @b = canonical_ipv6_address($B);
        while ($a[1] and $a[0] == $b[0]) {shift @a; shift @b;}
        $a[0] <=> $b[0];
    } elsif ($A =~ /:/) {
        -1;
    } elsif ($B =~ /:/) {
        1;
    } else {
        my ($a1, $a2, $a3, $a4) = split /\./, $A;
        my ($b1, $b2, $b3, $b4) = split /\./, $B;
        $a1 <=> $b1 || $a2 <=> $b2 || $a3 <=> $b3 || $a4 <=> $b4;
    }
}

sub unitize {
    my ($num) = @_;
    my $kilobyte = 1024;
    my $megabyte = 1048576;
    my $gigabyte = 1073741824;
    my $terabyte = 1099511627776;

    return sprintf "%.3f TB", ($num / $terabyte)  if ($num >= $terabyte);
    return sprintf "%.3f GB", ($num / $gigabyte)  if ($num >= $gigabyte);
    return sprintf "%.3f MB", ($num / $megabyte)  if ($num >= $megabyte);
    return sprintf "%.3f KB", ($num / $kilobyte)  if ($num >= $kilobyte);
    return sprintf "%.3f  B", ($num);
}

1;<commit_msg>Rearranged utils, and added a LookupIP function. It uses gethostbyaddr which is more? portable? than Net::DNS.<commit_after>package LogReporter::Util;
use strict;
use warnings;
use feature ':5.10';

use Exporter 'import';
our @EXPORT = (qw());
our @EXPORT_OK = (qw(unitize SortIP LookupIP));

sub unitize {
    my ($num) = @_;
    my $kilobyte = 1024;
    my $megabyte = 1048576;
    my $gigabyte = 1073741824;
    my $terabyte = 1099511627776;

    return sprintf "%.3f TB", ($num / $terabyte)  if ($num >= $terabyte);
    return sprintf "%.3f GB", ($num / $gigabyte)  if ($num >= $gigabyte);
    return sprintf "%.3f MB", ($num / $megabyte)  if ($num >= $megabyte);
    return sprintf "%.3f KB", ($num / $kilobyte)  if ($num >= $kilobyte);
    return sprintf "%.3f  B", ($num);
}

sub canonical_ipv6_address {
    my @a = split /:/, shift;
    my @b = qw(0 0 0 0 0 0 0 0);
    my $i = 0;
    # comparison is numeric, so we use hex function
    while (defined $a[0] and $a[0] ne '') {$b[$i++] = hex(shift @a);}
    @a = reverse @a;
    $i = 7;
    while (defined $a[0] and $a[0] ne '') {$b[$i--] = hex(shift @a);}
    @b;
}

sub SortIP {
    # $a & $b are in the caller's namespace.
    my $package = (caller)[0];
    no strict 'refs'; # Back off, man. I'm a scientist.
    my $A = $ {"${package}::a"};
    my $B = $ {"${package}::b"};
    $A =~ s/^::(ffff:)?(\d+\.\d+\.\d+\.\d+)$/$2/;
    $B =~ s/^::(ffff:)?(\d+\.\d+\.\d+\.\d+)$/$2/;
    use strict 'refs'; # We are a hedge. Please move along.
    if ($A =~ /:/ and $B =~ /:/) {
        my @a = canonical_ipv6_address($A);
        my @b = canonical_ipv6_address($B);
        while ($a[1] and $a[0] == $b[0]) {shift @a; shift @b;}
        $a[0] <=> $b[0];
    } elsif ($A =~ /:/) {
        -1;
    } elsif ($B =~ /:/) {
        1;
    } else {
        my ($a1, $a2, $a3, $a4) = split /\./, $A;
        my ($b1, $b2, $b3, $b4) = split /\./, $B;
        $a1 <=> $b1 || $a2 <=> $b2 || $a3 <=> $b3 || $a4 <=> $b4;
    }
}

my %LookupCache = ();
sub LookupIP {
    my ($Addr) = @_;
    return $LookupCache{$Addr} if exists $LookupCache{$Addr};
    
    $Addr =~ s/^::ffff://;
    
    my ($name);
    if ($Addr =~ /^[\d\.]*$/) {
        my $PackedAddr = pack('C4', split /\./,$Addr);
        $name = gethostbyaddr($PackedAddr,AF_INET());
    } elsif ($Addr =~ /^[0-9a-zA-Z:]*/) {
        my $PackedAddr = pack('n8', canonical_ipv6_address($Addr));
        $name = gethostbyaddr($PackedAddr, AF_INET6());
    }
    
    if ($name) {
        my $val = "$Addr ($name)";
        $LookupCache{$Addr} = $val;
        return $val;
    } else {
        $LookupCache{$Addr} = $Addr;
        return ($Addr);
    }
}
{'fraction_non_alphanumeric': 0.23866138265081463, 'fraction_numerical': 0.039189784236019376, 'mean_word_length': 2.318480642804967, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2847402', 'n_tokens_mistral': 1994, 'n_tokens_neox': 1903, 'n_words': 503}