<commit_before>#!/usr/bin/env perl

use 5.014;
use strict;
use warnings;
use autodie qw(open);
use Data::Dump qw(dd);
use JSON;
use List::Util qw(sum);

my $usage = "$0 idlist vmatches\n";
my $infile = shift or die $usage;
my $vmatches = shift or die $usage;
open my $in, '<', $infile;
open my $vm, '<', $vmatches;

my %family_map;

while (my $line = <$in>) {
    chomp $line;
    my ($f, $sf, $source)  = split /\t/, $line;
    next unless defined $sf && defined $f; ## why?
    if ($sf =~ /(\s+)/) {
	$sf =~ s/$1/\_/;
    }
    $f =~ s/\s/\_/;
    if (exists $family_map{$sf}) {
	push @{$family_map{$sf}}, {$f => 0};
    }
    else {
	$family_map{$sf} = [];
    }
}
close $in;
<commit_msg>add desc; lower version req<commit_after>#!/usr/bin/env perl

## Take the output from Vmatch and generate a file of counts by repeat type (superfamily)

use 5.010;
use strict;
use warnings;
use autodie qw(open);
use Data::Dump qw(dd);
use JSON;
use List::Util qw(sum);

my $usage    = "$0 idlist vmatches\n";
my $infile   = shift or die $usage;
my $vmatches = shift or die $usage;
open my $in, '<', $infile;
open my $vm, '<', $vmatches;

my %family_map;

while (my $line = <$in>) {
    chomp $line;
    my ($f, $sf, $source)  = split /\t/, $line;
    next unless defined $sf && defined $f; ## why?
    if ($sf =~ /(\s+)/) {
	$sf =~ s/$1/\_/;
    }
    $f =~ s/\s/\_/;
    if (exists $family_map{$sf}) {
	push @{$family_map{$sf}}, {$f => 0};
    }
    else {
	$family_map{$sf} = [];
    }
}
close $in;

{'fraction_non_alphanumeric': 0.21380697050938338, 'fraction_numerical': 0.00938337801608579, 'mean_word_length': 2.9602122015915118, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1874805', 'n_tokens_mistral': 632, 'n_tokens_neox': 619, 'n_words': 187}