<commit_before><commit_msg>subsample a few reads and print the range of quality scores observed<commit_after>#!/usr/bin/perl -w
use strict;
my $i = 0;
my %nums;
while(<>) {
    next unless /^\+/;
    my $line= <>;
    chomp($line);
    while( my $n = substr($line,0,1,'') ) {
	$nums{ord($n)}++;
    }
    last if $i++ > 10_000;
}
for my $n ( sort { $a <=> $b } keys %nums ) {
    printf "%d\t%d\t%s\t%s\n",$n-64,$n,chr($n),$nums{$n};
}
{'fraction_non_alphanumeric': 0.2413793103448276, 'fraction_numerical': 0.022988505747126436, 'mean_word_length': 3.233009708737864, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1348070', 'n_tokens_mistral': 199, 'n_tokens_neox': 187, 'n_words': 45}