<commit_before>#!/usr/bin/env perl

##TODO: don't store sequences

use 5.010;
use strict;
use warnings;
use autodie;
use Getopt::Long;
use Time::HiRes qw(gettimeofday);

my $infile;
my $outfile;
my $format; 
my $usage = "$0 -i in -o sorted [-f fastq]\n";

GetOptions(
	   'i|infile=s'   => \$infile,
	   'o|outfile=s'  => \$outfile,
           'f|format=s'   => \$format,
	   );

if (!$infile or !$outfile) {
    say $usage,"\nERROR: No input was given. Exiting.";
    exit(1);
}

$format //= 'fasta';

open my $in, '<', $infile;
open my $out, '>', $outfile;

my @aux = undef;
my ($name, $comm, $seq, $qual);
my %seqhash;
my $seqct = 0;
my $t0 = gettimeofday();

while (($name, $comm, $seq, $qual) = readfq(\*$in, \@aux)) {
    my $rec;
    if ($format =~ /fastq/i) {
	$rec = join "||", $seq, $qual;
    }
    else {
	$rec = $seq;
    }
    $seqhash{$name} = $rec;
    $seqct++;
}

for my $k (sort keys %seqhash) {
    if ($format =~ /fastq/i) {
	my ($seq, $qual) = split /\|\|/, $seqhash{$k};<commit_msg>correctly sort any set of ids now<commit_after>#!/usr/bin/env perl

##TODO: don't store sequences

use 5.010;
use strict;
use warnings;
use autodie;
use Getopt::Long;
use Sort::Naturally qw(nsort);
use Time::HiRes     qw(gettimeofday);

my $infile;
my $outfile;
my $format; 
my $usage = "$0 -i in -o sorted [-f fastq]\n";

GetOptions(
	   'i|infile=s'   => \$infile,
	   'o|outfile=s'  => \$outfile,
           'f|format=s'   => \$format,
	   );

if (!$infile or !$outfile) {
    say $usage,"\nERROR: No input was given. Exiting.";
    exit(1);
}

$format //= 'fasta';

open my $in, '<', $infile;
open my $out, '>', $outfile;

my @aux = undef;
my ($name, $comm, $seq, $qual);
my %seqhash;
my $seqct = 0;
my $t0 = gettimeofday();

while (($name, $comm, $seq, $qual) = readfq(\*$in, \@aux)) {
    my $rec;
    if ($format =~ /fastq/i) {
	$rec = join "||", $seq, $qual;
    }
    else {
	$rec = $seq;
    }
    $seqhash{$name} = $rec;
    $seqct++;
}

for my $k (nsort keys %seqhash) {
    if ($format =~ /fastq/i) {
	my ($seq, $qual) = split /\|\|/, $seqhash{$k};
{'fraction_non_alphanumeric': 0.2260406582768635, 'fraction_numerical': 0.007744433688286544, 'mean_word_length': 2.9903474903474905, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '219707', 'n_tokens_mistral': 921, 'n_tokens_neox': 895, 'n_words': 238}