<commit_before>#!/usr/bin/perl
use strict;
use File::Basename;
use File::Temp qw(tempfile cleanup);
use Getopt::Long;
use Pod::Usage;
use FindBin;
use lib "$FindBin::Bin/lib";
require Subfunctions;

if (@ARGV == 0) {
    pod2usage(-verbose => 1);
}

my $short_read_archive = "";
my $output_file = "";
my $numlibraries = 8;
my $help = 0;
my $half = 0;

GetOptions ('input=s' => \$short_read_archive,
            'output=s' => \$output_file,
            'number=i' => \$numlibraries,
            'half' => \$half,
            'help|?' => \$help) or pod2usage(-msg => "GetOptions failed.", -exitval => 2);

if ($help) {
    pod2usage(-verbose => 1);
}

unless($short_read_archive) {
    pod2usage(-msg => "Must specify a short read archive in fasta or fastq form.");
}

unless ($output_file) {
    $output_file = $short_read_archive;
}

my $tempdir = dirname ("$output_file");
my @tempfiles = ();

my @out1_fhs = ();
my @out2_fhs = ();
my @out1_bucketfiles = ();
my @out2_bucketfiles = ();
my @out1_sortedfiles = ();
my @out2_sortedfiles = ();

# setting up tempfiles for sorting:
for (my $i=0; $i<$numlibraries; $i++) {
	my ($fh, $filename) = tempfile("$output_file.bucket.$i.1.XXXX", UNLINK => 1);
	push @out1_bucketfiles, $filename;
	push @out1_fhs, $fh;

	($fh, $filename) = tempfile("$output_file.bucket.$i.2.XXXX", UNLINK => 1);
	push @out2_bucketfiles, $filename;
	push @out2_fhs, $fh;

	($fh, $filename) = tempfile("$output_file.sorted.$i.1.XXXX", UNLINK => 1);
	push @out1_sortedfiles, $filename;

	if ($half) {
		$filename = "$output_file.$i.2.sorted";
	} else {
		($fh, $filename) = tempfile("$output_file.sorted.$i.2.XXXX", UNLINK => 1);
	}
	push @out2_sortedfiles, $filename;
}

# Divide fasta/fastq short reads into buckets for sorting.
print "" . timestamp() . ": Dividing fasta/fastq file into buckets for sorting.\n";
my $name = "";
my $seq = "";
my $seqlen = 0;

open SEARCH_FH, "<", $short_read_archive;
while (my $line = readline SEARCH_FH) {
	chomp $line;
	if ($line =~ /^[@>](.*?)([\s\/])([12])/) {
		if ($name ne "") {
			if ($name =~ /\/1/) {
				print {$out1_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
			} elsif ($name =~ /\/2/) {
				print {$out2_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
			}
		}
		$name = "$1\/$3";
		$seq = "";
	} elsif ($line =~ /^\+/){
		# is this a fastq quality line? eat chars to the length of the full sequence.
		while ($seqlen > 0) {
			$line = readline SEARCH_FH;
			chomp $line;
			$seqlen = $seqlen - length($line);
		}
	} else {
		$seq .= $line;
		$seqlen = length ($seq);
	}
}

if ($name =~ /\/1/) {
	print {$out1_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
} elsif ($name =~ /\/2/) {
	print {$out2_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
}
close SEARCH_FH;
my @pids = ();
print "" . timestamp() . ": starting sort.\n";

for (my $i=0; $i<$numlibraries; $i++) {
	close $out1_fhs[$i];
	close $out2_fhs[$i];
	if ($half == 0) {
		push @pids, fork_cmd("sort -t',' -k 1 -T $tempdir @out1_bucketfiles[$i] > @out1_sortedfiles[$i]");
	}
}
wait_for_forks(\@pids);

for (my $i=0; $i<$numlibraries; $i++) {
    push @pids, fork_cmd("sort -t',' -k 1 -T $tempdir @out2_bucketfiles[$i] > @out2_sortedfiles[$i]");
}
wait_for_forks(\@pids);
print "" . timestamp() . ": sorted.\n";

for (my $i=0; $i<$numlibraries; $i++) {
	print "" . timestamp() . ": Making $output_file.$i.1.fasta.\n";
	my $infile;
	if ($half == 0) {
		$infile = "@out1_sortedfiles[$i]";
	} else {
		$infile = "@out1_bucketfiles[$i]";
		push @tempfiles, "$output_file.$i.1.fasta";
	}
	open my $in_fh, "<", $infile or exit_with_msg ("couldn't read $infile");
	open my $out_fh, ">", "$output_file.$i.1.fasta" or exit_with_msg ("couldn't create $output_file.$i.1.fasta");
	while (my $line = readline $in_fh) {
		chomp $line;
		my ($name, $seq) = split(/,/,$line);
		print $out_fh "$name\n$seq\n";
	}
	close $in_fh;
	close $out_fh;

	if ($half == 0) {
		print "" . timestamp() . ": Making $output_file.$i.2.fasta.\n";
		open my $in_fh, "<", "@out2_sortedfiles[$i]" or exit_with_msg ("couldn't read @out2_sortedfiles[$i]");
		open my $out_fh, ">", "$output_file.$i.2.fasta" or exit_with_msg ("couldn't create $output_file.$i.2.fasta");
		while (my $line = readline $in_fh) {<commit_msg>Sorts are hugely memory-intensive: just run four at once.<commit_after>#!/usr/bin/perl
use strict;
use File::Basename;
use File::Temp qw(tempfile cleanup);
use Getopt::Long;
use Pod::Usage;
use FindBin;
use lib "$FindBin::Bin/lib";
require Subfunctions;

if (@ARGV == 0) {
    pod2usage(-verbose => 1);
}

my $short_read_archive = "";
my $output_file = "";
my $numlibraries = 8;
my $help = 0;
my $half = 0;

GetOptions ('input=s' => \$short_read_archive,
            'output=s' => \$output_file,
            'number=i' => \$numlibraries,
            'half' => \$half,
            'help|?' => \$help) or pod2usage(-msg => "GetOptions failed.", -exitval => 2);

if ($help) {
    pod2usage(-verbose => 1);
}

unless($short_read_archive) {
    pod2usage(-msg => "Must specify a short read archive in fasta or fastq form.");
}

unless ($output_file) {
    $output_file = $short_read_archive;
}

my $tempdir = dirname ("$output_file");
my @tempfiles = ();

my @out1_fhs = ();
my @out2_fhs = ();
my @out1_bucketfiles = ();
my @out2_bucketfiles = ();
my @out1_sortedfiles = ();
my @out2_sortedfiles = ();

# setting up tempfiles for sorting:
for (my $i=0; $i<$numlibraries; $i++) {
	my ($fh, $filename) = tempfile("$output_file.bucket.$i.1.XXXX", UNLINK => 1);
	push @out1_bucketfiles, $filename;
	push @out1_fhs, $fh;

	($fh, $filename) = tempfile("$output_file.bucket.$i.2.XXXX", UNLINK => 1);
	push @out2_bucketfiles, $filename;
	push @out2_fhs, $fh;

	($fh, $filename) = tempfile("$output_file.sorted.$i.1.XXXX", UNLINK => 1);
	push @out1_sortedfiles, $filename;

	if ($half) {
		$filename = "$output_file.$i.2.sorted";
	} else {
		($fh, $filename) = tempfile("$output_file.sorted.$i.2.XXXX", UNLINK => 1);
	}
	push @out2_sortedfiles, $filename;
}

# Divide fasta/fastq short reads into buckets for sorting.
print "" . timestamp() . ": Dividing fasta/fastq file into buckets for sorting.\n";
my $name = "";
my $seq = "";
my $seqlen = 0;

open SEARCH_FH, "<", $short_read_archive;
while (my $line = readline SEARCH_FH) {
	chomp $line;
	if ($line =~ /^[@>](.*?)([\s\/])([12])/) {
		if ($name ne "") {
			if ($name =~ /\/1/) {
				print {$out1_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
			} elsif ($name =~ /\/2/) {
				print {$out2_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
			}
		}
		$name = "$1\/$3";
		$seq = "";
	} elsif ($line =~ /^\+/){
		# is this a fastq quality line? eat chars to the length of the full sequence.
		while ($seqlen > 0) {
			$line = readline SEARCH_FH;
			chomp $line;
			$seqlen = $seqlen - length($line);
		}
	} else {
		$seq .= $line;
		$seqlen = length ($seq);
	}
}

if ($name =~ /\/1/) {
	print {$out1_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
} elsif ($name =~ /\/2/) {
	print {$out2_fhs[(hash_to_bucket($name))]} ">$name,$seq\n";
}
close SEARCH_FH;
my @pids = ();
print "" . timestamp() . ": starting sort.\n";

for (my $i=0; $i<$numlibraries; $i++) {
	close $out1_fhs[$i];
	close $out2_fhs[$i];
	if ($half == 0) {
		push @pids, fork_cmd("sort -t',' -k 1 -T $tempdir @out1_bucketfiles[$i] > @out1_sortedfiles[$i]");
	}
	if (@pids > 3) {
		# don't spawn off too many threads at once.
		wait_for_forks(\@pids);
	}
}
wait_for_forks(\@pids);

for (my $i=0; $i<$numlibraries; $i++) {
    push @pids, fork_cmd("sort -t',' -k 1 -T $tempdir @out2_bucketfiles[$i] > @out2_sortedfiles[$i]");
	if (@pids > 3) {
		# don't spawn off too many threads at once.
		wait_for_forks(\@pids);
	}
}
wait_for_forks(\@pids);
print "" . timestamp() . ": sorted.\n";

for (my $i=0; $i<$numlibraries; $i++) {
	print "" . timestamp() . ": Making $output_file.$i.1.fasta.\n";
	my $infile;
	if ($half == 0) {
		$infile = "@out1_sortedfiles[$i]";
	} else {
		$infile = "@out1_bucketfiles[$i]";
		push @tempfiles, "$output_file.$i.1.fasta";
	}
	open my $in_fh, "<", $infile or exit_with_msg ("couldn't read $infile");
	open my $out_fh, ">", "$output_file.$i.1.fasta" or exit_with_msg ("couldn't create $output_file.$i.1.fasta");
	while (my $line = readline $in_fh) {
		chomp $line;
		my ($name, $seq) = split(/,/,$line);
		print $out_fh "$name\n$seq\n";
	}
	close $in_fh;
	close $out_fh;

	if ($half == 0) {
		print "" . timestamp() . ": Making $output_file.$i.2.fasta.\n";
		open my $in_fh, "<", "@out2_sortedfiles[$i]" or exit_with_msg ("couldn't read @out2_sortedfiles[$i]");
		open my $out_fh, ">", "$output_file.$i.2.fasta" or exit_with_msg ("couldn't create $output_file.$i.2.fasta");
		while (my $line = readline $in_fh) {
{'fraction_non_alphanumeric': 0.2205644228544992, 'fraction_numerical': 0.01688642146657414, 'mean_word_length': 3.877044557247603, 'pattern_counts': {'":': 10, '<': 17, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '55528', 'n_tokens_mistral': 3757, 'n_tokens_neox': 3534, 'n_words': 806}