<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

use Carp;
use threads;

use File::Basename;
use File::Spec;
use FindBin;
use Getopt::Long qw(:config no_ignore_case bundling);
use lib("$FindBin::Bin/../../PerlLib");
use Thread_helper;
use Cwd;

my $CPU = 2;

my $usage = <<_EOUSAGE_;

########################################################################################################
#
#  Required:
#
#  --coord_sorted_SAM <string>      coordinate-sorted SAM file.
#
#  -I  <int>                       maximum intron length  
#                                     (reads with longer intron lengths are ignored, and fragment reads 
#                                      farther apart on the genome are treated as unpaired))
#  -C  <int>                       min coverage for region boundary (default: 1)
#                             
#  *If Strand-specific, specify:
#  --SS_lib_type <string>          library type:  if single: F or R,  if paired:  FR or RF
#
#
#  Optional:
#
#  --min_reads_per_partition <int>      default: 10 
#  --parts_per_directory <int>          default: 100
#
#  --sort_buffer <string>               default: '10G'  amount of RAM to allocate to sorting.
#
#  --CPU <int>                        number of threads
#
########################################################################################################


_EOUSAGE_

	;

#  -J  <int>                       region join length (neighboring coverage bins within this range are merged into larger piles)



my $help_flag;

#my $partition_join_size;
my $max_intron_length;
my $SAM_file;
my $SS_lib_type = "";
my $min_coverage = 1;
my $min_reads_per_partition = 10;
my $parts_per_dir = 100;
my $sort_buffer = '10G';

my $SYMLINK = ($ENV{NO_SYMLINK}) ? "cp" : "ln -sf";

&GetOptions ( 'h' => \$help_flag,

			  'coord_sorted_SAM=s' => \$SAM_file,
			  'SS_lib_type=s' => \$SS_lib_type,
			  
			  #'J=i' => \$partition_join_size,
			  'I=i' => \$max_intron_length,
              'C=i' => \$min_coverage,
              
              'min_reads_per_partition=i' => \$min_reads_per_partition,
              'parts_per_directory=i' => \$parts_per_dir,
              'CPU=i' => \$CPU,

              'sort_buffer=s' => \$sort_buffer,
              );


if ($help_flag) {
	die $usage;
}

unless (
	$SAM_file
        # && $partition_join_size
	&& $max_intron_length
		) {
	die $usage;
}

if ($SS_lib_type && $SS_lib_type !~ /^(F|R|FR|RF)$/) {
	die "Error, invalid --SS_lib_type, only F, R, FR, or RF are possible values";
}

my $UTIL_DIR = "$FindBin::RealBin/";

main: {

	my @sam_info;

	if ($SS_lib_type) {
        my $sam_basename = basename($SAM_file);
		my ($plus_strand_sam, $minus_strand_sam) = ("$sam_basename.+.sam", "$sam_basename.-.sam");
		if (-s $plus_strand_sam && $minus_strand_sam) {
			print STDERR "-strand partitioned SAM files already exist, so using them instead of re-creating them.\n";
		}
		else {
			my $cmd = "$UTIL_DIR/SAM_strand_separator.pl $SAM_file $SS_lib_type";
			&process_cmd($cmd);
		}

		push (@sam_info, [$plus_strand_sam, '+'], [$minus_strand_sam, '-']);
	}
	else {
        if (cwd() ne dirname(File::Spec->rel2abs($SAM_file))) {
            &process_cmd("$SYMLINK $SAM_file");
            $SAM_file = basename($SAM_file);
        }
		push (@sam_info, [$SAM_file, '+']);
	}
    

    my $thread_helper = new Thread_helper($CPU);
    
	foreach my $sam_info_aref (@sam_info) {
				
		my ($sam, $strand) = @$sam_info_aref;
				
		my $thread = threads->create('prep_read_partitions', $sam, $strand);
        #push (@threads, $thread);

        $thread_helper->add_thread($thread);
        
	}
    
    $thread_helper->wait_for_all_threads_to_complete();

    my @failures = $thread_helper->get_failed_threads();
    my $ret = 0;
    if (@failures) {
        foreach my $thread (@failures) {
            if (my $error = $thread->error()) {<commit_msg>provide destination for cp cmd<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

use Carp;
use threads;

use File::Basename;
use File::Spec;
use FindBin;
use Getopt::Long qw(:config no_ignore_case bundling);
use lib("$FindBin::Bin/../../PerlLib");
use Thread_helper;
use Cwd;

my $CPU = 2;

my $usage = <<_EOUSAGE_;

########################################################################################################
#
#  Required:
#
#  --coord_sorted_SAM <string>      coordinate-sorted SAM file.
#
#  -I  <int>                       maximum intron length  
#                                     (reads with longer intron lengths are ignored, and fragment reads 
#                                      farther apart on the genome are treated as unpaired))
#  -C  <int>                       min coverage for region boundary (default: 1)
#                             
#  *If Strand-specific, specify:
#  --SS_lib_type <string>          library type:  if single: F or R,  if paired:  FR or RF
#
#
#  Optional:
#
#  --min_reads_per_partition <int>      default: 10 
#  --parts_per_directory <int>          default: 100
#
#  --sort_buffer <string>               default: '10G'  amount of RAM to allocate to sorting.
#
#  --CPU <int>                        number of threads
#
########################################################################################################


_EOUSAGE_

	;

#  -J  <int>                       region join length (neighboring coverage bins within this range are merged into larger piles)



my $help_flag;

#my $partition_join_size;
my $max_intron_length;
my $SAM_file;
my $SS_lib_type = "";
my $min_coverage = 1;
my $min_reads_per_partition = 10;
my $parts_per_dir = 100;
my $sort_buffer = '10G';

my $SYMLINK = ($ENV{NO_SYMLINK}) ? "cp" : "ln -sf";

&GetOptions ( 'h' => \$help_flag,

			  'coord_sorted_SAM=s' => \$SAM_file,
			  'SS_lib_type=s' => \$SS_lib_type,
			  
			  #'J=i' => \$partition_join_size,
			  'I=i' => \$max_intron_length,
              'C=i' => \$min_coverage,
              
              'min_reads_per_partition=i' => \$min_reads_per_partition,
              'parts_per_directory=i' => \$parts_per_dir,
              'CPU=i' => \$CPU,

              'sort_buffer=s' => \$sort_buffer,
              );


if ($help_flag) {
	die $usage;
}

unless (
	$SAM_file
        # && $partition_join_size
	&& $max_intron_length
		) {
	die $usage;
}

if ($SS_lib_type && $SS_lib_type !~ /^(F|R|FR|RF)$/) {
	die "Error, invalid --SS_lib_type, only F, R, FR, or RF are possible values";
}

my $UTIL_DIR = "$FindBin::RealBin/";

main: {

	my @sam_info;

	if ($SS_lib_type) {
        my $sam_basename = basename($SAM_file);
		my ($plus_strand_sam, $minus_strand_sam) = ("$sam_basename.+.sam", "$sam_basename.-.sam");
		if (-s $plus_strand_sam && $minus_strand_sam) {
			print STDERR "-strand partitioned SAM files already exist, so using them instead of re-creating them.\n";
		}
		else {
			my $cmd = "$UTIL_DIR/SAM_strand_separator.pl $SAM_file $SS_lib_type";
			&process_cmd($cmd);
		}

		push (@sam_info, [$plus_strand_sam, '+'], [$minus_strand_sam, '-']);
	}
	else {
        if (cwd() ne dirname(File::Spec->rel2abs($SAM_file))) {
            &process_cmd("$SYMLINK $SAM_file " . basename($SAM_file));
            $SAM_file = basename($SAM_file);
        }
		push (@sam_info, [$SAM_file, '+']);
	}
    

    my $thread_helper = new Thread_helper($CPU);
    
	foreach my $sam_info_aref (@sam_info) {
				
		my ($sam, $strand) = @$sam_info_aref;
				
		my $thread = threads->create('prep_read_partitions', $sam, $strand);
        #push (@threads, $thread);

        $thread_helper->add_thread($thread);
        
	}
    
    $thread_helper->wait_for_all_threads_to_complete();

    my @failures = $thread_helper->get_failed_threads();
    my $ret = 0;
    if (@failures) {
        foreach my $thread (@failures) {
            if (my $error = $thread->error()) {
{'fraction_non_alphanumeric': 0.1918545081967213, 'fraction_numerical': 0.004866803278688525, 'mean_word_length': 2.6713681241184766, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5834105', 'n_tokens_mistral': 2895, 'n_tokens_neox': 2612, 'n_words': 611}