<commit_before>#!/usr/bin/perl -w

package illumina_mapping;

use strict;
use POSIX qw(tmpnam);
use lib "$FindBin::Bin";
use illumina_sge;
use illumina_template;

sub runMapping {
    my $configuration = shift;
    my %opt = %{$configuration};
    my $runName = (split("/", $opt{OUTPUT_DIR}))[-1];
    
    my $FAI = "$opt{GENOME}\.fai";
    die "GENOME: $opt{GENOME} does not exists!\n" if !-e "$opt{GENOME}";
    die "GENOME BWT: $opt{GENOME}.bwt does not exists!\n" if !-e "$opt{GENOME}.bwt";
    die "GENOME FAI: $FAI does not exists!\n" if !-e $FAI;

    my $mainJobID = "$opt{OUTPUT_DIR}/jobs/MapMainJob_".get_job_id().".sh";

    open (my $QSUB,">$mainJobID") or die "ERROR: Couldn't create $mainJobID\n";
    print $QSUB "\#!/bin/sh\n\n. $opt{CLUSTER_PATH}/settings.sh\n\n";

    my $samples = {};
    my $toMap = {};

    ### Try to search for matching pairs in the input FASTQ files
    foreach my $input (keys %{$opt{FASTQ}}) {
		if($input =~ m/\_R1/) {
			my $pairName = $input;
			$pairName =~ s/\_R1/\_R2/;
			if(exists ($opt{FASTQ}->{$pairName})) {
				$toMap->{$input."#".$pairName} = 1;
			} else {
				$toMap->{$input} = 1;
			}
		} elsif ($input =~ m/\_R2/) {
			my $pairName = $input;
			$pairName =~ s/\_R2/\_R1/;
			if (exists ($opt{FASTQ}->{$pairName})) {
				$toMap->{$pairName."#".$input} = 1;
			} else {
				$toMap->{$input} = 1;
			}
		}
    }

    foreach my $input (keys %{$toMap}) {
		my @files = split("#",$input);
		my $R1 = undef;
        my $R2 = undef;
		my $coreName = undef;

		if (scalar(@files) == 2) {
			print "Switching to paired end mode!\n";
			$R1 = $files[0];
			$R2 = $files[1];
			if($R1 !~ m/fastq.gz$/ or $R2 !~ m/fastq.gz$/) {
				die "ERROR: Invalid input files:\n\t$R1\n\t$R2\n";
			}
		} elsif (scalar(@files) == 1) {
			print "Switching to fragment mode!\n";
			$R1 = $files[0];
			$opt{SINGLE_END} = 1;
			if ($R1 !~ m/fastq.gz$/) {
				die "ERROR: Invalid input file:\n\t$R1\n";
			}
		} else {
			die "ERROR: Invalid input pair: $input\n";
		}

		$coreName = (split("/", $R1))[-1];
		my ($sampleName, $flowcellID, $index, $lane, $tag) =  split("_", $coreName);
		$coreName =~ s/\.fastq.gz//;
		$coreName =~ s/\_R1//;
		$coreName =~ s/\_R2//;

		print "Creating $opt{OUTPUT_DIR}/$sampleName/mapping/$coreName\_sorted.bam with:\n";

        createIndividualMappingJobs(\%opt, $QSUB, $samples, $sampleName, $coreName, $R1, $R2, $flowcellID);
    }

    print "\n";
    foreach my $sample (keys %{$samples}) {
        my @bamList = ();
        my @jobIds = ();

        foreach my $chunk (@{$samples->{$sample}}) {
            push( @bamList, $chunk->{'file'} );
            push( @jobIds, $chunk->{'jobId'} );
        }

        $opt{BAM_FILES}->{$sample} = "$sample\_dedup.bam";
        print "Creating $opt{BAM_FILES}->{$sample}\n";

        if (-e "$opt{OUTPUT_DIR}/$sample/logs/Mapping_$sample.done") {
            print "\tWARNING: $opt{OUTPUT_DIR}/$sample/logs/Mapping_$sample.done exists, skipping\n";
            next;
        }

        my $jobId = "Merge_$sample\_".get_job_id();
        push(@{$opt{RUNNING_JOBS}->{$sample}}, $jobId);
        my $bams = join(" ", @bamList);

        from_template("Merge.sh.tt", "$opt{OUTPUT_DIR}/$sample/jobs/$jobId.sh", sample => $sample, bamList => \@bamList, bams => $bams, runName => $runName, opt => \%opt);

        my $qsub = &qsubTemplate(\%opt, "MARKDUP");
        print $QSUB $qsub," -l excl=true -o ",$opt{OUTPUT_DIR},"/",$sample,"/logs/Merge_",$sample,".out -e ",$opt{OUTPUT_DIR},"/",$sample,"/logs/Merge_",$sample,".err -N ",$jobId,<commit_msg>Remove exclusive tag<commit_after>#!/usr/bin/perl -w

package illumina_mapping;

use strict;
use POSIX qw(tmpnam);
use lib "$FindBin::Bin";
use illumina_sge;
use illumina_template;

sub runMapping {
    my $configuration = shift;
    my %opt = %{$configuration};
    my $runName = (split("/", $opt{OUTPUT_DIR}))[-1];
    
    my $FAI = "$opt{GENOME}\.fai";
    die "GENOME: $opt{GENOME} does not exists!\n" if !-e "$opt{GENOME}";
    die "GENOME BWT: $opt{GENOME}.bwt does not exists!\n" if !-e "$opt{GENOME}.bwt";
    die "GENOME FAI: $FAI does not exists!\n" if !-e $FAI;

    my $mainJobID = "$opt{OUTPUT_DIR}/jobs/MapMainJob_".get_job_id().".sh";

    open (my $QSUB,">$mainJobID") or die "ERROR: Couldn't create $mainJobID\n";
    print $QSUB "\#!/bin/sh\n\n. $opt{CLUSTER_PATH}/settings.sh\n\n";

    my $samples = {};
    my $toMap = {};

    ### Try to search for matching pairs in the input FASTQ files
    foreach my $input (keys %{$opt{FASTQ}}) {
		if($input =~ m/\_R1/) {
			my $pairName = $input;
			$pairName =~ s/\_R1/\_R2/;
			if(exists ($opt{FASTQ}->{$pairName})) {
				$toMap->{$input."#".$pairName} = 1;
			} else {
				$toMap->{$input} = 1;
			}
		} elsif ($input =~ m/\_R2/) {
			my $pairName = $input;
			$pairName =~ s/\_R2/\_R1/;
			if (exists ($opt{FASTQ}->{$pairName})) {
				$toMap->{$pairName."#".$input} = 1;
			} else {
				$toMap->{$input} = 1;
			}
		}
    }

    foreach my $input (keys %{$toMap}) {
		my @files = split("#",$input);
		my $R1 = undef;
        my $R2 = undef;
		my $coreName = undef;

		if (scalar(@files) == 2) {
			print "Switching to paired end mode!\n";
			$R1 = $files[0];
			$R2 = $files[1];
			if($R1 !~ m/fastq.gz$/ or $R2 !~ m/fastq.gz$/) {
				die "ERROR: Invalid input files:\n\t$R1\n\t$R2\n";
			}
		} elsif (scalar(@files) == 1) {
			print "Switching to fragment mode!\n";
			$R1 = $files[0];
			$opt{SINGLE_END} = 1;
			if ($R1 !~ m/fastq.gz$/) {
				die "ERROR: Invalid input file:\n\t$R1\n";
			}
		} else {
			die "ERROR: Invalid input pair: $input\n";
		}

		$coreName = (split("/", $R1))[-1];
		my ($sampleName, $flowcellID, $index, $lane, $tag) =  split("_", $coreName);
		$coreName =~ s/\.fastq.gz//;
		$coreName =~ s/\_R1//;
		$coreName =~ s/\_R2//;

		print "Creating $opt{OUTPUT_DIR}/$sampleName/mapping/$coreName\_sorted.bam with:\n";

        createIndividualMappingJobs(\%opt, $QSUB, $samples, $sampleName, $coreName, $R1, $R2, $flowcellID);
    }

    print "\n";
    foreach my $sample (keys %{$samples}) {
        my @bamList = ();
        my @jobIds = ();

        foreach my $chunk (@{$samples->{$sample}}) {
            push( @bamList, $chunk->{'file'} );
            push( @jobIds, $chunk->{'jobId'} );
        }

        $opt{BAM_FILES}->{$sample} = "$sample\_dedup.bam";
        print "Creating $opt{BAM_FILES}->{$sample}\n";

        if (-e "$opt{OUTPUT_DIR}/$sample/logs/Mapping_$sample.done") {
            print "\tWARNING: $opt{OUTPUT_DIR}/$sample/logs/Mapping_$sample.done exists, skipping\n";
            next;
        }

        my $jobId = "Merge_$sample\_".get_job_id();
        push(@{$opt{RUNNING_JOBS}->{$sample}}, $jobId);
        my $bams = join(" ", @bamList);

        from_template("Merge.sh.tt", "$opt{OUTPUT_DIR}/$sample/jobs/$jobId.sh", sample => $sample, bamList => \@bamList, bams => $bams, runName => $runName, opt => \%opt);

        my $qsub = &qsubTemplate(\%opt, "MARKDUP");
        print $QSUB $qsub," -o ",$opt{OUTPUT_DIR},"/",$sample,"/logs/Merge_",$sample,".out -e ",$opt{OUTPUT_DIR},"/",$sample,"/logs/Merge_",$sample,".err -N ",$jobId,
{'fraction_non_alphanumeric': 0.2232845894263217, 'fraction_numerical': 0.009561304836895388, 'mean_word_length': 2.869967355821545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2928105', 'n_tokens_mistral': 3078, 'n_tokens_neox': 2822, 'n_words': 585}