<commit_before>#! /usr/bin/bash

# Exit on First Error - to Prevent Invalid File Modifications
# set -o errexit

# Load ~/.bash_profile if Not Found
if [ -z $PIPELINE_HOME ]; then
    echo "Reloading ~/.bash_profile"
    source ~/.bash_profile
fi

# Assign Arguments
for i in "$@"
    do case $i in

    # Standard Arguments

        -f=*|--fileprefix=*)
        fileprefix="${i#*=}"
        shift # Access & Write Files With This Prefix
        ;;
        -s=*|--subset=*)
        subset="${i#*=}"
        shift # Access & Write Files With This Subset
        ;;
        -c=*|--condition=*)
        condition="${i#*=}"
        shift # Access & Write Files With This Condition
        ;;
        -x=*|--experiment=*)
        experiment="${i#*=}"
        shift # Access & Write Files With This Experiment
        ;;
        -p=*|--parameters=*)
        parameters="${i#*=}"
        shift # Access & Write Files With This Parameter Set
        ;;

    # Additional Arguments

        -a=*|--align=*)
        alignOpt="${i#*=}"
        shift
        ;;

    # Optional Arguments With Defaults

        -n=*|--ncores=*)
        ncoresOpt="${i#*=}"
        shift # Number of Cores to Use
        ;;
        -m=*|--memory=*)
        memoryOpt="${i#*=}"
        shift # Per Core Memory Requirement
        ;;

    # Directory Cleanup (Voids All Other Parameters)

        --clean)
        cleanOpt=true
        ;;

    # Invalid Argument Handler

        *)
        # invalid option
        printf "Invalid/Unused Parameter: $i"
        ;;
        
    esac
done

# Defaults if No Arguments Passed
ncoresDef="16"
memoryDef="6G"
alignDef="mem"

# Set Optional Values
ncores=${ncoresOpt:-$ncoresDef}
memory=${memoryOpt:-$memoryDef}
cleanDef=false
align=${alignOpt:-$alignDef}

# Get Max Allowable Memory
allocMemory=${memory//[GgMmKk]/}
allocSize=${memory//[0-9]/}
maxMemory=$((allocMemory * ncores))$allocSize
 
printf "\nPARAMETERS: 
Data File Prefix    = $fileprefix
Data Subset         = $subset
Condition           = $condition
Experiment          = $experiment
Parameter Set       = $parameters
BWA Alignment       = $align
Memory              = $memory
Cores               = $ncores
Max Memory          = $maxMemory
Do Cleanup          = $clean
\n"

# Set Directories
paramDir=$PIPELINE_HOME/$subset/model/$experiment/param/$parameters

format_status "Running BWA Script"

#
# BWA Index
#

# State Check - Run Block if it Has Not Already Been Executed Successfully
state="$fileprefix:BWA:1"
if !(has_state $state); then

    format_status "BWA Index"
    # Define Command
    call="bwa index -a bwtsw $PIPELINE_REF/Homo_sapiens_assembly19.fasta"
    # Print & Call
    format_status "Command:\n$call"
    eval $call

    # Update State on Exit
    put_state $? $state
    format_status "BWA Index Complete"

fi

#
# Shuffle Reads Inplace
#

# State Management Delegated to Substates
format_status "Shuffling Input FastQ Reads"
# Retrieve Files
files=$(echo $(ls $paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.*.fastq))

for file in $files
    do (
        # Extract Read Group to Pass to BWA mem
        suffix=$(echo "$file" | sed "s|$paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.||")
        readgroup=$(echo "$suffix" | sed "s|.fastq$||")
        substate="$fileprefix.$subset.$condition.$experiment.$parameters.$readgroup:BWA:2"
        
        # Run Command
        if !(has_state $substate); then

            # Call BBMap shuffle.sh
            # Define Command
            call="shuffle.sh -Xmx$memory in=$file out=$paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.shuffled.$readgroup.fastq && mv $paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.shuffled.$readgroup.fastq $paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.$readgroup.fastq"
            # Print & Call<commit_msg>Version: 0.7.9 - Give Shuffle More Memory<commit_after>#! /usr/bin/bash

# Exit on First Error - to Prevent Invalid File Modifications
# set -o errexit

# Load ~/.bash_profile if Not Found
if [ -z $PIPELINE_HOME ]; then
    echo "Reloading ~/.bash_profile"
    source ~/.bash_profile
fi

# Assign Arguments
for i in "$@"
    do case $i in

    # Standard Arguments

        -f=*|--fileprefix=*)
        fileprefix="${i#*=}"
        shift # Access & Write Files With This Prefix
        ;;
        -s=*|--subset=*)
        subset="${i#*=}"
        shift # Access & Write Files With This Subset
        ;;
        -c=*|--condition=*)
        condition="${i#*=}"
        shift # Access & Write Files With This Condition
        ;;
        -x=*|--experiment=*)
        experiment="${i#*=}"
        shift # Access & Write Files With This Experiment
        ;;
        -p=*|--parameters=*)
        parameters="${i#*=}"
        shift # Access & Write Files With This Parameter Set
        ;;

    # Additional Arguments

        -a=*|--align=*)
        alignOpt="${i#*=}"
        shift
        ;;

    # Optional Arguments With Defaults

        -n=*|--ncores=*)
        ncoresOpt="${i#*=}"
        shift # Number of Cores to Use
        ;;
        -m=*|--memory=*)
        memoryOpt="${i#*=}"
        shift # Per Core Memory Requirement
        ;;

    # Directory Cleanup (Voids All Other Parameters)

        --clean)
        cleanOpt=true
        ;;

    # Invalid Argument Handler

        *)
        # invalid option
        printf "Invalid/Unused Parameter: $i"
        ;;
        
    esac
done

# Defaults if No Arguments Passed
ncoresDef="16"
memoryDef="6G"
alignDef="mem"

# Set Optional Values
ncores=${ncoresOpt:-$ncoresDef}
memory=${memoryOpt:-$memoryDef}
cleanDef=false
align=${alignOpt:-$alignDef}

# Get Max Allowable Memory
allocMemory=${memory//[GgMmKk]/}
allocSize=${memory//[0-9]/}
maxMemory=$((allocMemory * ncores))$allocSize
 
printf "\nPARAMETERS: 
Data File Prefix    = $fileprefix
Data Subset         = $subset
Condition           = $condition
Experiment          = $experiment
Parameter Set       = $parameters
BWA Alignment       = $align
Memory              = $memory
Cores               = $ncores
Max Memory          = $maxMemory
Do Cleanup          = $clean
\n"

# Set Directories
paramDir=$PIPELINE_HOME/$subset/model/$experiment/param/$parameters

format_status "Running BWA Script"

#
# BWA Index
#

# State Check - Run Block if it Has Not Already Been Executed Successfully
state="$fileprefix:BWA:1"
if !(has_state $state); then

    format_status "BWA Index"
    # Define Command
    call="bwa index -a bwtsw $PIPELINE_REF/Homo_sapiens_assembly19.fasta"
    # Print & Call
    format_status "Command:\n$call"
    eval $call

    # Update State on Exit
    put_state $? $state
    format_status "BWA Index Complete"

fi

#
# Shuffle Reads Inplace
#

# State Management Delegated to Substates
format_status "Shuffling Input FastQ Reads"
# Retrieve Files
files=$(echo $(ls $paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.*.fastq))

for file in $files
    do (
        # Extract Read Group to Pass to BWA mem
        suffix=$(echo "$file" | sed "s|$paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.||")
        readgroup=$(echo "$suffix" | sed "s|.fastq$||")
        substate="$fileprefix.$subset.$condition.$experiment.$parameters.$readgroup:BWA:2"
        
        # Run Command
        if !(has_state $substate); then

            # Call BBMap shuffle.sh
            # Define Command
            call="shuffle.sh -Xmx$maxMemory in=$file out=$paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.shuffled.$readgroup.fastq && mv $paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.shuffled.$readgroup.fastq $paramDir/modeled/$fileprefix.$subset.$condition.$experiment.$parameters.$readgroup.fastq"
            # Print & Call
{'fraction_non_alphanumeric': 0.13482859691601887, 'fraction_numerical': 0.0026761819803746653, 'mean_word_length': 2.5479204339963832, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '870413', 'n_tokens_mistral': 2471, 'n_tokens_neox': 2361, 'n_words': 594}