<commit_before>#!/usr/bin/env sh

# Process the samples of one Run, from .sra to _barcode_report.pdf

set -e

## Parameters ##
function usage() {
    echo "Usage:"
    echo "      $0 -d BASEDIR -b BATCHDIR -i BOWTIE2_IDX [-D DATADIR] [-p PROCESSING_DIR] [-r RESULTS_DIR] [-a AUX_DIR] [[-s] | [-f]]"
		exit 1
}
# Defaults
data='data'
process='process'
results='results'
aux='aux'
# Parse options.
while getopts 'd:D:b:p:r:a:i:sf' flag; do
  case "${flag}" in
    d) base="${OPTARG}" ;;        # Base dir
		D) data="${OPTARG}" ;;     		# Data dir in which the batch is located, relative to base
		b) run="${OPTARG}" ;;         # Batch dir, relative to data dir
		p) process="${OPTARG}" ;;     # Dir in which intermediate steps output files are created, relative to base
		r) results="${OPTARG}" ;;     # Dir for the final files, relative to base
		a) aux="${OPTARG}" ;;     		# Dir where the bowtie index is located, relative to base
    i) bowtie2idx="${OPTARG}" ;;  # Bowtie2 index prefix
		s) issra="${OPTARG}" ;;       # Input is .sra format
		f) renamefq="${OPTARG}" ;;    # Files are uncompressed .fq instead of compressed .fastq.gz
		*) usage ;;
  esac
done


# For R to save plotly widgets
module load pandoc/2.0.5


if [[ ! -d "${base}/${process}/${run}" ]]; then
	mkdir -p "${base}/${process}/${run}"
  echo "Created destination: ${base}/${process}/${run}"
fi
if [[ ! -d "${base}/${results}/${run}" ]]; then
	mkdir -p "${base}/${results}/${run}"
  echo "Created destination: ${base}/${results}/${run}"
fi


if [[ $renamefq ]]; then
	echo "Renaming & compressing .fq to .fastq.gz"
	fileutilities.py T ${base}/${data}/${run} --dir fq | fileutilities.py P --loop mv {abs} {dir}/{bas}.fastq \; srun gzip {dir}/{bas}.fastq
fi

if [[ $issra ]]; then
	echo "Converting ${base}/${data}/${run}/*.sra to ${base}/${data}/${run}/*.fastq.gz"
	fileutilities.py T ${base}/${data}/${run} --dir .sra | fileutilities.py P --loop srun ~/utility_scripts/sra2fastq.sh ${base}/${data}/${run} ${base}/${data}/${run}/{val}
fi

echo "Extracting ${base}/${data}/${run}/*.bam to ${base}/${data}/${run}/*_1/2.fastq.gz"
fileutilities.py T ${base}/${data}/${run} --dir .bam | fileutilities.py P --loop samtools fastq ,-c 1 ,-1 {dir}/{ali}_1.fastq.gz ,-2 {dir}/{ali}_2.fastq.gz {abs}

echo "Merging pairs in ${base}/${data}/${run} to ${base}/${process}/${run}/*_x25..fastq.gz"
fileutilities.py T ${base}/${data}/${run} --dir _1.fastq.gz | perl -e 'while(<>){~s/_1\.fastq$//;print}' | fileutilities.py P --loop srun flash ,-t 1 ,-M 150 ,-x 0.25 ,-d ${base}/${process}/${run} ,-o {ali}_x25 ,-z ${base}/${data}/${run}/{ali}_1.fastq.gz ${base}/${data}/${run}/{ali}_2.fastq.gz

echo "Aligning ${base}/${data}/${run}/*.fastq.gz to ${base}/${aux}/${bowtie2idx} and outputting to ${base}/${process}/${run}/*_x25.extendedFrags.bam"
fileutilities.py T ${base}/${data}/${run} --dir _1.fastq.gz | perl -e 'while(<>){~s/_1\.fastq$//;print}' | fileutilities.py P --loop srun ,-c 4 ~/utility_scripts/bowtie2aln.sh ${base}/${aux}/${bowtie2idx} ${base}/${process}/${run} ${base}/${process}/${run} {ali}_x25.extendedFrags false 4 ,--very-sensitive-local


echo "Stratifying ${base}/${process}/${run}/*.sorted.bam by number of mismatches"
fileutilities.py T ${base}/${process}/${run} --dir 'sorted.bam$' | fileutilities.py P --loop srun ~/utility_scripts/split_BAM_by_numMismatches.sh {abs} 1 2
fileutilities.py T ${base}/${process}/${run} --dir 'sorted.bam$' | fileutilities.py P --loop srun ~/utility_scripts/split_BAM_by_numMismatches.sh {abs} 3 10
fileutilities.py T ${base}/${process}/${run} --dir 'sorted.bam$' | fileutilities.py P --loop srun ~/utility_scripts/split_BAM_by_numMismatches.sh {abs} 11 30

<commit_msg>don't run samtools on the login<commit_after>#!/usr/bin/env sh

# Process the samples of one Run, from .sra to _barcode_report.pdf

set -e

## Parameters ##
function usage() {
    echo "Usage:"
    echo "      $0 -d BASEDIR -b BATCHDIR -i BOWTIE2_IDX [-D DATADIR] [-p PROCESSING_DIR] [-r RESULTS_DIR] [-a AUX_DIR] [[-s] | [-f]]"
		exit 1
}
# Defaults
data='data'
process='process'
results='results'
aux='aux'
# Parse options.
while getopts 'd:D:b:p:r:a:i:sf' flag; do
  case "${flag}" in
    d) base="${OPTARG}" ;;        # Base dir
		D) data="${OPTARG}" ;;     		# Data dir in which the batch is located, relative to base
		b) run="${OPTARG}" ;;         # Batch dir, relative to data dir
		p) process="${OPTARG}" ;;     # Dir in which intermediate steps output files are created, relative to base
		r) results="${OPTARG}" ;;     # Dir for the final files, relative to base
		a) aux="${OPTARG}" ;;     		# Dir where the bowtie index is located, relative to base
    i) bowtie2idx="${OPTARG}" ;;  # Bowtie2 index prefix
		s) issra="${OPTARG}" ;;       # Input is .sra format
		f) renamefq="${OPTARG}" ;;    # Files are uncompressed .fq instead of compressed .fastq.gz
		*) usage ;;
  esac
done


# For R to save plotly widgets
module load pandoc/2.0.5


if [[ ! -d "${base}/${process}/${run}" ]]; then
	mkdir -p "${base}/${process}/${run}"
  echo "Created destination: ${base}/${process}/${run}"
fi
if [[ ! -d "${base}/${results}/${run}" ]]; then
	mkdir -p "${base}/${results}/${run}"
  echo "Created destination: ${base}/${results}/${run}"
fi


if [[ $renamefq ]]; then
	echo "Renaming & compressing .fq to .fastq.gz"
	fileutilities.py T ${base}/${data}/${run} --dir fq | fileutilities.py P --loop mv {abs} {dir}/{bas}.fastq \; srun gzip {dir}/{bas}.fastq
fi

if [[ $issra ]]; then
	echo "Converting ${base}/${data}/${run}/*.sra to ${base}/${data}/${run}/*.fastq.gz"
	fileutilities.py T ${base}/${data}/${run} --dir .sra | fileutilities.py P --loop srun ~/utility_scripts/sra2fastq.sh ${base}/${data}/${run} ${base}/${data}/${run}/{val}
fi

echo "Extracting ${base}/${data}/${run}/*.bam to ${base}/${data}/${run}/*_1/2.fastq.gz"
fileutilities.py T ${base}/${data}/${run} --dir .bam | fileutilities.py P --loop srun samtools fastq ,-c 1 ,-1 {dir}/{ali}_1.fastq.gz ,-2 {dir}/{ali}_2.fastq.gz {abs}

echo "Merging pairs in ${base}/${data}/${run} to ${base}/${process}/${run}/*_x25..fastq.gz"
fileutilities.py T ${base}/${data}/${run} --dir _1.fastq.gz | perl -e 'while(<>){~s/_1\.fastq$//;print}' | fileutilities.py P --loop srun flash ,-t 1 ,-M 150 ,-x 0.25 ,-d ${base}/${process}/${run} ,-o {ali}_x25 ,-z ${base}/${data}/${run}/{ali}_1.fastq.gz ${base}/${data}/${run}/{ali}_2.fastq.gz

echo "Aligning ${base}/${data}/${run}/*.fastq.gz to ${base}/${aux}/${bowtie2idx} and outputting to ${base}/${process}/${run}/*_x25.extendedFrags.bam"
fileutilities.py T ${base}/${data}/${run} --dir _1.fastq.gz | perl -e 'while(<>){~s/_1\.fastq$//;print}' | fileutilities.py P --loop srun ,-c 4 ~/utility_scripts/bowtie2aln.sh ${base}/${aux}/${bowtie2idx} ${base}/${process}/${run} ${base}/${process}/${run} {ali}_x25.extendedFrags false 4 ,--very-sensitive-local


echo "Stratifying ${base}/${process}/${run}/*.sorted.bam by number of mismatches"
fileutilities.py T ${base}/${process}/${run} --dir 'sorted.bam$' | fileutilities.py P --loop srun ~/utility_scripts/split_BAM_by_numMismatches.sh {abs} 1 2
fileutilities.py T ${base}/${process}/${run} --dir 'sorted.bam$' | fileutilities.py P --loop srun ~/utility_scripts/split_BAM_by_numMismatches.sh {abs} 3 10
fileutilities.py T ${base}/${process}/${run} --dir 'sorted.bam$' | fileutilities.py P --loop srun ~/utility_scripts/split_BAM_by_numMismatches.sh {abs} 11 30


{'fraction_non_alphanumeric': 0.22816519972918078, 'fraction_numerical': 0.013811780636425186, 'mean_word_length': 3.8946322067594434, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7440878', 'n_tokens_mistral': 2967, 'n_tokens_neox': 2826, 'n_words': 704}