<commit_before>#!/usr/bin/env bash

##############################################################################################################################################################
#
# Improves a set of bins by aligning reads back to the bins and reassembling them.
#
# Author of pipeline: <NAME>. I do not clain any authorship of the many programs this pipeline uses.
# For questions, bugs, and suggestions, contact me at <EMAIL>.
# 
##############################################################################################################################################################


help_message () {
	echo ""
	echo "Usage: metaWRAP reassemble_bins [options] -o output_dir -b bin_folder -1 reads_1.fastq -2 reads_2.fastq"
	echo ""
	echo "Options:"
	echo ""
	echo "	-b STR		folder with metagenomic bins"
	echo "	-o STR		output directory"
	echo "	-1 STR          forward reads to use for reassembly"
	echo "	-2 STR          reverse reads to use for reassembly"
	echo ""
	echo "	-t INT		number of threads (default=1)"
	echo "	-m INT		memory in GB (default=40)"
	echo "	-c INT		minimum desired bin completion %"
	echo "	-x INT		maximum desired bin contamination %"
	echo "	-l INT		minimum contig length to be included in reassembly (default=500)"
	echo ""
	echo "	--strict-cut-off	maximum allowed SNPs for strict read mapping (default=2)"
	echo "	--permissive-cut-off	maximum allowed SNPs for permissive read mapping (default=5)"
	echo "	--skip-checkm		dont run CheckM to assess bins"
	echo "	--parallel		run spades reassembly in parallel, but only using 1 thread per bin"
	echo "";}

comm () { ${SOFT}/print_comment.py "$1" "-"; }
error () { ${SOFT}/print_comment.py "$1" "*"; exit 1; }
warning () { ${SOFT}/print_comment.py "$1" "*"; }
announcement () { ${SOFT}/print_comment.py "$1" "#"; }

# these functions are for parallelizing the reassembly
open_sem(){
    mkfifo pipe-$$
    exec 3<>pipe-$$
    rm pipe-$$
    local i=$1
    for((;i>0;i--)); do
        printf %s 000 >&3
    done
}
run_with_lock(){
    local x
    read -u 3 -n 3 x && ((0==x)) || exit $x
    (
    "$@" 
    printf '%.3d' $? >&3
    )&
}

########################################################################################################
########################               LOADING IN THE PARAMETERS                ########################
########################################################################################################


# setting scripts and databases from config file (should be in same folder as main script)
config_file=$(which config-metawrap)
source $config_file


# default params
threads=1; mem=40; comp=70; cont=10; len=500
bins=None; f_reads=None; r_reads=None; out=None
# long options defaults
strict_max=2; permissive_max=5
run_checkm=true
run_parallel=false
# load in params
OPTS=`getopt -o ht:m:o:x:c:l:b:1:2: --long help,parallel,skip-checkm,strict-cut-off,permissive-cut-off -- "$@"`
# make sure the params are entered correctly
if [ $? -ne 0 ]; then help_message; exit 1; fi

# loop through input params
while true; do
        case "$1" in
                -t) threads=$2; shift 2;;
                -m) mem=$2; shift 2;;
                -o) out=$2; shift 2;;
		-x) cont=$2; shift 2;;
		-c) comp=$2; shift 2;;
		-b) bins=$2; shift 2;;
		-l) len=$2; shift 2;;
		-1) f_reads=$2; shift 2;;
		-2) r_reads=$2; shift 2;;
                -h | --help) help_message; exit 0; shift 1;;
		--strict-cut-off) strict_max=$2; shift 2;;
		--permissive-cut-off) permissive_max=$2; shift 2;;
		--skip-checkm) run_checkm=false; shift 1;;
		--parallel) run_parallel=true; shift 1;;
                --) help_message; exit 1; shift; break ;;
                *) break;;
        esac
done

########################################################################################################
########################           MAKING SURE EVERYTHING IS SET UP             ########################
########################################################################################################

# check if all parameters are entered
if [ $out = None ] || [  $bins = None ] || [ $f_reads = None ] || [ $r_reads = None ] ; then 
	comm "Some non-optional parameters were not entered"
	help_message; exit 1
fi

# Checks for correctly configures meta-scripts folder
if [ ! -s $SOFT/sort_contigs.py ]; then
	error "The folder $SOFT doesnt exist. Please make sure config.sh is in the same filder as the mains scripts and all the paths in the config.sh file are correct"
fi

########################################################################################################
########################               BEGIN REASSEMBLY PIPELINE!               ########################
########################################################################################################
announcement "BEGIN PIPELINE!"
comm "setting up output folder and copything over bins..."
if [ ! -d $out ]; then
        mkdir $out;
else
        warning "Warning: $out already exists!"
fi

if [ -d ${out}/original_bins ]; then rm -r ${out}/original_bins; fi
cp -r $bins ${out}/original_bins
if [ ! -d ${out}/binned_assembly ]; then mkdir ${out}/binned_assembly; fi

# combinde the bins into one big assembly file
if [ -s ${out}/binned_assembly/assembly.fa ]; then rm ${out}/binned_assembly/assembly.fa; fi
for i in $(ls ${out}/original_bins); do cat ${out}/original_bins/$i >> ${out}/binned_assembly/assembly.fa; done


########################################################################################################
########################        RECRUITING READS TO BINS FOR REASSEMBLY         ########################
########################################################################################################
announcement "RECRUITING READS TO BINS FOR REASSEMBLY"

ulimit -n 10000
if [[ $? -new 0 ]]; then
	ULIMIT=$(ulimit -n)
	warning "Your operating system will allow you to process up to $ULIMIT files at a time. If this is number is less than 4X times of the number of bins you are reassembling, you will likely get an error. Try re-assembling a smaller number of bins."
fi


if [[ ! -s ${out}/binned_assembly/assembly.fa.amb ]]; then
	comm "Indexing the assembly"
	bwa index ${out}/binned_assembly/assembly.fa
	if [[ $? -ne 0 ]]; then error "BWA failed to index $i"; fi

	if [ -d ${out}/reads_for_reassembly ]; then rm -r ${out}/reads_for_reassembly; fi
	mkdir ${out}/reads_for_reassembly

	comm "Aligning all reads back to entire assembly and splitting reads into individual fastq files based on their bin membership"
	bwa mem -t $threads ${out}/binned_assembly/assembly.fa $f_reads $r_reads \
	 | ${SOFT}/filter_reads_for_bin_reassembly.py ${out}/original_bins ${out}/reads_for_reassembly $strict_max $permissive_max
	if [[ $? -ne 0 ]]; then error "Something went wrong with pulling out reads for reassembly..."; fi
else
	comm "WARNING: Looks like the assembly was already indexed. Skipping indexing, and also skipping splitting the reads, because it is assumed you already got to this stage. Will proceed directly to assembly. This is because your output folder $out already has outputs from previous runs. If this is not what you intended, re-run the module with a new output folder, or clear $out."
fi<commit_msg>fixed operator typo<commit_after>#!/usr/bin/env bash

##############################################################################################################################################################
#
# Improves a set of bins by aligning reads back to the bins and reassembling them.
#
# Author of pipeline: <NAME>. I do not clain any authorship of the many programs this pipeline uses.
# For questions, bugs, and suggestions, contact me at <EMAIL>.
# 
##############################################################################################################################################################


help_message () {
	echo ""
	echo "Usage: metaWRAP reassemble_bins [options] -o output_dir -b bin_folder -1 reads_1.fastq -2 reads_2.fastq"
	echo ""
	echo "Options:"
	echo ""
	echo "	-b STR		folder with metagenomic bins"
	echo "	-o STR		output directory"
	echo "	-1 STR          forward reads to use for reassembly"
	echo "	-2 STR          reverse reads to use for reassembly"
	echo ""
	echo "	-t INT		number of threads (default=1)"
	echo "	-m INT		memory in GB (default=40)"
	echo "	-c INT		minimum desired bin completion %"
	echo "	-x INT		maximum desired bin contamination %"
	echo "	-l INT		minimum contig length to be included in reassembly (default=500)"
	echo ""
	echo "	--strict-cut-off	maximum allowed SNPs for strict read mapping (default=2)"
	echo "	--permissive-cut-off	maximum allowed SNPs for permissive read mapping (default=5)"
	echo "	--skip-checkm		dont run CheckM to assess bins"
	echo "	--parallel		run spades reassembly in parallel, but only using 1 thread per bin"
	echo "";}

comm () { ${SOFT}/print_comment.py "$1" "-"; }
error () { ${SOFT}/print_comment.py "$1" "*"; exit 1; }
warning () { ${SOFT}/print_comment.py "$1" "*"; }
announcement () { ${SOFT}/print_comment.py "$1" "#"; }

# these functions are for parallelizing the reassembly
open_sem(){
    mkfifo pipe-$$
    exec 3<>pipe-$$
    rm pipe-$$
    local i=$1
    for((;i>0;i--)); do
        printf %s 000 >&3
    done
}
run_with_lock(){
    local x
    read -u 3 -n 3 x && ((0==x)) || exit $x
    (
    "$@" 
    printf '%.3d' $? >&3
    )&
}

########################################################################################################
########################               LOADING IN THE PARAMETERS                ########################
########################################################################################################


# setting scripts and databases from config file (should be in same folder as main script)
config_file=$(which config-metawrap)
source $config_file


# default params
threads=1; mem=40; comp=70; cont=10; len=500
bins=None; f_reads=None; r_reads=None; out=None
# long options defaults
strict_max=2; permissive_max=5
run_checkm=true
run_parallel=false
# load in params
OPTS=`getopt -o ht:m:o:x:c:l:b:1:2: --long help,parallel,skip-checkm,strict-cut-off,permissive-cut-off -- "$@"`
# make sure the params are entered correctly
if [ $? -ne 0 ]; then help_message; exit 1; fi

# loop through input params
while true; do
        case "$1" in
                -t) threads=$2; shift 2;;
                -m) mem=$2; shift 2;;
                -o) out=$2; shift 2;;
		-x) cont=$2; shift 2;;
		-c) comp=$2; shift 2;;
		-b) bins=$2; shift 2;;
		-l) len=$2; shift 2;;
		-1) f_reads=$2; shift 2;;
		-2) r_reads=$2; shift 2;;
                -h | --help) help_message; exit 0; shift 1;;
		--strict-cut-off) strict_max=$2; shift 2;;
		--permissive-cut-off) permissive_max=$2; shift 2;;
		--skip-checkm) run_checkm=false; shift 1;;
		--parallel) run_parallel=true; shift 1;;
                --) help_message; exit 1; shift; break ;;
                *) break;;
        esac
done

########################################################################################################
########################           MAKING SURE EVERYTHING IS SET UP             ########################
########################################################################################################

# check if all parameters are entered
if [ $out = None ] || [  $bins = None ] || [ $f_reads = None ] || [ $r_reads = None ] ; then 
	comm "Some non-optional parameters were not entered"
	help_message; exit 1
fi

# Checks for correctly configures meta-scripts folder
if [ ! -s $SOFT/sort_contigs.py ]; then
	error "The folder $SOFT doesnt exist. Please make sure config.sh is in the same filder as the mains scripts and all the paths in the config.sh file are correct"
fi

########################################################################################################
########################               BEGIN REASSEMBLY PIPELINE!               ########################
########################################################################################################
announcement "BEGIN PIPELINE!"
comm "setting up output folder and copything over bins..."
if [ ! -d $out ]; then
        mkdir $out;
else
        warning "Warning: $out already exists!"
fi

if [ -d ${out}/original_bins ]; then rm -r ${out}/original_bins; fi
cp -r $bins ${out}/original_bins
if [ ! -d ${out}/binned_assembly ]; then mkdir ${out}/binned_assembly; fi

# combinde the bins into one big assembly file
if [ -s ${out}/binned_assembly/assembly.fa ]; then rm ${out}/binned_assembly/assembly.fa; fi
for i in $(ls ${out}/original_bins); do cat ${out}/original_bins/$i >> ${out}/binned_assembly/assembly.fa; done


########################################################################################################
########################        RECRUITING READS TO BINS FOR REASSEMBLY         ########################
########################################################################################################
announcement "RECRUITING READS TO BINS FOR REASSEMBLY"

ulimit -n 10000
if [[ $? -ne 0 ]]; then
	ULIMIT=$(ulimit -n)
	warning "Your operating system will allow you to process up to $ULIMIT files at a time. If this is number is less than 4X times of the number of bins you are reassembling, you will likely get an error. Try re-assembling a smaller number of bins."
fi


if [[ ! -s ${out}/binned_assembly/assembly.fa.amb ]]; then
	comm "Indexing the assembly"
	bwa index ${out}/binned_assembly/assembly.fa
	if [[ $? -ne 0 ]]; then error "BWA failed to index $i"; fi

	if [ -d ${out}/reads_for_reassembly ]; then rm -r ${out}/reads_for_reassembly; fi
	mkdir ${out}/reads_for_reassembly

	comm "Aligning all reads back to entire assembly and splitting reads into individual fastq files based on their bin membership"
	bwa mem -t $threads ${out}/binned_assembly/assembly.fa $f_reads $r_reads \
	 | ${SOFT}/filter_reads_for_bin_reassembly.py ${out}/original_bins ${out}/reads_for_reassembly $strict_max $permissive_max
	if [[ $? -ne 0 ]]; then error "Something went wrong with pulling out reads for reassembly..."; fi
else
	comm "WARNING: Looks like the assembly was already indexed. Skipping indexing, and also skipping splitting the reads, because it is assumed you already got to this stage. Will proceed directly to assembly. This is because your output folder $out already has outputs from previous runs. If this is not what you intended, re-run the module with a new output folder, or clear $out."
fi
{'fraction_non_alphanumeric': 0.2836508700735952, 'fraction_numerical': 0.01210537175871793, 'mean_word_length': 4.121521662557239, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6632095', 'n_tokens_mistral': 4334, 'n_tokens_neox': 3977, 'n_words': 1457}