<commit_before>#!/bin/bash -l

##############################################################################################################################################################
#
# Improves a set of bins by aligning reads back to the bins and reassembling them.
#
# Author of pipeline: <NAME>. I do not clain any authorship of the many programs this pipeline uses.
# For questions, bugs, and suggestions, contact me at <EMAIL>.
# 
##############################################################################################################################################################


help_message () {
	echo ""
	echo "Usage: metaWRAP reassemble_bins [options] -o output_dir -b bin_folder -1 reads_1.fastq -2 reads_2.fastq"
	echo ""
	echo "Options:"
	echo ""
	echo "	-o STR		output directory"
	echo "	-1 STR          forward reads to use for reassembly"
	echo "	-2 STR          reverse reads to use for reassembly"
	echo "	-t INT		number of threads (default=1)"
	echo "	-m INT		memory in GB (default=4)"
	echo "	-b STR		folder with metagenomic bins"
	echo "	-c INT		minimum desired bin completion %"
	echo "	-x INT		maximum desired bin contamination %"
	echo ""
	echo "	--skip-checkm		dont run CheckM to assess bins"
	echo "";}

comm () { ${SOFT}/print_comment.py "$1" "-"; }
error () { ${SOFT}/print_comment.py "$1" "*"; exit 1; }
warning () { ${SOFT}/print_comment.py "$1" "*"; }
announcement () { ${SOFT}/print_comment.py "$1" "#"; }
function pwait() {
    while [ $(jobs -p | wc -l) -ge $1 ]; do
        sleep 1
    done
}


########################################################################################################
########################               LOADING IN THE PARAMETERS                ########################
########################################################################################################


# setting scripts and databases from config file (should be in same folder as main script)
source config-metawrap

# default params
threads=1; mem=4; comp=70; cont=10; 
bins=None; f_reads=None; r_reads=None; out=None
# long options defaults
run_checkm=true

# load in params
OPTS=`getopt -o ht:m:o:x:c:b:1:2: --long help,skip-checkm -- "$@"`
# make sure the params are entered correctly
if [ $? -ne 0 ]; then help_message; exit 1; fi

# loop through input params
while true; do
        case "$1" in
                -t) threads=$2; shift 2;;
                -m) mem=$2; shift 2;;
                -o) out=$2; shift 2;;
		-x) cont=$2; shift 2;;
		-c) comp=$2; shift 2;;
		-b) bins=$2; shift 2;;
		-1) f_reads=$2; shift 2;;
		-2) r_reads=$2; shift 2;;
                -h | --help) help_message; exit 0; shift 1;;
		--skip-checkm) run_checkm=false; shift 1;;
                --) help_message; exit 1; shift; break ;;
                *) break;;
        esac
done


########################################################################################################
########################           MAKING SURE EVERYTHING IS SET UP             ########################
########################################################################################################

# check if all parameters are entered
if [ $out = None ] || [  bins = None ] || [ f_reads = None ] || [ r_reads = None ] ; then 
	comm "Some non-optional parameters were not entered"
	help_message; exit 1
fi

# Checks for correctly configures meta-scripts folder
if [ ! -s $SOFT/sort_contigs.py ]; then
	error "The folder $SOFT doesnt exist. Please make sure config.sh is in the same filder as the mains scripts and all the paths in the config.sh file are correct"
fi

########################################################################################################
########################               BEGIN REASSEMBLY PIPELINE!               ########################
########################################################################################################
announcement "PEGIN PIPELINE!"
comm "setting up output folder and copything over bins..."
if [ ! -d $out ]; then
        mkdir $out;
else
        echo "Warning: $out already exists."
fi

if [ -d ${out}//original_bins ]; then rm -r ${out}/original_bins; fi
cp -r $bins ${out}/original_bins
if [ ! -d ${out}/binned_assembly ]; then mkdir ${out}/binned_assembly; fi
if [ -s ${out}/binned_assembly/assembly.fa ]; then rm ${out}/binned_assembly/assembly.fa; fi
for i in $(ls ${out}/original_bins); do cat ${out}/original_bins/$i >> ${out}/binned_assembly/assembly.fa; done


########################################################################################################
########################        RECRUITING READS TO BINS FOR REASSEMBLY         ########################
########################################################################################################
announcement "RECRUITING READS TO BINS FOR REASSEMBLY"
comm "Indexing the assembly"
bwa index ${out}/binned_assembly/assembly.fa
if [[ $? -ne 0 ]]; then error "BWA failed to index $i"; fi

comm "Align reads back to assembly, saving every possible alignment of each read, and splitting them according to which bin they aligned to."
mkdir ${out}/reads_for_reassembly
bwa mem -t $threads ${out}/binned_assembly/assembly.fa $f_reads $r_reads\
 | ${SOFT}/filter_reads_for_bin_reassembly.py ${out}/original_bins $f_reads $r_reads ${out}/reads_for_reassembly



########################################################################################################
########################             REASSEMBLING BINS WITH SPADES              ########################
########################################################################################################
announcement "REASSEMBLING BINS WITH SPADES"

mkdir ${out}/reassemblies
assemble () {
	bin_name=${1%_*}
	tmp_dir=${out}/reassemblies/${bin_name}.tmp
	mkdir $tmp_dir
	comm "NOW REASSEMBLING ${bin_name}"
	spades.py -t 1 -m $mem --tmp $tmp_dir --careful \
	--untrusted-contigs ${out}/original_bins/${bin_name%.*}.fa \
	-1 ${out}/reads_for_reassembly/${1%_*}_1.fastq \
	-2 ${out}/reads_for_reassembly/${1%_*}_2.fastq \
	-o ${out}/reassemblies/${bin_name}
	
	if [[ ! -s ${out}/reassemblies/${bin_name}/scaffolds.fasta ]]; then
                warning "Something went wrong with reassembling ${bin_name}"
	else 
		comm "${bin_name} was reassembled successfully!"
		rm -r $tmp_dir
	fi
}
for i in $(ls ${out}/reads_for_reassembly/ | grep _1.fastq); do 
	assemble $i & 
	pwait $threads<commit_msg>fixed another parallel bug<commit_after>#!/bin/bash -l

##############################################################################################################################################################
#
# Improves a set of bins by aligning reads back to the bins and reassembling them.
#
# Author of pipeline: <NAME>. I do not clain any authorship of the many programs this pipeline uses.
# For questions, bugs, and suggestions, contact me at <EMAIL>.
# 
##############################################################################################################################################################


help_message () {
	echo ""
	echo "Usage: metaWRAP reassemble_bins [options] -o output_dir -b bin_folder -1 reads_1.fastq -2 reads_2.fastq"
	echo ""
	echo "Options:"
	echo ""
	echo "	-o STR		output directory"
	echo "	-1 STR          forward reads to use for reassembly"
	echo "	-2 STR          reverse reads to use for reassembly"
	echo "	-t INT		number of threads (default=1)"
	echo "	-m INT		memory in GB (default=4)"
	echo "	-b STR		folder with metagenomic bins"
	echo "	-c INT		minimum desired bin completion %"
	echo "	-x INT		maximum desired bin contamination %"
	echo ""
	echo "	--skip-checkm		dont run CheckM to assess bins"
	echo "";}

comm () { ${SOFT}/print_comment.py "$1" "-"; }
error () { ${SOFT}/print_comment.py "$1" "*"; exit 1; }
warning () { ${SOFT}/print_comment.py "$1" "*"; }
announcement () { ${SOFT}/print_comment.py "$1" "#"; }

# these functions are for parallelizing the reassembly
open_sem(){
    mkfifo pipe-$$
    exec 3<>pipe-$$
    rm pipe-$$
    local i=$1
    for((;i>0;i--)); do
        printf %s 000 >&3
    done
}
run_with_lock(){
    local x
    read -u 3 -n 3 x && ((0==x)) || exit $x
    (
    "$@" 
    printf '%.3d' $? >&3
    )&
}

########################################################################################################
########################               LOADING IN THE PARAMETERS                ########################
########################################################################################################


# setting scripts and databases from config file (should be in same folder as main script)
source config-metawrap

# default params
threads=1; mem=4; comp=70; cont=10; 
bins=None; f_reads=None; r_reads=None; out=None
# long options defaults
run_checkm=true

# load in params
OPTS=`getopt -o ht:m:o:x:c:b:1:2: --long help,skip-checkm -- "$@"`
# make sure the params are entered correctly
if [ $? -ne 0 ]; then help_message; exit 1; fi

# loop through input params
while true; do
        case "$1" in
                -t) threads=$2; shift 2;;
                -m) mem=$2; shift 2;;
                -o) out=$2; shift 2;;
		-x) cont=$2; shift 2;;
		-c) comp=$2; shift 2;;
		-b) bins=$2; shift 2;;
		-1) f_reads=$2; shift 2;;
		-2) r_reads=$2; shift 2;;
                -h | --help) help_message; exit 0; shift 1;;
		--skip-checkm) run_checkm=false; shift 1;;
                --) help_message; exit 1; shift; break ;;
                *) break;;
        esac
done


########################################################################################################
########################           MAKING SURE EVERYTHING IS SET UP             ########################
########################################################################################################

# check if all parameters are entered
if [ $out = None ] || [  bins = None ] || [ f_reads = None ] || [ r_reads = None ] ; then 
	comm "Some non-optional parameters were not entered"
	help_message; exit 1
fi

# Checks for correctly configures meta-scripts folder
if [ ! -s $SOFT/sort_contigs.py ]; then
	error "The folder $SOFT doesnt exist. Please make sure config.sh is in the same filder as the mains scripts and all the paths in the config.sh file are correct"
fi

########################################################################################################
########################               BEGIN REASSEMBLY PIPELINE!               ########################
########################################################################################################
announcement "PEGIN PIPELINE!"
comm "setting up output folder and copything over bins..."
if [ ! -d $out ]; then
        mkdir $out;
else
        echo "Warning: $out already exists."
fi

if [ -d ${out}/original_bins ]; then rm -r ${out}/original_bins; fi
cp -r $bins ${out}/original_bins
if [ ! -d ${out}/binned_assembly ]; then mkdir ${out}/binned_assembly; fi
if [ -s ${out}/binned_assembly/assembly.fa ]; then rm ${out}/binned_assembly/assembly.fa; fi
for i in $(ls ${out}/original_bins); do cat ${out}/original_bins/$i >> ${out}/binned_assembly/assembly.fa; done


########################################################################################################
########################        RECRUITING READS TO BINS FOR REASSEMBLY         ########################
########################################################################################################
announcement "RECRUITING READS TO BINS FOR REASSEMBLY"
comm "Indexing the assembly"
bwa index ${out}/binned_assembly/assembly.fa
if [[ $? -ne 0 ]]; then error "BWA failed to index $i"; fi

comm "Align reads back to assembly, saving every possible alignment of each read, and splitting them according to which bin they aligned to."
mkdir ${out}/reads_for_reassembly
bwa mem -t $threads ${out}/binned_assembly/assembly.fa $f_reads $r_reads\
 | ${SOFT}/filter_reads_for_bin_reassembly.py ${out}/original_bins $f_reads $r_reads ${out}/reads_for_reassembly


########################################################################################################
########################             REASSEMBLING BINS WITH SPADES              ########################
########################################################################################################
announcement "REASSEMBLING BINS WITH SPADES"

mkdir ${out}/reassemblies
assemble () {
	bin_name=${1%_*}
	tmp_dir=${out}/reassemblies/${bin_name}.tmp
	mkdir $tmp_dir
	comm "NOW REASSEMBLING ${bin_name}"
	spades.py -t 1 -m $mem --tmp $tmp_dir --careful \
	--untrusted-contigs ${out}/original_bins/${bin_name%.*}.fa \
	-1 ${out}/reads_for_reassembly/${1%_*}_1.fastq \
	-2 ${out}/reads_for_reassembly/${1%_*}_2.fastq \
	-o ${out}/reassemblies/${bin_name}
	
	if [[ ! -s ${out}/reassemblies/${bin_name}/scaffolds.fasta ]]; then
                warning "Something went wrong with reassembling ${bin_name}"
	else 
		comm "${bin_name} was reassembled successfully!"
		rm -r $tmp_dir
	fi
}

open_sem $threads
for i in $(ls ${out}/reads_for_reassembly/ | grep _1.fastq); do 
	run_with_lock assemble $i
{'fraction_non_alphanumeric': 0.343870821014328, 'fraction_numerical': 0.009703585778182094, 'mean_word_length': 4.247414478918059, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4911170', 'n_tokens_mistral': 3895, 'n_tokens_neox': 3580, 'n_words': 1128}