<commit_before>#!/bin/bash

#liWGS-SV Pipeline: Module 4 (Physical Depth CNV Calling)
#August 2015
#Contact: <EMAIL>

#Read input
samples_list=$1
params=$2

#Source params file
. ${params}
cd ${WRKDIR}

#Update on Nov 30, 2015: No longer run DNAcopy by default b/c too memory intensive, too slow, and not highly informative

# #Load necessary modules for SE large CNV caller
# module load intel_parallel_xe/xe
# module load R/3.1.0
# Rscript -e "if(\"DNAcopy\" %in% rownames(installed.packages()) == FALSE){source(\"http://bioconductor.org/biocLite.R\"); biocLite(\"DNAcopy\")}; suppressPackageStartupMessages(library(DNAcopy))"
module rm gcc/4.9.0
module rm gcc-4.4
module load gcc/4.9.0

# #Submit SE large CNV caller
# Rscript ${liWGS_SV}/scripts/SE_largeCNV/getwithinlibrarynorm_query.R ${WRKDIR}/iCov/${COHORT_ID}.physical.cov_matrix.bed ${COHORT_ID} ${WRKDIR}
# while read ID bam sex; do
#   bsub -q normal -M 6000 -R 'rusage[mem=6000]' -v 10000 -sla miket_sc -u nobody -o ${OUTDIR}/logs/DNAcopy.log -e ${OUTDIR}/logs/DNAcopy.log -J ${COHORT_ID}_DNAcopy "${liWGS_SV}/scripts/SE_largeCNV/largescaleCNVpipeline.sh ${samples_list} ${params} ${ID}"
# done < ${samples_list}

#Load cnMOPS modules
module unload R/3.1.0
module load R/R-3.0.0

#Submit cnMOPS - autosomes
mkdir ${WRKDIR}/cnMOPS
cov=${WRKDIR}/iCov/${COHORT_ID}.physical.cov_matrix.bed
for contig in $( seq 1 22 ); do
  mkdir ${WRKDIR}/cnMOPS/${contig}
  cat <( head -n1 ${cov} ) <( awk -v chr=${contig} '{ if ($1==chr) print }' ${cov} ) > ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.bed
  for binsize in 1 3 10 30; do
    bsub -q big -M 30000 -sla miket_sc -u nobody -o ${OUTDIR}/logs/cnMOPS.log -e ${OUTDIR}/logs/cnMOPS.log -R 'rusage[mem=30000]' -v 40000 -J ${COHORT_ID}_cnMOPS "module load R/R-3.0.0; Rscript ${liWGS_SV}/scripts/cnMOPS_postcoverage.R -m insert -r ${binsize} -b ${binsize}000 -I ${COHORT_ID} ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.bed ${WRKDIR}/cnMOPS/${contig}/"
  done
done

#Submit cnMOPS - allosomes
if [ ${other_assign} == "MALE" ]; then
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="M" || $14=="O") print $1 }' > ${WRKDIR}/males.list
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="F") print $1 }' > ${WRKDIR}/females.list
else
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="M") print $1 }' > ${WRKDIR}/males.list
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="F" || $14=="O") print $1 }' > ${WRKDIR}/females.list
fi
Midx=$( echo "$( awk -v OFS="\t" '{ print NR, $1 }' ${samples_list} | fgrep -wf ${WRKDIR}/males.list | awk '{ print ($1)+3 }' )" | cat <( echo -e "1\n2\n3" ) - | paste -s -d, )
Fidx=$( echo "$( awk -v OFS="\t" '{ print NR, $1 }' ${samples_list} | fgrep -wf ${WRKDIR}/females.list | awk '{ print ($1)+3 }' )" | cat <( echo -e "1\n2\n3" ) - | paste -s -d, )
for contig in X Y; do
  mkdir ${WRKDIR}/cnMOPS/${contig}
  cat <( head -n1 ${cov} ) <( awk -v chr=${contig} '{ if ($1==chr) print }' ${cov} ) | cut -f${Midx} > ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.M.bed
  cat <( head -n1 ${cov} ) <( awk -v chr=${contig} '{ if ($1==chr) print }' ${cov} ) | cut -f${Fidx} > ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.F.bed
  for binsize in 1 3 10 30; do
    bsub -u nobody -q big -M 30000 -o ${OUTDIR}/logs/cnMOPS.log -e ${OUTDIR}/logs/cnMOPS.log -sla miket_sc -u rlc47 -R 'rusage[mem=30000]' -v 40000 -J ${COHORT_ID}_cnMOPS "Rscript ${liWGS_SV}/scripts/cnMOPS_postcoverage.R -m insert -r ${binsize} -b ${binsize}000 -I ${COHORT_ID}_M ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.M.bed ${WRKDIR}/cnMOPS/${contig}/"
    bsub -u nobody -q big -M 30000 -o ${OUTDIR}/logs/cnMOPS.log -e ${OUTDIR}/logs/cnMOPS.log -sla miket_sc -u rlc47 -R 'rusage[mem=30000]' -v 40000 -J ${COHORT_ID}_cnMOPS "Rscript ${liWGS_SV}/scripts/cnMOPS_postcoverage.R -m insert -r ${binsize} -b ${binsize}000 -I ${COHORT_ID}_F ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.F.bed ${WRKDIR}/cnMOPS/${contig}/"
  done
done

#Gate until complete; 20 sec check; 5 min report
GATEcount=$( bjobs -w | awk '{ print $7 }' | grep -e "${COHORT_ID}_cnMOPS" | wc -l )
GATEwait=0
until [[ $GATEcount == 0 ]]; do
  sleep 20s
  GATEcount=$( bjobs -w | awk '{ print $7 }' | grep -e "${COHORT_ID}_cnMOPS" | wc -l )
  GATEwait=$[${GATEwait} +1]
  if [[ $GATEwait == 15 ]]; then
    echo "$(date): INCOMPLETE"
    echo "$(date): Waiting on ${GATEcount} jobs to complete"
    GATEwait=0<commit_msg>Various bug fixes<commit_after>#!/bin/bash

#liWGS-SV Pipeline: Module 4 (Physical Depth CNV Calling)
#August 2015
#Contact: <EMAIL>

#Read input
samples_list=$1
params=$2

#Source params file
. ${params}
cd ${WRKDIR}

#Update on Nov 30, 2015: No longer run DNAcopy by default b/c too memory intensive, too slow, and not highly informative

# #Load necessary modules for SE large CNV caller
# module load intel_parallel_xe/xe
# module load R/3.1.0
# Rscript -e "if(\"DNAcopy\" %in% rownames(installed.packages()) == FALSE){source(\"http://bioconductor.org/biocLite.R\"); biocLite(\"DNAcopy\")}; suppressPackageStartupMessages(library(DNAcopy))"
module rm gcc/4.9.0
module rm gcc-4.4
module load gcc/4.9.0

# #Submit SE large CNV caller
# Rscript ${liWGS_SV}/scripts/SE_largeCNV/getwithinlibrarynorm_query.R ${WRKDIR}/iCov/${COHORT_ID}.physical.cov_matrix.bed ${COHORT_ID} ${WRKDIR}
# while read ID bam sex; do
#   bsub -q normal -M 6000 -R 'rusage[mem=6000]' -v 10000 -sla miket_sc -u nobody -o ${OUTDIR}/logs/DNAcopy.log -e ${OUTDIR}/logs/DNAcopy.log -J ${COHORT_ID}_DNAcopy "${liWGS_SV}/scripts/SE_largeCNV/largescaleCNVpipeline.sh ${samples_list} ${params} ${ID}"
# done < ${samples_list}

#Load cnMOPS modules
module unload R/3.1.0
module load R/R-3.0.0

#Submit cnMOPS - autosomes
mkdir ${WRKDIR}/cnMOPS
cov=${WRKDIR}/iCov/${COHORT_ID}.physical.cov_matrix.bed
for contig in $( seq 1 22 ); do
  mkdir ${WRKDIR}/cnMOPS/${contig}
  cat <( head -n1 ${cov} ) <( awk -v chr=${contig} '{ if ($1==chr) print }' ${cov} ) > ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.bed
  for binsize in 1 3 10 30; do
    bsub -q big -M 30000 -sla miket_sc -u nobody -o ${OUTDIR}/logs/cnMOPS.log -e ${OUTDIR}/logs/cnMOPS.log -R 'rusage[mem=30000]' -v 40000 -J ${COHORT_ID}_cnMOPS "module load R/R-3.0.0; Rscript ${liWGS_SV}/scripts/cnMOPS_postcoverage.R -m insert -r ${binsize} -b ${binsize}000 -I ${COHORT_ID} ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.bed ${WRKDIR}/cnMOPS/${contig}/"
  done
done

#Submit cnMOPS - allosomes
if [ ${other_assign} == "MALE" ]; then
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="M" || $14=="O") print $1 }' > ${WRKDIR}/males.list
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="F") print $1 }' > ${WRKDIR}/females.list
else
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="M") print $1 }' > ${WRKDIR}/males.list
  fgrep -v "#" ${OUTDIR}/QC/cohort/${COHORT_ID}.QC.metrics | awk '{ if ($14=="F" || $14=="O") print $1 }' > ${WRKDIR}/females.list
fi
Midx=$( echo "$( awk -v OFS="\t" '{ print NR, $1 }' ${samples_list} | fgrep -wf ${WRKDIR}/males.list | awk '{ print ($1)+3 }' )" | cat <( echo -e "1\n2\n3" ) - | paste -s -d, )
Fidx=$( echo "$( awk -v OFS="\t" '{ print NR, $1 }' ${samples_list} | fgrep -wf ${WRKDIR}/females.list | awk '{ print ($1)+3 }' )" | cat <( echo -e "1\n2\n3" ) - | paste -s -d, )
for contig in X Y; do
  rm -rf ${WRKDIR}/cnMOPS/${contig}
  mkdir ${WRKDIR}/cnMOPS/${contig}
  cat <( head -n1 ${cov} ) <( awk -v chr=${contig} '{ if ($1==chr) print }' ${cov} ) | cut -f${Midx} > ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.M.bed
  cat <( head -n1 ${cov} ) <( awk -v chr=${contig} '{ if ($1==chr) print }' ${cov} ) | cut -f${Fidx} > ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.F.bed
  for binsize in 1 3 10 30; do
    bsub -u nobody -q big -M 30000 -o ${OUTDIR}/logs/cnMOPS.log -e ${OUTDIR}/logs/cnMOPS.log -sla miket_sc -u rlc47 -R 'rusage[mem=30000]' -v 40000 -J ${COHORT_ID}_cnMOPS "Rscript ${liWGS_SV}/scripts/cnMOPS_postcoverage.R -m insert -r ${binsize} -b ${binsize}000 -I ${COHORT_ID}_M ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.M.bed ${WRKDIR}/cnMOPS/${contig}/"
    bsub -u nobody -q big -M 30000 -o ${OUTDIR}/logs/cnMOPS.log -e ${OUTDIR}/logs/cnMOPS.log -sla miket_sc -u rlc47 -R 'rusage[mem=30000]' -v 40000 -J ${COHORT_ID}_cnMOPS "Rscript ${liWGS_SV}/scripts/cnMOPS_postcoverage.R -m insert -r ${binsize} -b ${binsize}000 -I ${COHORT_ID}_F ${WRKDIR}/cnMOPS/${contig}/${COHORT_ID}.rawCov.chr${contig}.F.bed ${WRKDIR}/cnMOPS/${contig}/"
  done
done

#Gate until complete; 20 sec check; 5 min report
GATEcount=$( bjobs -w | awk '{ print $7 }' | grep -e "${COHORT_ID}_cnMOPS" | wc -l )
GATEwait=0
until [[ $GATEcount == 0 ]]; do
  sleep 20s
  GATEcount=$( bjobs -w | awk '{ print $7 }' | grep -e "${COHORT_ID}_cnMOPS" | wc -l )
  GATEwait=$[${GATEwait} +1]
  if [[ $GATEwait == 15 ]]; then
    echo "$(date): INCOMPLETE"
    echo "$(date): Waiting on ${GATEcount} jobs to complete"
    GATEwait=0
{'fraction_non_alphanumeric': 0.21230936819172114, 'fraction_numerical': 0.03616557734204793, 'mean_word_length': 4.0695748205411375, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2926450', 'n_tokens_mistral': 4239, 'n_tokens_neox': 3894, 'n_words': 914}