<commit_before>#! /bin/bash

# runs the analysis pipeline
source config.sh

# define input variables
inputPath=$1 # path to directory containing all tag directories
outputDir=$2 # path to the output directory
stepOne=$3 # run basic analysis for each tag directory
stepTwo=$4 # mege peaks and look for overlapping cistromes
stepThree=$5 # conduct differential motif analysis on overlapping cistromes

if [ ! -d $outputDir ]
then
mkdir $outputDir
fi



### basic analysis for each the peaks and define cistromes ###
if $stepOne
then
	# validation and update if necessary
	for path in $inputPath/*
	do
		[ -e "${path}/tagInfo.txt" ] || continue # if not a directory, skip
		# if mm9 is specified, it should appear in tagInfo.txt at least once
		mm9=$(grep $genome $path/tagInfo.txt | wc -l)
		if [ "$mm9" == "0" ]
		then
			echo "WARNING: the following tag directory does not use the genome ${genome}: $path"
		fi
		# update tag directories if option is on
		if $updateTags
		then
			command="makeTagDirectory ${path} -update -genome mm9"
			echo $command
			$($command)
		fi
	done

	#UCSC visualization (makeUCSCfile, makeBigWig.pl)
	if $makeUCSC
	then
		echo "making UCSC multibig wig file"
		# iterate over all tag directories and make a hub	
		command="makeMultiWigHub.pl $hub $genome -force -d"
		for path in $inputPath/*
		do
   			[ -d "${path}" ] || continue # if not a directory, skip
			# skip directories that contain input
			if [[ ! $path  =~ .*[iI]{1}nput.* ]]
			then
				if [[ $(basename $path) =~ ^[0-9]-.* ]]
				then
					command+=" "
					# append tag directory to command
					command+="$path"
				fi
			fi
		done
		echo "$command"
		$($command)
	fi
	#Peak finding / Transcript detection / Feature identification (findPeaks)
	if $findPeaks 
	then
		echo "finding peaks"
		rm -rf $outputDir/inputPeaks
		mkdir $outputDir/inputPeaks
	
		fileHeadings=()
		for path in $inputPath/*
		do
			[ -d "${path}" ] || continue # if not a directory, skip
			basepath=$(basename $path)
			if [[ $basepath =~ ^[0-9]-.* ]]; 
			then
				fileHeadings+=(${basepath%%-*})
			fi          
		done
		# get the list of unique headings
		uniqueHeadings=($(for v in "${fileHeadings[@]}"; do echo "$v";done| sort| uniq| xargs))
		for heading in ${uniqueHeadings[@]}
		do
			files=( $(find $inputPath/$heading* -maxdepth 0) )
			control=''
			actual=''
			if [[ ${files[0]} =~ .*[iI]{1}nput.* ]]
			then
				control=${files[0]}
				actual=${files[1]}
			else
				control=${files[1]}
				actual=${files[0]}
			fi  
				command="findPeaks "
				command+=" "$actual
				command+=" -style factor -o $outputDir/$(basename ${actual})_peaks.tsv"
				if $useControl
				then
					command+=" -i $control"
				fi
				echo $command
				$($command)
				inputCommand="findPeaks $control -style factor -o $outputDir/inputPeaks/$(basename ${control})_peaks.tsv"
				echo $inputCommand
				$($inputCommand)
		done
	fi

	#Motif analysis (findMotifsGenome.pl)
	if $findMotifs
	then
		echo "finding motifs"
		mkdir $outputDir/factorMotifs
		for path in $outputDir/*_peaks.tsv
		do
   			[ -f "${path}" ] || continue
			command="findMotifsGenome.pl "
			command+=" "$path
			# run commands in the background
			outPath=$path
			outPath=${outPath%_peaks.tsv}
			outPath=${outPath##*/}_motifs
			command+=" $genome ${outputDir}/factorMotifs/${outPath}"
			echo $command
			$($command)
		done

	fi
	#Annotation of Peaks (annotatePeaks.pl)
	if $annotatePeaks
	then
		echo "annotating peaks"
		for path in $outputDir/*_peaks.tsv
		do
   			[ -f "${path}" ] || continue
			command="annotatePeaks.pl"
			command+=" "$path
			# run commands in the background
			outPath=$path
			outPath=${outPath%_peaks.tsv}
			outPath=${outPath##*/}_annotatedPeaks.tsv
			#command+=" $genome > ${outputDir}/${outPath}"
			command+=" $genome"
			echo $command
			$($command > ${outputDir}/${outPath})
		done
		# annotate merged regions with all tag directories and all motifs
		command="annotatePeaks.pl $outputDir/merged.tsv mm9 -size 200 -d"
		for path in $inputPath/*
		do
			[ -d "${path}" ] || continue # if not a directory, skip
			# skip directories that contain input
			if [[ ! $path  =~ .*[iI]{1}nput.* ]]
			then
				# consider only files tagged with a number at the front
				if [[ $(basename $path) =~ ^[0-9]-.* ]]
				then
					command+=" "
					# append tag directory to command
					command+="$path"
				fi
			fi
	done
	command+=" -m $outputDir/mergedMotifs/homerMotifs.all.motifs"
	echo $command
	$command > $outputDir/merged_annotated.tsv

	fi
	#Quantification of Data at Peaks/Regions in the Genome/Histograms and Heatmaps (annotatePeaks.pl)
	if $quantifyPeaks 
	then
		echo "quantifying peaks"
		# not sure what I need to do here, this will evolve later on
		# the same script is used for annotating peaks, and so this can be wrapped in the previous section
	fi
fi

### merge the peaks and look for overlapping cistromes ##
if $stepTwo
then
	echo "calculating overlapping cistromes"
	echo "filtering peaks"
	# filter peaks summarize peak score and tag counts of each file
	for path in $outputDir/*_peaks.tsv
	do
		[ -f "${path}" ] || continue
		filteredPath=${path%_peaks.tsv}
		filteredPath+="_filteredPeaks.tsv"
		echo "python filterPeaks.py $path > $filteredPath"
		python filterPeaks.py $path $percentileThreshold > $filteredPath
		outpath=${path%_peaks.tsv}
		outpath+="_unfiltered"
		echo "python plotPeakScores.py $path $outpath"
		python plotPeakScores.py $path $filteredPath $outpath
	done
	echo "extending peaks"
	# iterate through each peak file and modify the start and the end
	for path in $outputDir/*_annotatedPeaks.tsv
	do
		[ -f "${path}" ] || continue
		outPath=$path
		outPath=${outPath%_annotatedPeaks.tsv}
		outPath=${outPath##*/}_ext.tsv
		echo "python extendPeaks.py $path ${outputDir}/${outPath} $overlapDistance"
		python extendPeaks.py $path ${outputDir}/${outPath} $overlapDistance

			
	done
	# call merge peaks
	command="mergePeaks -d given "
	for path in $outputDir/*_ext.tsv
	do
		command+=" $path"
	done
	echo $command
	$command > ${outputDir}/merged.tsv

	# produce bed file for visualization for merged region
	pos2bed.pl $outputDir/merged.tsv > $outputDir/peakfile.bed

	# find motifs for merged regions
	if $findMotifs
	then	
		findMotifsGenome.pl $outputDir/merged.tsv mm9 $outputDir/mergedMotifs/ -size 200
	fi
	
	# compute overlapping groups
	echo "computing stats for overlapping groups"
	python calcGroupStats.py $outputDir/merged.tsv $outputDir/merged_annotated.tsv> $outputDir/group_stats.tsv

	# create human readable file
	command="python makeSummaryFile.py"
	if [ -f $outputDir/factorNameMapping.tsv ]
	then
		command+=" 1"
	else
		command+=" 0"
	fi
	command+=" $outputDir/merged.tsv"
	command+=" $outputDir/group_stats.tsv"
	command+=" $outputDir/motif_stats.tsv"
	for path in $outputDir/*_peaks.tsv
	do
		[ -f "${path}" ] || continue
		command+=" "$path
	done
	if [ -f $outputDir/factorNameMapping.tsv ]
	then
		command+=" $outputDir/factorNameMapping.tsv" # factorNameMapping file is not created automatically!
	fi
	$command > $outputDir/group_summary.tsv
	echo $command

	# create different peak files for each group
	rm -rf $outputDir/splitPeaks
	mkdir $outputDir/splitPeaks
	python splitMergedPeaks.py $outputDir/merged_annotated.tsv $outputDir/group_stats.tsv $outputDir/splitPeaks

	# filter peaks according to threshold peak score
	

	
	# create bed files for each split group file
	for path in $outputDir/splitPeaks/groupPeaks*.tsv
	do
		[ -f "${path}" ] || continue
		outpath=$(basename $path)
		outpath=${outpath##groupPeaks}
		outpath=${outpath%%tsv}<commit_msg>now using filtered peaks<commit_after>#! /bin/bash

# runs the analysis pipeline
source config.sh

# define input variables
inputPath=$1 # path to directory containing all tag directories
outputDir=$2 # path to the output directory
stepOne=$3 # run basic analysis for each tag directory
stepTwo=$4 # mege peaks and look for overlapping cistromes
stepThree=$5 # conduct differential motif analysis on overlapping cistromes

if [ ! -d $outputDir ]
then
mkdir $outputDir
fi
# create a file summarizing the parameters used



### basic analysis for each the peaks and define cistromes ###
if $stepOne
then
	# validation and update if necessary
	for path in $inputPath/*
	do
		[ -e "${path}/tagInfo.txt" ] || continue # if not a directory, skip
		# if mm9 is specified, it should appear in tagInfo.txt at least once
		mm9=$(grep $genome $path/tagInfo.txt | wc -l)
		if [ "$mm9" == "0" ]
		then
			echo "WARNING: the following tag directory does not use the genome ${genome}: $path"
		fi
		# update tag directories if option is on
		if $updateTags
		then
			command="makeTagDirectory ${path} -update -genome mm9"
			echo $command
			$($command)
		fi
	done

	#UCSC visualization (makeUCSCfile, makeBigWig.pl)
	if $makeUCSC
	then
		echo "making UCSC multibig wig file"
		# iterate over all tag directories and make a hub	
		command="makeMultiWigHub.pl $hub $genome -force -d"
		for path in $inputPath/*
		do
   			[ -d "${path}" ] || continue # if not a directory, skip
			# skip directories that contain input
			if [[ ! $path  =~ .*[iI]{1}nput.* ]]
			then
				if [[ $(basename $path) =~ ^[0-9]-.* ]]
				then
					command+=" "
					# append tag directory to command
					command+="$path"
				fi
			fi
		done
		echo "$command"
		$($command)
	fi
	#Peak finding / Transcript detection / Feature identification (findPeaks)
	if $findPeaks 
	then
		echo "finding peaks"
		rm -rf $outputDir/inputPeaks
		mkdir $outputDir/inputPeaks
	
		fileHeadings=()
		for path in $inputPath/*
		do
			[ -d "${path}" ] || continue # if not a directory, skip
			basepath=$(basename $path)
			if [[ $basepath =~ ^[0-9]-.* ]]; 
			then
				fileHeadings+=(${basepath%%-*})
			fi          
		done
		# get the list of unique headings
		uniqueHeadings=($(for v in "${fileHeadings[@]}"; do echo "$v";done| sort| uniq| xargs))
		for heading in ${uniqueHeadings[@]}
		do
			files=( $(find $inputPath/$heading* -maxdepth 0) )
			control=''
			actual=''
			if [[ ${files[0]} =~ .*[iI]{1}nput.* ]]
			then
				control=${files[0]}
				actual=${files[1]}
			else
				control=${files[1]}
				actual=${files[0]}
			fi  
				command="findPeaks "
				command+=" "$actual
				command+=" -style factor -o $outputDir/$(basename ${actual})_peaks.tsv"
				if $useControl
				then
					command+=" -i $control"
				fi
				echo $command
				$($command)
				inputCommand="findPeaks $control -style factor -o $outputDir/inputPeaks/$(basename ${control})_peaks.tsv"
				echo $inputCommand
				$($inputCommand)
		done
	fi

	echo "filtering peaks"
	# filter peaks summarize peak score and tag counts of each file
	for path in $outputDir/*_peaks.tsv
	do
		[ -f "${path}" ] || continue
		filteredPath=${path%_peaks.tsv}
		filteredPath+="_filteredPeaks.tsv"
		echo "python filterPeaks.py $path > $filteredPath"
		python filterPeaks.py $path $percentileThreshold > $filteredPath
		outpath=${path%_peaks.tsv}
		echo "python plotPeakScores.py $path $filteredPath $outpath"
		python plotPeakScores.py $path $filteredPath $outpath
	done
	if [ -d $outputDir/peakScorePlots ]
	then
	rm -rf $outputDir/peakScorePlots
	fi
	mkdir $outputDir/peakScorePlots
	mv $outputDir/*_peakScore.png $outputDir/peakScorePlots
	mv $outputDir/*_tagCount.png $outputDir/peakScorePlots

	#Motif analysis (findMotifsGenome.pl)
	if $findMotifs
	then
		echo "finding motifs"
		mkdir $outputDir/factorMotifs
		for path in $outputDir/*_filteredPeaks.tsv
		do
   			[ -f "${path}" ] || continue
			command="findMotifsGenome.pl "
			command+=" "$path
			# run commands in the background
			outPath=$path
			outPath=${outPath%_filteredPeaks.tsv}
			outPath=${outPath##*/}_motifs
			command+=" $genome ${outputDir}/factorMotifs/${outPath}"
			echo $command
			$($command)
		done

	fi
	#Annotation of Peaks (annotatePeaks.pl)
	if $annotatePeaks
	then
		echo "annotating peaks"
		for path in $outputDir/*_filteredPeaks.tsv
		do
   			[ -f "${path}" ] || continue
			command="annotatePeaks.pl"
			command+=" "$path
			# run commands in the background
			outPath=$path
			outPath=${outPath%_filteredPeaks.tsv}
			outPath=${outPath##*/}_annotatedPeaks.tsv
			#command+=" $genome > ${outputDir}/${outPath}"
			command+=" $genome"
			echo $command
			$($command > ${outputDir}/${outPath})
		done
		# annotate merged regions with all tag directories and all motifs
		command="annotatePeaks.pl $outputDir/merged.tsv mm9 -size 200 -d"
		for path in $inputPath/*
		do
			[ -d "${path}" ] || continue # if not a directory, skip
			# skip directories that contain input
			if [[ ! $path  =~ .*[iI]{1}nput.* ]]
			then
				# consider only files tagged with a number at the front
				if [[ $(basename $path) =~ ^[0-9]-.* ]]
				then
					command+=" "
					# append tag directory to command
					command+="$path"
				fi
			fi
	done
	command+=" -m $outputDir/mergedMotifs/homerMotifs.all.motifs"
	echo $command
	$command > $outputDir/merged_annotated.tsv

	fi
	#Quantification of Data at Peaks/Regions in the Genome/Histograms and Heatmaps (annotatePeaks.pl)
	if $quantifyPeaks 
	then
		echo "quantifying peaks"
		# not sure what I need to do here, this will evolve later on
		# the same script is used for annotating peaks, and so this can be wrapped in the previous section
	fi
fi

### merge the peaks and look for overlapping cistromes ##
if $stepTwo
then
	echo "calculating overlapping cistromes"

	echo "extending peaks"
	# iterate through each peak file and modify the start and the end
	for path in $outputDir/*_filteredPeaks.tsv
	do
		[ -f "${path}" ] || continue
		outPath=$path
		outPath=${outPath%_filteredPeaks.tsv}
		outPath=${outPath##*/}_ext.tsv
		echo "python extendPeaks.py $path ${outputDir}/${outPath} $overlapDistance"
		python extendPeaks.py $path ${outputDir}/${outPath} $overlapDistance

			
	done
	# call merge peaks
	command="mergePeaks -d given "
	for path in $outputDir/*_ext.tsv
	do
		command+=" $path"
	done
	echo $command
	$command > ${outputDir}/merged.tsv

	# produce bed file for visualization for merged region
	pos2bed.pl $outputDir/merged.tsv > $outputDir/peakfile.bed

	# find motifs for merged regions
	if $findMotifs
	then	
		findMotifsGenome.pl $outputDir/merged.tsv mm9 $outputDir/mergedMotifs/ -size 200
	fi
	
	# compute overlapping groups
	echo "computing stats for overlapping groups"
	python calcGroupStats.py $outputDir/merged.tsv $outputDir/merged_annotated.tsv> $outputDir/group_stats.tsv

	# create human readable file
	command="python makeSummaryFile.py"
	if [ -f $outputDir/factorNameMapping.tsv ]
	then
		command+=" 1"
	else
		command+=" 0"
	fi
	command+=" $outputDir/merged.tsv"
	command+=" $outputDir/group_stats.tsv"
	command+=" $outputDir/motif_stats.tsv"
	for path in $outputDir/*_filteredPeaks.tsv
	do
		[ -f "${path}" ] || continue
		command+=" "$path
	done
	if [ -f $outputDir/factorNameMapping.tsv ]
	then
		command+=" $outputDir/factorNameMapping.tsv" # factorNameMapping file is not created automatically!
	fi
	$command > $outputDir/group_summary.tsv
	echo $command

	# create different peak files for each group
	rm -rf $outputDir/splitPeaks
	mkdir $outputDir/splitPeaks
	python splitMergedPeaks.py $outputDir/merged_annotated.tsv $outputDir/group_stats.tsv $outputDir/splitPeaks
	
	# create bed files for each split group file
	for path in $outputDir/splitPeaks/groupPeaks*.tsv
	do
		[ -f "${path}" ] || continue
		outpath=$(basename $path)
		outpath=${outpath##groupPeaks}
		outpath=${outpath%%tsv}
{'fraction_non_alphanumeric': 0.12340480663341323, 'fraction_numerical': 0.0046641186759085315, 'mean_word_length': 3.7604070305272894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2151166', 'n_tokens_mistral': 5939, 'n_tokens_neox': 5170, 'n_words': 1556}