<commit_before>################################################################
#
#
# Pipeline pipeline_IDR.py configuration file for sphinxreport
#
# 
################################################################
## general options
[general]

# the genome to use
genome=hg19

# directory where exported data is located
exportdir=export

# location of indexed genome 
genome_dir=/ifs/mirror/genomes/plain

# scratchdir for data not to be backed up
scratchdir=/tmp

# directory for publishing on the web
web_dir=../web

# a version string
version=

# Should be 1 if you have input bam files
input=1
################################################################
#
# Location of annotation database
#
################################################################
[annotations]
database=/ifs/mirror/annotations/hg19_ensembl75/csvdb

# directory with annotation information
dir=/ifs/mirror/annotations/hg19_ensembl75

[IDR]
###NOTE###
# MACS2 IS RECOMDNDED FOR IDR BY AUTHORS SO CURRENTLY ONLY THIS IS IMPLIMENTED WITH IDR
#
# MANY IDR SETTINGS ARE SPECIFIC TO THE PEAK CALLER USED
# PLEASE CONFIGURE THESE SETTINGS IN THE APPROPRIATE PEAKCALLER SPECIFIC SECTION BELOW
##########

# this sources the python3 conda environment - this is required to run the
# IDR software
# outside of CGAT a different command would be needed
sourcecommand=PATH=/ifs/apps/conda-py35/envs/idr-2.0.3/bin;CONDA_PREFIX=/ifs/apps/conda-py35/envs/idr-2.0.3;CONDA_PREFIX=/ifs/apps/conda-py35/envs/idr-2.0.3
unsourcecommand=

# 1 = run IDR, 0 = don't run IDR
run=1

# seed for randomly allocating reads to pseudo bam files
randomseed=100

# if input has low read depth, it can be better to pool either
# all inputs or all inputs for each condition / tissue
# specify "all" to pool all, "none" to use inputs as in the design table
# or "condition" to pool inputs by condition
# see https://groups.google.com/forum/#!msg/idr-discuss/Vp3mlWOTgJ4/veaM-BHe7-kJ;context-place=searchin/idr-discuss/replicate$20input%7Csort:relevance
poolinputs=none

# turns on the --peak-list parameter for the IDR
# this will be filled in automatically with peaks called from pooled bam
# files generated for each condition and tissue combination
# usually this should be 0 - oracle peak lists are only used when
# the peaks are not well-defined
useoracle=1

# --rank and --input-file-type should be provided in the peakcalling
# section and selected for the specific peakcaller you are running
# --samples and --plot are always set
# --random-seed is taken from above (the same random seed is used to
# generate the pseudo bam files

# general options to use for all IDR combinations
options=--use-best-multisummit-IDR

# soft_thresh sets the --soft-idr-threshold parameter which should
# be optimised for each part of the IDR analysis

# self consistency is consistency between pseudo replicates within a replicate
# pooled consistency is consistency between pseudo replicates generated from all pooled samples
# replicate consistency is consistency between replicates within a condition and tissue

# separate options can be provided for each of these scenarios
softthresh_selfconsistency=0.05
options_selfconsistency=

softthresh_pooledconsistency=0.05
options_pooledconsistency=

softthresh_replicateconsistency=0.05
options_replicateconsistency=

[filters]
# implemented:
# unmapped - remove unmapped reads - samtools -F 4
# unpaired - remove unpaired and not properly paired - samtools -f 1 -f 2 and remove any secondary mapped reads
# duplicates - remove duplicated reads, Picard MarkDuplicates
# lowqual - remove low scoring rats with samtools -bQ - minimum score set below
bamfilters=unmapped,secondary,unpaired,duplicates,lowqual

# minimum mapping quality (MAPQ) of reads to take to the next stage - depends on the mapper used
# integer - all MAPQ values *below* (not including) this value will be excluded! 
qual=40

# comma seperated list of bed files of blacklisted regions
bedfiles=/ifs/mirror/blacklists/hg19-ENCODE-blacklist.bed.gz
# minimum overlap with a blacklisted region to exclude a read or read pair (default
# 1E-9 (one base)
blacklistthresh=1E-9

# if 1 keep intermediate files (in filtering) - generates lots of bams, for
# troubleshooting on small test files only
keepint=0

# strip sequence from bams
strip=1

[insert]
# Insert size estimation parameter - number of alignments - see Bamtools docs
alignments=10000
macs2opts=--mfold 10 30 --gsize hs

[peakcalling]
# comma seperated list of peakcallers to use
# NOTE: If IDR is set peakcalling is repeated multiple times - do
# not specifiy too many peakcallers
# NOTE: Not all peakcallers are recommended for IDR
# Only macs2 is currently implemented
# encode recommend SPP, Gem and PeakSeq - these should be implemented in the future
peakcallers=sicer

# one of the above peakcallers can be used for IDR
idrpeakcaller=macs2

#######################################################
#######################################################
#######################################################
# options for running macs2
# These come into effect if macs2 is chosen as peak caller
#######################################################
[macs2]
contigsfile=/ifs/mirror/annotations/hg19_ensembl75_hierarchical/assembly.dir/contigs.tsv

# Specify whether options include running with --broad --broad-cutoff
broad_peak=0

# options for macs
# --bw: bandwidth (sonication size) - used for peak shape estimation. MACS
#       slides 2*bw windows over the genome and selects 1000 with the highest
#	fold enrichment to determine the tag shift size.
#	default = 300
# see [macs] section above for gsize advice
# note that macs2 now prefers a default qvalue of 0.01
options= --mfold 10 30 --gsize hs 

# maximum q-value to accept from MACS - the minimum is 0.01 (this options sets --qvalue)
max_qvalue=0.01

# p-value to accept from MACS (note: as -10log10, use min)
#min_pvalue=0

# minimum fold change
min_fold=0

# If set to true, use single end mode of macs2 even if
# data is paired end. In single end mode, only the 5' ends
# of a read pair will be used similar to unpaired data.
# In paired end mode, the peak width
# will be inferred from the insert lengths.
force_single_end=0

# suffix for the output file to use as IDR input (recommended: use narrowPeak for MACS2) else specify broadPeak
idrsuffix=narrowPeak

# column from idr input to use to rank peaks (recommended: use 8 for MACS2)
idrcol=8

# name of the column above
idrcolname=p.value

# keep this many of the top called peaks for IDR analysis
# the suggested number depends on the peak caller used
idrkeeppeaks=125000


#######################################################
#######################################################
#######################################################
# options for running SICER
# These come into effect if SICER is chosen as peak caller
# Please note that two separate SICER analyses are run
# to allow for simultaneous calling of narrower and 
# broader regions
#######################################################

[sicer]

# fdr threshold
fdr_threshold=0.05

# evalue threshold (if no control file)
evalue_threshold=100

# reduncancy threshold<commit_msg>update pipeline.ini with contigs to remove<commit_after>################################################################
#
#
# Pipeline pipeline_IDR.py configuration file for sphinxreport
#
# 
################################################################
## general options
[general]

# the genome to use
genome=mm10
# directory where exported data is located
exportdir=export

# location of indexed genome 
genome_dir=/ifs/mirror/genomes/index

# scratchdir for data not to be backed up
scratchdir=/tmp

# directory for publishing on the web
web_dir=../web

# a version string
version=

# Should be 1 if you have 'input' bam files (e.g. ChIP input control)
input=0
################################################################
#
# Location of annotation database
#
################################################################
[annotations]
database=/ifs/mirror/annotations/mm10_ensembl81/csvdb

# directory with annotation information
dir=/ifs/mirror/annotations/mm10_ensembl81


################################################################
#
# IDR options
#
###NOTE###
# MACS2 IS RECOMDNDED FOR IDR BY AUTHORS SO CURRENTLY ONLY THIS IS IMPLIMENTED 
# WITH IDR
#
# MANY IDR SETTINGS ARE SPECIFIC TO THE PEAK CALLER USED - PLEASE CONFIGURE 
# THESE SETTINGS IN THE APPROPRIATE PEAKCALLER SPECIFIC SECTION BELOW
#
################################################################
[IDR]

#Run IDR (0= no IDR)
run=0

# this sources the python3 conda environment - this is required to run the
# IDR software
# outside of CGAT a different command would be needed
sourcecommand=PATH=/ifs/apps/conda-py35/envs/idr-2.0.3/bin;CONDA_PREFIX=/ifs/apps/conda-py35/envs/idr-2.0.3;CONDA_PREFIX=/ifs/apps/conda-py35/envs/idr-2.0.3
unsourcecommand=

# set seed for randomly allocating reads to pseudo bam files for reproducibility
randomseed=100


# if input has low read depth, it might be better to pool all inputs or
# all inputs for each condition/tissue.
# see https://groups.google.com/forum/#!msg/idr-discuss/Vp3mlWOTgJ4/veaM-BHe7-kJ;context-place=searchin/idr-discuss/replicate$20input%7Csort:relevance
#   "all" = to pool all
#   "none" =  use inputs as specified in the design table
#   "condition" to pool inputs by condition
#poolinputs=none,all,condition 
poolinputs=none

# turn on the --peak-list parameter for the IDR
# this is be filled in automatically with peaks called from pooled-bam
# files generated for each condition & tissue combination
# Usually this should be 0 - oracle peak lists are only used when
# the peaks are not well-defined e.g. poor sample or broadpeaks
useoracle=0

# --rank and --input-file-type should be provided in the peakcalling
# section and selected for the specific peakcaller you are running
# --samples and --plot are always set
# --random-seed is taken from above (the same random seed is used to
# generate the pseudo bam files

# general options to use for all IDR combinations
options=--use-best-multisummit-IDR

# soft_thresh sets the --soft-idr-threshold parameter which should
# be optimised for each part of the IDR analysis

# self consistency is consistency between pseudo replicates within a replicate
# pooled consistency is consistency between pseudo replicates generated from all pooled samples
# replicate consistency is consistency between replicates within a condition and tissue

# separate options can be provided for each of these scenarios
softthresh_selfconsistency=0.05
options_selfconsistency=

softthresh_pooledconsistency=0.05
options_pooledconsistency=

softthresh_replicateconsistency=0.05
options_replicateconsistency=

############################################################################
#
# Filter Bams Settings
#
############################################################################
[filters]


# the following filters are implemented:
#    duplicates - remove duplicated reads, Picard MarkDuplicates
#    unmapped - remove unmapped reads - samtools -F 4
#    secondary - remove secondary aligned reads
#    unpaired - remove unpaired and not properly paired -
#                     samtools -f 1 -f 2 and remove any secondary mapped reads
#    lowqual - remove reads based on MAPQ with samtools -bQ 
#    contigs - remove all reads mapping to certain contigs

bamfilters=unmapped,secondary,unpaired,duplicates,lowqual,contigs

# minimum mapping quality (MAPQ) of reads to take to the next stage
# note this depends on the mapper used
# integer - all MAPQ values *below* (not including) this value will be excluded! 
qual=30

# comma seperated list of bed files of blacklisted regions
bedfiles=/ifs/mirror/blacklists/mm10-combined-kundaje-greenleaf-blacklist.bed.gz
# minimum overlap with a blacklisted region to exclude a read or read pair (default
# 1E-9 (one base)
blacklistthresh=1E-9

#contigs to remove from bams
# ote for ATAC-seq probably want to remove chrM
contigs_to_remove=_alt|_hap|chrM|_random

# if 1 keep intermediate files (in filtering) - generates lots of bams, for
# troubleshooting on small test files only
keepint=0

# strip sequence from bams
strip=1

#######################################################
#
# Peakcalling options
#
#######################################################

[insert]
# Insert size estimation parameter - number of alignments - see Bamtools docs
alignments=10000
macs2opts=--mfold 10 30 --gsize mm

[peakcalling]
# comma seperated list of peakcallers to use
# NOTE: If IDR is set peakcalling is repeated multiple times - do
# not specifiy too many peakcallers
# NOTE: Not all peakcallers are recommended for IDR
# Only macs2 is currently implemented for IDR
# encode recommend SPP, Gem and PeakSeq also- these should be implemented in the future
# Peakcallers:
#           -macs2
#           -sicer
peakcallers=macs2

# one of the above peakcallers can be used for IDR (currently only macs2)
idrpeakcaller=macs2




#######################################################
# options for running macs2
# These come into effect if macs2 is chosen as peak caller
#######################################################
[macs2]
contigsfile=/ifs/mirror/annotations/mm10_ensembl81/assembly.dir/contigs.tsv

# Specify whether options include running with --broad --broad-cutoff
broad_peak=0

# options for macs
# --bw: bandwidth (sonication size) - used for peak shape estimation. MACS
#       slides 2*bw windows over the genome and selects 1000 with the highest
#	fold enrichment to determine the tag shift size.
#	default = 300
# see [macs] section above for gsize advice
# note that macs2 now prefers a default qvalue of 0.01
# --mfold 10 30 --gsize mm
options= --nomodel --nolambda --call-summits --keep-dup all --gsize mm 

# maximum q-value to accept from MACS - the minimum is 0.01 (this options sets --qvalue)
max_qvalue=0.01

# p-value to accept from MACS (note: as -10log10, use min)
#min_pvalue=0

# minimum fold change
min_fold=0

# If set to true, use single end mode of macs2 even if
# data is paired end. In single end mode, only the 5'' ends.
# of a read pair will be used similar to unpaired data.
# In paired end mode, the peak width
# will be inferred from the insert lengths.
force_single_end=0

# BELOW OPTIONS ARE ONLY NEEDED IF USING IDR
# suffix for the output file to use as IDR input (recommended: use narrowPeak for MACS2) else specify broadPeak
idrsuffix=narrowPeak

# column from idr input to use to rank peaks (recommended: use 8 for MACS2)
idrcol=8

# name of the column above
idrcolname=p.value

# keep this many of the top called peaks for IDR analysis
# the suggested number depends on the peak caller used
idrkeeppeaks=125000



#######################################################
# options for running SICER
# These come into effect if SICER is chosen as peak caller
# Please note that two separate SICER analyses are run
# to allow for simultaneous calling of narrower and 
# broader regions
#######################################################

[sicer]

# fdr threshold
fdr_threshold=0.05

# evalue threshold (if no control file)
evalue_threshold=100

# reduncancy threshold
{'fraction_non_alphanumeric': 0.16078919769271108, 'fraction_numerical': 0.017566858940744626, 'mean_word_length': 4.9250485436893205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5445853', 'n_tokens_mistral': 4557, 'n_tokens_neox': 4197, 'n_words': 1742}