<commit_before>#!/bin/bash
#call this pipeline something unique, replace spaces with underlines
name="junjun_fluidigm"

#(1) single, (2) pooled, (3) both
single=1

#contact info for pipeline completion, doesnt work
email=""

#default freebayes parameters or not: (3) custom (not implemented), (1) default, (0) paper
default=1

#maf cutoff 
mafcut=0.3

# paired files (e.g. R1 & R2) vs unpaired files: (1) paired, (0) unpaired
paired=1

#ploidy values for single and pooled runs: (1) haploid, (2) diploid
singlep=1
pooledp=2


# (1) run the full pipeline, (2) just process the data and do not generate reports (ie. just run the first half of the pipeline), (3) just generate reports based on data that has already been processed by the first half of the pipeline (ie. just run the second half of the pipeline assuming the first half has already been run).
what_to_run=1

# some optional reports which you might choose not to generate in order to save time:

# (1) yes, (0) no
generate_chi_sq_report=1

# (1) yes, (0) no
generate_probability_report=0

# (1) yes, (0) no
generate_depth_stats_report=1

# Scroll down to see the code.
























########################################################################
# function formatPairedFileNames()
# This gets the list of the R1 and R2 raw data input file names, chops  
# off the end portion according to the various allowed input file types, 
# and returns in the $datapoints variable the unique id portion of the
# file names, after eliminating adjacent duplicates due to the R1 and
# R2 files sharing the same base name.
# Functions in bash are a little crazy, so the input parameter needs to 
# be $datapoints. So you'd call it like:
# datapoints=""
# formatFileNames $datapoints
formatPairedFileNames()
{
datapoints=$(ls "./data" \
              | sed "s/_R1.fastq.gz//g" \
              | sed "s/_R2.fastq.gz//g" \
              | sed "s/_R1.fastq//g" \
              | sed "s/_R2.fastq//g" \
              | sed "s/_R1.fasta//g" \
              | sed "s/_R2.fasta//g" \
              | sed "s/_R1.mfa//g" \
              | sed "s/_R2.mfa//g" \
              | sed "s/_R1.fna//g" \
              | sed "s/_R2.fna//g" \
              | sed "s/_R1.fa//g" \
              | sed "s/_R2.fa//g" \
              | uniq)
}




if [ $single -eq 3 ]; then<commit_msg>Added a few more .gz input file formats for paired data.<commit_after>#!/bin/bash
#call this pipeline something unique, replace spaces with underlines
name="junjun_fluidigm"

#(1) single, (2) pooled, (3) both
single=1

#contact info for pipeline completion, doesnt work
email=""

#default freebayes parameters or not: (3) custom (not implemented), (1) default, (0) paper
default=1

#maf cutoff 
mafcut=0.3

# paired files (e.g. R1 & R2) vs unpaired files: (1) paired, (0) unpaired
paired=1

#ploidy values for single and pooled runs: (1) haploid, (2) diploid
singlep=1
pooledp=2


# (1) run the full pipeline, (2) just process the data and do not generate reports (ie. just run the first half of the pipeline), (3) just generate reports based on data that has already been processed by the first half of the pipeline (ie. just run the second half of the pipeline assuming the first half has already been run).
what_to_run=3

# some optional reports which you might choose not to generate in order to save time:

# (1) yes, (0) no
generate_chi_sq_report=1

# (1) yes, (0) no
generate_probability_report=0

# (1) yes, (0) no
generate_depth_stats_report=1

# Scroll down to see the code.
























########################################################################
# function formatPairedFileNames()
# This gets the list of the R1 and R2 raw data input file names, chops  
# off the end portion according to the various allowed input file types, 
# and returns in the $datapoints variable the unique id portion of the
# file names, after eliminating adjacent duplicates due to the R1 and
# R2 files sharing the same base name.
# Functions in bash are a little crazy, so the input parameter needs to 
# be $datapoints. So you'd call it like:
# datapoints=""
# formatFileNames $datapoints
formatPairedFileNames()
{
datapoints=$(ls "./data" \
              | sed "s/_R1.fastq.gz//g" \
              | sed "s/_R2.fastq.gz//g" \
              | sed "s/_R1.fasta.gz//g" \
              | sed "s/_R2.fasta.gz//g" \
              | sed "s/_R1.mfa.gz//g" \
              | sed "s/_R2.mfa.gz//g" \
              | sed "s/_R1.fna.gz//g" \
              | sed "s/_R2.fna.gz//g" \
              | sed "s/_R1.fa.gz//g" \
              | sed "s/_R2.fa.gz//g" \
              | sed "s/_R1.fastq//g" \
              | sed "s/_R2.fastq//g" \
              | sed "s/_R1.fasta//g" \
              | sed "s/_R2.fasta//g" \
              | sed "s/_R1.mfa//g" \
              | sed "s/_R2.mfa//g" \
              | sed "s/_R1.fna//g" \
              | sed "s/_R2.fna//g" \
              | sed "s/_R1.fa//g" \
              | sed "s/_R2.fa//g" \
              | uniq)
}




if [ $single -eq 3 ]; then
{'fraction_non_alphanumeric': 0.14299940250946028, 'fraction_numerical': 0.021111332403903605, 'mean_word_length': 2.409368635437882, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2902388', 'n_tokens_mistral': 1707, 'n_tokens_neox': 1633, 'n_words': 587}