<commit_before>k=(1000 10000 100000 1000000 10000000)

for IN_FILE in *.fastq; do 

    # Grab the entire number of reads once for downsampling
    NUM_READS="$(grep -c "^>" ${IN_FILE})"

    mkdir ${IN_FILE//.fastq/}

    for i in ${k[@]}; do
        # Make the directory to store down-sampling results
        test -d ${TRIAL_HOME}/hits_${i} | mkdir -p ${TRIAL_HOME}/hits_${i}
        for j in `seq 1 5`; do
            # An experiment
            python /project/flatiron/ben/projects/NINJA-DOJO/ninja_dojo/scripts/subset_fasta.py -n ${NUM_READS} -k ${i} ${IN_FILE} > ${IN_FILE//.fastq/}/i.j.${IN_FILE}
        done
    done
done<commit_msg>Added the new analysis shogun.<commit_after>k=(1000 10000 100000 1000000 10000000)

for IN_FILE in *.fastq; do 

    fastq_to_fastq -i ${IN_FILE} -o ${IN_FILE//.fastq/fna}

    for i in ${k[@]}; do
        # Make the directory to store down-sampling results
        for j in `seq 1 5`; do
            # An experiment
            subset_fasta -k ${i} -i {IN_FILE//.fastq/fna} -o ${IN_FILE//.fastq}.i.j.fna
        done
    done
done
{'fraction_non_alphanumeric': 0.1413953488372093, 'fraction_numerical': 0.059534883720930236, 'mean_word_length': 2.4050632911392404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4986955', 'n_tokens_mistral': 462, 'n_tokens_neox': 383, 'n_words': 111}