<commit_before>#!/usr/bin/env bash

# validate_mapping_file
validate_mapping_file.py \
    -m 'test-data/map.tsv' \
    -o validate_mapping_file_output \
    -c '_'
cp validate_mapping_file_output/*.html 'test-data/map.tsv.html'
cp validate_mapping_file_output/*.log 'test-data/map.tsv.log'
cp validate_mapping_file_output/*corrected.txt 'test-data/map.tsv_corrected.txt'
rm -rf validate_mapping_file_output

# split_libraries_fastq
split_libraries_fastq.py \
    --sequence_read_fps 'test-data/forward_reads.fastq' \
    -o split_libraries \
    --mapping_fps 'test-data/map.tsv' \
    --barcode_read_fps 'test-data/barcodes.fastq' \
    --store_qual_scores \
    --store_demultiplexed_fastq \
    --max_bad_run_length 3 \
    --min_per_read_length_fraction 0.75 \
    --sequence_max_n 0 \
    --start_seq_id 0 \
    --barcode_type 'golay_12' \
    --max_barcode_errors 1.5
cp split_libraries/split_library_log.txt 'test-data/split_fastq_libraries_log.txt'
cp split_libraries/histograms.txt 'test-data/split_fastq_libraries_histograms.tabular'
cp split_libraries/seqs.fna 'test-data/split_fastq_libraries_sequences.fasta'
cp split_libraries/seqs.qual 'test-data/split_fastq_libraries_sequence_qualities.qual'
cp split_libraries/seqs.fastq 'test-data/split_fastq_libraries_demultiplexed_sequences.fastq'
rm -rf split_libraries

# split_libraries
split_libraries.py \
    --map 'test-data/split_libraries_mapping_file.txt' \
    -o split_libraries \
    --fasta 'test-data/split_libraries_reads_1.fna,test-data/split_libraries_reads_2.fna' \
    --qual 'test-data/split_libraries_reads_1.qual,test-data/split_libraries_reads_2.qual' \
    --min_qual_score 25 \
    --qual_score_window 0 \
    --record_qual_scores \
    --min_seq_length 200 \
    --max_seq_length 1000 \
    --max_ambig 6 \
    --max_homopolymer 6 \
    --max_primer_mismatch 0 \
    --barcode_type 'golay_12' \
    --max_barcode_errors 1.5 \
    --start_numbering_at 1
cp split_libraries/seqs.fna 'test-data/split_libraries_seqs.fna'
cp split_libraries/split_library_log.txt 'test-data/split_libraries_split_library_log'
cp split_libraries/histograms.txt 'test-data/split_libraries_histograms.txt'
cp split_libraries/seqs_filtered.qual 'test-data/split_libraries_seqs_filtered.qual'
rm -rf split_libraries

# pick_open_reference_otus
pick_open_reference_otus.py \
    --input_fps 'test-data/split_fastq_libraries_sequences.fasta' \
    -o pick_open_reference_otus_1 \
    --reference_fp 'test-data/gg_13_8_79_otus.fasta' \
    --otu_picking_method 'uclust' \
    --new_ref_set_id 'New' \
    --parallel \
    --percent_subsample '0.001' \
    --prefilter_percent_id '0.0' \
    --minimum_failure_threshold '100000' \
    --min_otu_size '2'
cp pick_open_reference_otus_1/final_otu_map.txt 'test-data/pick_open_reference_otus_1_final_otu_map.txt'
cp pick_open_reference_otus_1/final_otu_map_mc*.txt 'test-data/pick_open_reference_otus_1_final_otu_map_mc.txt'
cp pick_open_reference_otus_1/rep_set.fna 'test-data/pick_open_reference_otus_1_rep_set.fna'
cp pick_open_reference_otus_1/rep_set.tre 'test-data/pick_open_reference_otus_1_rep_set_tree.tre'
rm -rf pick_open_reference_otus_1

pick_open_reference_otus.py \
    --input_fps 'test-data/split_fastq_libraries_sequences.fasta' \
    -o pick_open_reference_otus_2 \
    --reference_fp 'test-data/gg_13_8_79_otus.fasta' \
    --otu_picking_method 'uclust' \
    --new_ref_set_id 'New' \
    --parallel \
    --percent_subsample '0.001' \
    --prefilter_percent_id '0.0' \
    --minimum_failure_threshold '100000' \
    --min_otu_size '3' \
    --suppress_taxonomy_assignment \
    --suppress_align_and_tree
cp pick_open_reference_otus_2/final_otu_map.txt 'test-data/pick_open_reference_otus_2_final_otu_map.txt'
cp pick_open_reference_otus_2/final_otu_map_mc*.txt 'test-data/pick_open_reference_otus_2_final_otu_map_mc.txt'
rm -rf pick_open_reference_otus_2

pick_open_reference_otus.py \
    --input_fps 'test-data/split_fastq_libraries_sequences.fasta' \<commit_msg>Remove some useless copy of test data<commit_after>#!/usr/bin/env bash

# validate_mapping_file
validate_mapping_file.py \
    -m 'test-data/map.tsv' \
    -o validate_mapping_file_output \
    -c '_'
cp validate_mapping_file_output/*.html 'test-data/map.tsv.html'
cp validate_mapping_file_output/*.log 'test-data/map.tsv.log'
cp validate_mapping_file_output/*corrected.txt 'test-data/map.tsv_corrected.txt'
rm -rf validate_mapping_file_output

# split_libraries_fastq
split_libraries_fastq.py \
    --sequence_read_fps 'test-data/forward_reads.fastq' \
    -o split_libraries \
    --mapping_fps 'test-data/map.tsv' \
    --barcode_read_fps 'test-data/barcodes.fastq' \
    --store_qual_scores \
    --store_demultiplexed_fastq \
    --max_bad_run_length 3 \
    --min_per_read_length_fraction 0.75 \
    --sequence_max_n 0 \
    --start_seq_id 0 \
    --barcode_type 'golay_12' \
    --max_barcode_errors 1.5
cp split_libraries/histograms.txt 'test-data/split_fastq_libraries_histograms.tabular'
cp split_libraries/seqs.fna 'test-data/split_fastq_libraries_sequences.fasta'
cp split_libraries/seqs.qual 'test-data/split_fastq_libraries_sequence_qualities.qual'
cp split_libraries/seqs.fastq 'test-data/split_fastq_libraries_demultiplexed_sequences.fastq'
rm -rf split_libraries

# split_libraries
split_libraries.py \
    --map 'test-data/split_libraries_mapping_file.txt' \
    -o split_libraries \
    --fasta 'test-data/split_libraries_reads_1.fna,test-data/split_libraries_reads_2.fna' \
    --qual 'test-data/split_libraries_reads_1.qual,test-data/split_libraries_reads_2.qual' \
    --min_qual_score 25 \
    --qual_score_window 0 \
    --record_qual_scores \
    --min_seq_length 200 \
    --max_seq_length 1000 \
    --max_ambig 6 \
    --max_homopolymer 6 \
    --max_primer_mismatch 0 \
    --barcode_type 'golay_12' \
    --max_barcode_errors 1.5 \
    --start_numbering_at 1
cp split_libraries/seqs.fna 'test-data/split_libraries_seqs.fna'
cp split_libraries/split_library_log.txt 'test-data/split_libraries_split_library_log'
cp split_libraries/histograms.txt 'test-data/split_libraries_histograms.txt'
cp split_libraries/seqs_filtered.qual 'test-data/split_libraries_seqs_filtered.qual'
rm -rf split_libraries

# pick_open_reference_otus
pick_open_reference_otus.py \
    --input_fps 'test-data/split_fastq_libraries_sequences.fasta' \
    -o pick_open_reference_otus_1 \
    --reference_fp 'test-data/gg_13_8_79_otus.fasta' \
    --otu_picking_method 'uclust' \
    --new_ref_set_id 'New' \
    --parallel \
    --percent_subsample '0.001' \
    --prefilter_percent_id '0.0' \
    --minimum_failure_threshold '100000' \
    --min_otu_size '2'
cp pick_open_reference_otus_1/final_otu_map.txt 'test-data/pick_open_reference_otus_1_final_otu_map.txt'
cp pick_open_reference_otus_1/final_otu_map_mc*.txt 'test-data/pick_open_reference_otus_1_final_otu_map_mc.txt'
cp pick_open_reference_otus_1/rep_set.tre 'test-data/pick_open_reference_otus_1_rep_set_tree.tre'
rm -rf pick_open_reference_otus_1

pick_open_reference_otus.py \
    --input_fps 'test-data/split_fastq_libraries_sequences.fasta' \
    -o pick_open_reference_otus_2 \
    --reference_fp 'test-data/gg_13_8_79_otus.fasta' \
    --otu_picking_method 'uclust' \
    --new_ref_set_id 'New' \
    --parallel \
    --percent_subsample '0.001' \
    --prefilter_percent_id '0.0' \
    --minimum_failure_threshold '100000' \
    --min_otu_size '3' \
    --suppress_taxonomy_assignment \
    --suppress_align_and_tree
cp pick_open_reference_otus_2/final_otu_map.txt 'test-data/pick_open_reference_otus_2_final_otu_map.txt'
cp pick_open_reference_otus_2/final_otu_map_mc*.txt 'test-data/pick_open_reference_otus_2_final_otu_map_mc.txt'
rm -rf pick_open_reference_otus_2

pick_open_reference_otus.py \
    --input_fps 'test-data/split_fastq_libraries_sequences.fasta' \
{'fraction_non_alphanumeric': 0.1040920716112532, 'fraction_numerical': 0.02122762148337596, 'mean_word_length': 4.155570204350692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5608955', 'n_tokens_mistral': 3234, 'n_tokens_neox': 3082, 'n_words': 358}