<commit_before>###
# Notes compiled by <NAME> (<EMAIL>)
# Analyses run with QIIME 1.5.0-dev on compy.colorado.edu
###


## OTU picking on 30 EMP fasta files (demultiplexed using the QIIME database by <NAME>)
echo "pick_subsampled_reference_otus_through_otu_table.py -i
/home/shared/emp-isme14/study_1031_split_library_seqs_and_mapping/study_1031_split_library_seqs.fna,/home/shared/emp-isme14/study_1033_split_library_seqs_and_mapping/study_1033_split_library_seqs.fna,/home/shared/emp-isme14/study_1034_split_library_seqs_and_mapping/study_1034_split_library_seqs.fna,/home/shared/emp-isme14/study_1035_split_library_seqs_and_mapping/study_1035_split_library_seqs.fna,/home/shared/emp-isme14/study_1036_split_library_seqs_and_mapping/study_1036_split_library_seqs.fna,/home/shared/emp-isme14/study_1037_split_library_seqs_and_mapping/study_1037_split_library_seqs.fna,/home/shared/emp-isme14/study_1038_split_library_seqs_and_mapping/study_1038_split_library_seqs.fna,/home/shared/emp-isme14/study_1039_split_library_seqs_and_mapping/study_1039_split_library_seqs.fna,/home/shared/emp-isme14/study_1043_split_library_seqs_and_mapping/study_1043_split_library_seqs.fna,/home/shared/emp-isme14/study_1197_split_library_seqs_and_mapping/study_1197_split_library_seqs.fna,/home/shared/emp-isme14/study_1198_split_library_seqs_and_mapping/study_1198_split_library_seqs.fna,/home/shared/emp-isme14/study_1222_split_library_seqs_and_mapping/study_1222_split_library_seqs.fna,/home/shared/emp-isme14/study_1240_split_library_seqs_and_mapping/study_1240_split_library_seqs.fna,/home/shared/emp-isme14/study_1242_split_library_seqs_and_mapping/study_1242_split_library_seqs.fna,/home/shared/emp-isme14/study_1288_split_library_seqs_and_mapping/study_1288_split_library_seqs.fna,/home/shared/emp-isme14/study_1289_split_library_seqs_and_mapping/study_1289_split_library_seqs.fna,/home/shared/emp-isme14/study_1453_split_library_seqs_and_mapping/study_1453_split_library_seqs.fna,/home/shared/emp-isme14/study_1526_split_library_seqs_and_mapping/study_1526_split_library_seqs.fna,/home/shared/emp-isme14/study_632_split_library_seqs_and_mapping/study_632_split_library_seqs.fna,/home/shared/emp-isme14/study_638_split_library_seqs_and_mapping/study_638_split_library_seqs.fna,/home/shared/emp-isme14/study_659_split_library_seqs_and_mapping/study_659_split_library_seqs.fna,/home/shared/emp-isme14/study_662_split_library_seqs_and_mapping/study_662_split_library_seqs.fna,/home/shared/emp-isme14/study_678_split_library_seqs_and_mapping/study_678_split_library_seqs.fna,/home/shared/emp-isme14/study_723_split_library_seqs_and_mapping/study_723_split_library_seqs.fna,/home/shared/emp-isme14/study_776_split_library_seqs_and_mapping/study_776_split_library_seqs.fna,/home/shared/emp-isme14/study_808_split_library_seqs_and_mapping/study_808_split_library_seqs.fna,/home/shared/emp-isme14/study_809_split_library_seqs_and_mapping/study_809_split_library_seqs.fna,/home/shared/emp-isme14/study_810_split_library_seqs_and_mapping/study_810_split_library_seqs.fna,/home/shared/emp-isme14/study_925_split_library_seqs_and_mapping/study_925_split_library_seqs.fna,/home/shared/emp-isme14/study_933_split_library_seqs_and_mapping/study_933_split_library_seqs.fna
-r /scratch/caporaso/gg_otus_4feb2011/rep_set/gg_97_otus_4feb2011.fasta -o
/home/shared/emp-isme14/ucrss_fast/ -aO 50 -n emp.isme14. -p
/home/shared/emp-isme14/ucrss_params.txt" | qsub -k oe -N emp-otus -q
norestrict -l nodes=1:ppn=60

## OTU picking second round of EMP data
pick_subsampled_reference_otus_through_otu_table.py -i
/home/shared/study_1235_split_library_seqs_and_mapping_updated/study_1235_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_850_split_library_seqs_and_mapping/study_850_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_550_split_library_seqs_and_mapping/study_550_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_722_split_library_seqs_and_mapping/study_722_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_714_split_library_seqs_and_mapping/study_714_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_940_split_library_seqs_and_mapping/study_940_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_1530_split_library_seqs_and_mapping/study_1530_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_1030_split_library_seqs_and_mapping/study_1030_split_library_seqs.fna.gz
-r /home/shared/emp-isme14/ucrss_fast/29/new_refseqs.fna -o
/home/shared/emp-isme14/ucrss_fast_r2/ -aO 100 -n emp.isme14.r2. -p
/home/shared/emp-isme14/ucrss_params.txt --prefilter_refseqs_fp
/scratch/caporaso/gg_otus_4feb2011/rep_set/gg_97_otus_4feb2011.fasta

# Previous command died after first fasta file: I had thought that this script supported gzipped input, 
# but it doesn't. Picking up from where that one left off...

pick_subsampled_reference_otus_through_otu_table.py -i
/home/shared/emp-isme14-additional/study_550_split_library_seqs_and_mapping/study_550_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_722_split_library_seqs_and_mapping/study_722_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_714_split_library_seqs_and_mapping/study_714_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_850_split_library_seqs_and_mapping/study_850_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_940_split_library_seqs_and_mapping/study_940_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_1530_split_library_seqs_and_mapping/study_1530_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_1030_split_library_seqs_and_mapping/study_1030_split_library_seqs.fna
-r /home/shared/emp-isme14/ucrss_fast_r2/0/new_refseqs.fna -o
/home/shared/emp-isme14/ucrss_fast_r3/ -aO 150 -n emp.isme14.r3. -p
/home/shared/emp-isme14/ucrss_params.txt --prefilter_refseqs_fp
/scratch/caporaso/gg_otus_4feb2011/rep_set/gg_97_otus_4feb2011.fasta


# Merge the initial round of 33 OTU tables. The code used here is in the EarthMicrobiomeProject/isme14 GitHub repo (commit e8971e3428).

echo "parallel_merge_otu_tables.py -i /home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_810.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1033.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1222.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1031.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_776.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_662.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_925.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1034.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_808.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_809.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1038.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_632.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1240.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1039.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1037.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_659.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_723.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1035.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_638.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1453.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1036.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1526.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1043.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1242.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1289.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1235.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_722.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_550.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1197.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1288.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_678.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1198.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_933.biom -o /home/caporaso/analysis/isme14/merged_otu_table/" | qsub -keo -N emp_merge
<commit_msg>updated otu picking and merge commands<commit_after>###
# Notes compiled by <NAME> (<EMAIL>)
# Analyses run with QIIME 1.5.0-dev on compy.colorado.edu
###


## OTU picking on 30 EMP fasta files (demultiplexed using the QIIME database by <NAME>)
echo "pick_subsampled_reference_otus_through_otu_table.py -i
/home/shared/emp-isme14/study_1031_split_library_seqs_and_mapping/study_1031_split_library_seqs.fna,/home/shared/emp-isme14/study_1033_split_library_seqs_and_mapping/study_1033_split_library_seqs.fna,/home/shared/emp-isme14/study_1034_split_library_seqs_and_mapping/study_1034_split_library_seqs.fna,/home/shared/emp-isme14/study_1035_split_library_seqs_and_mapping/study_1035_split_library_seqs.fna,/home/shared/emp-isme14/study_1036_split_library_seqs_and_mapping/study_1036_split_library_seqs.fna,/home/shared/emp-isme14/study_1037_split_library_seqs_and_mapping/study_1037_split_library_seqs.fna,/home/shared/emp-isme14/study_1038_split_library_seqs_and_mapping/study_1038_split_library_seqs.fna,/home/shared/emp-isme14/study_1039_split_library_seqs_and_mapping/study_1039_split_library_seqs.fna,/home/shared/emp-isme14/study_1043_split_library_seqs_and_mapping/study_1043_split_library_seqs.fna,/home/shared/emp-isme14/study_1197_split_library_seqs_and_mapping/study_1197_split_library_seqs.fna,/home/shared/emp-isme14/study_1198_split_library_seqs_and_mapping/study_1198_split_library_seqs.fna,/home/shared/emp-isme14/study_1222_split_library_seqs_and_mapping/study_1222_split_library_seqs.fna,/home/shared/emp-isme14/study_1240_split_library_seqs_and_mapping/study_1240_split_library_seqs.fna,/home/shared/emp-isme14/study_1242_split_library_seqs_and_mapping/study_1242_split_library_seqs.fna,/home/shared/emp-isme14/study_1288_split_library_seqs_and_mapping/study_1288_split_library_seqs.fna,/home/shared/emp-isme14/study_1289_split_library_seqs_and_mapping/study_1289_split_library_seqs.fna,/home/shared/emp-isme14/study_1453_split_library_seqs_and_mapping/study_1453_split_library_seqs.fna,/home/shared/emp-isme14/study_1526_split_library_seqs_and_mapping/study_1526_split_library_seqs.fna,/home/shared/emp-isme14/study_632_split_library_seqs_and_mapping/study_632_split_library_seqs.fna,/home/shared/emp-isme14/study_638_split_library_seqs_and_mapping/study_638_split_library_seqs.fna,/home/shared/emp-isme14/study_659_split_library_seqs_and_mapping/study_659_split_library_seqs.fna,/home/shared/emp-isme14/study_662_split_library_seqs_and_mapping/study_662_split_library_seqs.fna,/home/shared/emp-isme14/study_678_split_library_seqs_and_mapping/study_678_split_library_seqs.fna,/home/shared/emp-isme14/study_723_split_library_seqs_and_mapping/study_723_split_library_seqs.fna,/home/shared/emp-isme14/study_776_split_library_seqs_and_mapping/study_776_split_library_seqs.fna,/home/shared/emp-isme14/study_808_split_library_seqs_and_mapping/study_808_split_library_seqs.fna,/home/shared/emp-isme14/study_809_split_library_seqs_and_mapping/study_809_split_library_seqs.fna,/home/shared/emp-isme14/study_810_split_library_seqs_and_mapping/study_810_split_library_seqs.fna,/home/shared/emp-isme14/study_925_split_library_seqs_and_mapping/study_925_split_library_seqs.fna,/home/shared/emp-isme14/study_933_split_library_seqs_and_mapping/study_933_split_library_seqs.fna
-r /scratch/caporaso/gg_otus_4feb2011/rep_set/gg_97_otus_4feb2011.fasta -o
/home/shared/emp-isme14/ucrss_fast/ -aO 50 -n emp.isme14. -p
/home/shared/emp-isme14/ucrss_params.txt" | qsub -k oe -N emp-otus -q
norestrict -l nodes=1:ppn=60

## OTU picking second round of EMP data
pick_subsampled_reference_otus_through_otu_table.py -i
/home/shared/study_1235_split_library_seqs_and_mapping_updated/study_1235_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_850_split_library_seqs_and_mapping/study_850_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_550_split_library_seqs_and_mapping/study_550_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_722_split_library_seqs_and_mapping/study_722_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_714_split_library_seqs_and_mapping/study_714_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_940_split_library_seqs_and_mapping/study_940_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_1530_split_library_seqs_and_mapping/study_1530_split_library_seqs.fna.gz,/home/shared/emp-isme14-additional/study_1030_split_library_seqs_and_mapping/study_1030_split_library_seqs.fna.gz
-r /home/shared/emp-isme14/ucrss_fast/29/new_refseqs.fna -o
/home/shared/emp-isme14/ucrss_fast_r2/ -aO 100 -n emp.isme14.r2. -p
/home/shared/emp-isme14/ucrss_params.txt --prefilter_refseqs_fp
/scratch/caporaso/gg_otus_4feb2011/rep_set/gg_97_otus_4feb2011.fasta

# Previous command died after first fasta file: I had thought that this script supported gzipped input, 
# but it doesn't. Picking up from where that one left off...

pick_subsampled_reference_otus_through_otu_table.py -i
/home/shared/emp-isme14-additional/study_550_split_library_seqs_and_mapping/study_550_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_722_split_library_seqs_and_mapping/study_722_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_714_split_library_seqs_and_mapping/study_714_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_850_split_library_seqs_and_mapping/study_850_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_940_split_library_seqs_and_mapping/study_940_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_1530_split_library_seqs_and_mapping/study_1530_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_1030_split_library_seqs_and_mapping/study_1030_split_library_seqs.fna
-r /home/shared/emp-isme14/ucrss_fast_r2/0/new_refseqs.fna -o
/home/shared/emp-isme14/ucrss_fast_r3/ -aO 150 -n emp.isme14.r3. -p
/home/shared/emp-isme14/ucrss_params.txt --prefilter_refseqs_fp
/scratch/caporaso/gg_otus_4feb2011/rep_set/gg_97_otus_4feb2011.fasta

## Killed previous command while processing study 850. That is by far the largest (Yatsuneko Global Gut) and I don't think it will finish in time. Adding a few remaining studies, and added 850 as the last one in this list. We'll see if it completes. 
pick_subsampled_reference_otus_through_otu_table.py -i /home/shared/emp-isme14-additional/study_940_split_library_seqs_and_mapping/study_940_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_1530_split_library_seqs_and_mapping/study_1530_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_1030_split_library_seqs_and_mapping/study_1030_split_library_seqs.fna,/home/shared/emp-isme14-additional/study_850_split_library_seqs_and_mapping/study_850_split_library_seqs.fna -r /home/shared/emp-isme14/ucrss_fast_r3/2/new_refseqs.fna -o /home/shared/emp-isme14/ucrss_fast_r4/ -aO 150 -n emp.isme14.r4. -p /home/shared/emp-isme14/ucrss_params.txt --prefilter_refseqs_fp /scratch/caporaso/gg_otus_4feb2011/rep_set/gg_97_otus_4feb2011.fasta


# Merge the initial round of 33 OTU tables. The code used here is in the EarthMicrobiomeProject/isme14 GitHub repo (commit e8971e3428).

echo "parallel_merge_otu_tables.py -i /home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_810.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1033.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1222.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1031.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_776.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_662.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_925.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1034.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_808.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_809.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1038.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_632.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_714.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1240.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1039.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1037.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_659.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_723.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1035.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_638.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1453.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1036.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1526.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1043.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1242.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1289.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1235.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_722.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_550.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1288.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_678.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_1198.biom,/home/caporaso/analysis/isme14/per_study_otu_tables/otu_table_mc2_933.biom -o /home/caporaso/analysis/isme14/merged_otu_table/" | qsub -keo -N emp_merge -l pvmem=64gb -q memroute

{'fraction_non_alphanumeric': 0.09232678609971941, 'fraction_numerical': 0.08115691776386791, 'mean_word_length': 8.926620246384575, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2311493', 'n_tokens_mistral': 8873, 'n_tokens_neox': 7686, 'n_words': 357}