<commit_before># establish any minimals for the notebook environment
import os
import shutil
from distutils.spawn import find_executable

import qiime
import qiime_default_reference as qdr

import americangut as ag
from americangut.results_utils import get_repository_dir
from americangut.util import get_existing_path


_TEST_ENV = os.environ.get('AG_TESTING') == 'True'
_EBI_ACCESSIONS = ['ERP012511']
_TEST_ACCESSIONS = ['ag_testing']


# essential paths relative to the working_dir to be used between notebooks

paths = {
    # raw files
    'raw-sequences': '1/raw-sequences.fna',
    'raw-metadata': '1/raw-metadata.txt',

    # sequences filtered for blooms
    'filtered-sequences': '2/filtered-sequences.fna',
    'filtered-sequences-100nt': '2/filtered-sequences-100nt.fna',

    # only fecal sequences (for filtering for blooms)
    'fecal-sequences': '2/fecal-sequences.fna',

    # observed bloom sequences in samples
    'observed-blooms': '2/observed-blooms',
    'observed-blooms-biom': '2/observed-blooms/otu_table.biom',
    'observed-blooms-otu-map':
        '2/observed-blooms/sortmerna_picked_otus/fecal-sequences_otus.txt',

    # resulting OTU directories
    'ag-otus': '3/otus/gg-13_8-97-per-otus',
    'ag-otus-100nt': '3/otus/gg-13_8-97-per-otus-with-100nt',
    'ag-biom': '3/otus/gg-13_8-97-per-otus/otu_table.biom',
    'ag-100nt-biom': '3/otus/gg-13_8-97-per-otus-with-100nt/otu_table.biom',

    # merged files for diversity analyses
    'ag-gg-100nt-biom': '4/ag-gg-100nt.biom',
    'pgp-hmp-100nt-biom': '4/pgp-hmp-100nt.biom',
    'ag-pgp-hmp-gg-100nt-biom': '4/ag-pgp-hmp-gg-100nt.biom',
    'ag-cleaned-md': '4/ag-cleaned.txt',
    'gg-cleaned-md': '4/gg-cleaned.txt',
    'pgp-cleaned-md': '4/pgp-cleaned.txt',
    'hmp-cleaned-md': '4/hmp-cleaned.txt',
    'ag-gg-cleaned-md': '4/ag-gg-cleaned.txt',
    'pgp-hmp-cleaned-md': '4/pgp-hmp-cleaned.txt',
    'ag-pgp-hmp-gg-cleaned-md': '4/ag-pgp-hmp-gg-cleaned.txt',

    # alpha diversity analysis files
    'ag-pgp-hmp-gg-100nt-1k-multiple': '5/ag-pgp-hmp-gg-100nt-1k-multiple',


    'ag-pgp-hmp-gg-100nt-1k-adiv': '5/ag-pgp-hmp-gg-100nt-1k-adiv',
    'ag-pgp-hmp-gg-100nt-1k-adiv-pd':
        '5/ag-pgp-hmp-gg-100nt-1k-adiv/PD_whole_tree.txt',
    'ag-pgp-hmp-gg-100nt-1k-adiv-chao1':
        '5/ag-pgp-hmp-gg-100nt-1k-adiv/chao1.txt',
    'ag-pgp-hmp-gg-100nt-1k-adiv-observedotus':
        '5/ag-pgp-hmp-gg-100nt-1k-adiv/observed_otus.txt',

    # beta diversity analysis files
    'ag-pgp-hmp-gg-100nt-1k-biom': '6/ag-pgp-hmp-gg-100nt-1k.biom',
    'ag-pgp-hmp-gg-100nt-1k-bdiv': '6/ag-pgp-hmp-gg-100nt-1k-bdiv',

    'ag-100nt-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-1k.txt'),
    'ag-100nt-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-1k-pc.txt'),

    'ag-100nt-oral-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-oral-1k.txt'),
    'ag-100nt-oral-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-oral-1k-pc.txt'),

    'ag-100nt-fecal-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-fecal-1k.txt'),
    'ag-100nt-fecal-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-fecal-1k-pc.txt'),

    'ag-100nt-skin-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-skin-1k.txt'),
    'ag-100nt-skin-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-skin-1k-pc.txt'),

    'ag-pgp-hmp-gg-100nt-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-pgp-hmp-gg-100nt-1k.txt'),
    'ag-pgp-hmp-gg-100nt-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-pgp-hmp-gg-100nt-1k-pc.txt'),

    'ag-pgp-hmp-gg-100nt-1k-bdiv-wunifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-pgp-hmp-gg-100nt-1k.txt'),
    'ag-pgp-hmp-gg-100nt-1k-wunifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-pgp-hmp-gg-100nt-1k-pc.txt'),

    'ag-gg-100nt-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k.txt'),
    'ag-gg-100nt-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k-pc.txt'),

    'ag-gg-100nt-1k-bdiv-subsampled-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k-subsampled.txt'),
    'ag-gg-100nt-1k-subsampled-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k-subsampled-pc.txt'),

    'ag-gg-100nt-1k-bdiv-wunifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-gg-100nt-1k.txt'),
    'ag-gg-100nt-1k-wunifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-gg-100nt-1k-pc.txt'),

    # taxonomy summaries
    'ag-taxa': '7/taxa',
    'ag-L2-taxa-tsv': '7/taxa/otu_table_L2.txt',
    'ag-L2-taxa-biom': '7/taxa/otu_table_L2.biom',
    'ag-L2-taxa-md': '7/taxa/ag-cleaned_L2.txt',
    'ag-L3-taxa-tsv': '7/taxa/otu_table_L3.txt',
    'ag-L3-taxa-biom': '7/taxa/otu_table_L3.biom',
    'ag-L6-taxa-tsv': '7/taxa/otu_table_L6.txt',
    'ag-L6-taxa-biom': '7/taxa/otu_table_L6.biom',
    'ag-L2-taxa-skin-biom': '7/taxa/otu_table_skin_L2.biom',
    'ag-L2-taxa-oral-biom': '7/taxa/otu_table_oral_L2.biom',
    'ag-L2-taxa-fecal-biom': '7/taxa/otu_table_fecal_L2.biom',
    'ag-L6-taxa-skin-biom': '7/taxa/otu_table_skin_L6.biom',
    'ag-L6-taxa-oral-biom': '7/taxa/otu_table_oral_L6.biom',
    'ag-L6-taxa-fecal-biom': '7/taxa/otu_table_fecal_L6.biom',

    # collapsed samples
    'ag-100nt-1k-biom': '7.5/ag-100nt-1k.biom',
    'ag-100nt-1k-fecal-biom': '7.5/ag-100nt-1k-fecal.biom',
    'ag-100nt-1k-skin-biom': '7.5/ag-100nt-1k-oral.biom',
    'ag-100nt-1k-oral-biom': '7.5/ag-100nt-1k-skin.biom',

    'ag-100nt-1k-fecal-sex-biom':  '7.5/ag-100nt-1k-fecal-sex.biom',
    'ag-100nt-1k-fecal-diet-biom': '7.5/ag-100nt-1k-fecal-diet.biom',
    'ag-100nt-1k-fecal-age-biom':  '7.5/ag-100nt-1k-fecal-age.biom',
    'ag-100nt-1k-fecal-bmi-biom':  '7.5/ag-100nt-1k-fecal-bmi.biom',

    'ag-100nt-1k-oral-sex-biom':  '7.5/ag-100nt-1k-oral-sex.biom',
    'ag-100nt-1k-oral-diet-biom': '7.5/ag-100nt-1k-oral-diet.biom',
    'ag-100nt-1k-oral-age-biom':  '7.5/ag-100nt-1k-oral-age.biom',
    'ag-100nt-1k-oral-flossing-biom':  '7.5/ag-100nt-1k-oral-flossing.biom',

    'ag-100nt-1k-skin-sex-biom':  '7.5/ag-100nt-1k-skin-sex.biom',
    'ag-100nt-1k-skin-cosmetics-biom': '7.5/ag-100nt-1k-skin-cosmetics.biom',
    'ag-100nt-1k-skin-age-biom':  '7.5/ag-100nt-1k-skin-age.biom',
    'ag-100nt-1k-skin-hand-biom':  '7.5/ag-100nt-1k-skin-hand.biom',

    # per-sample results
    'successful-ids': '8/successful_ids.txt',
    'unsuccessful-ids': '8/unsuccessful_ids.txt',
    'per-sample-results': '8/per-sample-results',
}


def _assert_environment():
    if qiime.__version__ != '1.9.1':
        obs_version = qiime.__version__
        raise ImportError("QIIME 1.9.1 is not in the environment, found "
                          "%s." % obs_version)

    if find_executable('print_qiime_config.py') is None:
        raise EnvironmentError("The QIIME executables are not in $PATH.")

    if find_executable('mod2_pcoa.py') is None:
        raise EnvironmentError("The AG scripts are not in $PATH.")
_assert_environment()


def activate(chp):
    """Activate a chapter

    Parameters
    ----------
    chp : str
        The chapter.<commit_msg>updated chp refs for 8<commit_after># establish any minimals for the notebook environment
import os
import shutil
from distutils.spawn import find_executable

import qiime
import qiime_default_reference as qdr

import americangut as ag
from americangut.results_utils import get_repository_dir
from americangut.util import get_existing_path


_TEST_ENV = os.environ.get('AG_TESTING') == 'True'
_EBI_ACCESSIONS = ['ERP012511']
_TEST_ACCESSIONS = ['ag_testing']


# essential paths relative to the working_dir to be used between notebooks

paths = {
    # raw files
    'raw-sequences': '1/raw-sequences.fna',
    'raw-metadata': '1/raw-metadata.txt',

    # sequences filtered for blooms
    'filtered-sequences': '2/filtered-sequences.fna',
    'filtered-sequences-100nt': '2/filtered-sequences-100nt.fna',

    # only fecal sequences (for filtering for blooms)
    'fecal-sequences': '2/fecal-sequences.fna',

    # observed bloom sequences in samples
    'observed-blooms': '2/observed-blooms',
    'observed-blooms-biom': '2/observed-blooms/otu_table.biom',
    'observed-blooms-otu-map':
        '2/observed-blooms/sortmerna_picked_otus/fecal-sequences_otus.txt',

    # resulting OTU directories
    'ag-otus': '3/otus/gg-13_8-97-per-otus',
    'ag-otus-100nt': '3/otus/gg-13_8-97-per-otus-with-100nt',
    'ag-biom': '3/otus/gg-13_8-97-per-otus/otu_table.biom',
    'ag-100nt-biom': '3/otus/gg-13_8-97-per-otus-with-100nt/otu_table.biom',

    # merged files for diversity analyses
    'ag-gg-100nt-biom': '4/ag-gg-100nt.biom',
    'pgp-hmp-100nt-biom': '4/pgp-hmp-100nt.biom',
    'ag-pgp-hmp-gg-100nt-biom': '4/ag-pgp-hmp-gg-100nt.biom',
    'ag-cleaned-md': '4/ag-cleaned.txt',
    'gg-cleaned-md': '4/gg-cleaned.txt',
    'pgp-cleaned-md': '4/pgp-cleaned.txt',
    'hmp-cleaned-md': '4/hmp-cleaned.txt',
    'ag-gg-cleaned-md': '4/ag-gg-cleaned.txt',
    'pgp-hmp-cleaned-md': '4/pgp-hmp-cleaned.txt',
    'ag-pgp-hmp-gg-cleaned-md': '4/ag-pgp-hmp-gg-cleaned.txt',

    # alpha diversity analysis files
    'ag-pgp-hmp-gg-100nt-1k-multiple': '5/ag-pgp-hmp-gg-100nt-1k-multiple',


    'ag-pgp-hmp-gg-100nt-1k-adiv': '5/ag-pgp-hmp-gg-100nt-1k-adiv',
    'ag-pgp-hmp-gg-100nt-1k-adiv-pd':
        '5/ag-pgp-hmp-gg-100nt-1k-adiv/PD_whole_tree.txt',
    'ag-pgp-hmp-gg-100nt-1k-adiv-chao1':
        '5/ag-pgp-hmp-gg-100nt-1k-adiv/chao1.txt',
    'ag-pgp-hmp-gg-100nt-1k-adiv-observedotus':
        '5/ag-pgp-hmp-gg-100nt-1k-adiv/observed_otus.txt',

    # beta diversity analysis files
    'ag-pgp-hmp-gg-100nt-1k-biom': '6/ag-pgp-hmp-gg-100nt-1k.biom',
    'ag-pgp-hmp-gg-100nt-1k-bdiv': '6/ag-pgp-hmp-gg-100nt-1k-bdiv',

    'ag-100nt-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-1k.txt'),
    'ag-100nt-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-1k-pc.txt'),

    'ag-100nt-oral-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-oral-1k.txt'),
    'ag-100nt-oral-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-oral-1k-pc.txt'),

    'ag-100nt-fecal-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-fecal-1k.txt'),
    'ag-100nt-fecal-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-fecal-1k-pc.txt'),

    'ag-100nt-skin-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-skin-1k.txt'),
    'ag-100nt-skin-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-100nt-skin-1k-pc.txt'),

    'ag-pgp-hmp-gg-100nt-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-pgp-hmp-gg-100nt-1k.txt'),
    'ag-pgp-hmp-gg-100nt-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-pgp-hmp-gg-100nt-1k-pc.txt'),

    'ag-pgp-hmp-gg-100nt-1k-bdiv-wunifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-pgp-hmp-gg-100nt-1k.txt'),
    'ag-pgp-hmp-gg-100nt-1k-wunifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-pgp-hmp-gg-100nt-1k-pc.txt'),

    'ag-gg-100nt-1k-bdiv-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k.txt'),
    'ag-gg-100nt-1k-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k-pc.txt'),

    'ag-gg-100nt-1k-bdiv-subsampled-unifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k-subsampled.txt'),
    'ag-gg-100nt-1k-subsampled-unifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'unweighted_unifrac_ag-gg-100nt-1k-subsampled-pc.txt'),

    'ag-gg-100nt-1k-bdiv-wunifrac':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-gg-100nt-1k.txt'),
    'ag-gg-100nt-1k-wunifrac-pc':
        ('6/ag-pgp-hmp-gg-100nt-1k-bdiv/'
         'weighted_unifrac_ag-gg-100nt-1k-pc.txt'),

    # taxonomy summaries
    'ag-taxa': '7/taxa',
    'ag-L2-taxa-tsv': '7/taxa/otu_table_L2.txt',
    'ag-L2-taxa-biom': '7/taxa/otu_table_L2.biom',
    'ag-L2-taxa-md': '7/taxa/ag-cleaned_L2.txt',
    'ag-L3-taxa-tsv': '7/taxa/otu_table_L3.txt',
    'ag-L3-taxa-biom': '7/taxa/otu_table_L3.biom',
    'ag-L6-taxa-tsv': '7/taxa/otu_table_L6.txt',
    'ag-L6-taxa-biom': '7/taxa/otu_table_L6.biom',
    'ag-L2-taxa-skin-biom': '7/taxa/otu_table_skin_L2.biom',
    'ag-L2-taxa-oral-biom': '7/taxa/otu_table_oral_L2.biom',
    'ag-L2-taxa-fecal-biom': '7/taxa/otu_table_fecal_L2.biom',
    'ag-L6-taxa-skin-biom': '7/taxa/otu_table_skin_L6.biom',
    'ag-L6-taxa-oral-biom': '7/taxa/otu_table_oral_L6.biom',
    'ag-L6-taxa-fecal-biom': '7/taxa/otu_table_fecal_L6.biom',

    # collapsed samples
    'ag-100nt-1k-biom': '8/ag-100nt-1k.biom',
    'ag-100nt-1k-fecal-biom': '8/ag-100nt-1k-fecal.biom',
    'ag-100nt-1k-skin-biom': '8/ag-100nt-1k-oral.biom',
    'ag-100nt-1k-oral-biom': '8/ag-100nt-1k-skin.biom',

    'ag-100nt-1k-fecal-sex-biom':  '8/ag-100nt-1k-fecal-sex.biom',
    'ag-100nt-1k-fecal-diet-biom': '8/ag-100nt-1k-fecal-diet.biom',
    'ag-100nt-1k-fecal-age-biom':  '8/ag-100nt-1k-fecal-age.biom',
    'ag-100nt-1k-fecal-bmi-biom':  '8/ag-100nt-1k-fecal-bmi.biom',

    'ag-100nt-1k-oral-sex-biom':  '8/ag-100nt-1k-oral-sex.biom',
    'ag-100nt-1k-oral-diet-biom': '8/ag-100nt-1k-oral-diet.biom',
    'ag-100nt-1k-oral-age-biom':  '8/ag-100nt-1k-oral-age.biom',
    'ag-100nt-1k-oral-flossing-biom':  '8/ag-100nt-1k-oral-flossing.biom',

    'ag-100nt-1k-skin-sex-biom':  '8/ag-100nt-1k-skin-sex.biom',
    'ag-100nt-1k-skin-cosmetics-biom': '8/ag-100nt-1k-skin-cosmetics.biom',
    'ag-100nt-1k-skin-age-biom':  '8/ag-100nt-1k-skin-age.biom',
    'ag-100nt-1k-skin-hand-biom':  '8/ag-100nt-1k-skin-hand.biom',

    # per-sample results
    'successful-ids': '9/successful_ids.txt',
    'unsuccessful-ids': '9/unsuccessful_ids.txt',
    'per-sample-results': '9/per-sample-results',
}


def _assert_environment():
    if qiime.__version__ != '1.9.1':
        obs_version = qiime.__version__
        raise ImportError("QIIME 1.9.1 is not in the environment, found "
                          "%s." % obs_version)

    if find_executable('print_qiime_config.py') is None:
        raise EnvironmentError("The QIIME executables are not in $PATH.")

    if find_executable('mod2_pcoa.py') is None:
        raise EnvironmentError("The AG scripts are not in $PATH.")
_assert_environment()


def activate(chp):
    """Activate a chapter

    Parameters
    ----------
    chp : str
        The chapter.
{'fraction_non_alphanumeric': 0.19738638580066314, 'fraction_numerical': 0.07613289122943892, 'mean_word_length': 2.7109770808202653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2345725', 'n_tokens_mistral': 8101, 'n_tokens_neox': 7299, 'n_words': 651}