<jupyter_start><jupyter_text>**author**: <EMAIL>**date**: 7 Oct 2017**language**: Python 3.5**license**: BSD3 effect_size_main.ipynb<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Merging the cleaned mapping file (output of clean_map_emp.py) with alpha diversity for redundancy analysis<jupyter_code>path_map = '../../data/mapping-files/emp_qiime_mapping_qc_filtered.tsv'
df_map = pd.read_csv(path_map, sep='\t', index_col=0)<jupyter_output>/Users/luke.thompson/.local/lib/python3.5/site-packages/IPython/core/interactiveshell.py:2717: DtypeWarning: Columns (46,47,48,49,50,64,65,66,67) have mixed types. Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>for adiv<jupyter_code># df_qrt is (now also) metadata in deciles for alpha-diversity analysis
df_qrt = pd.read_csv('../../data/effect-size/emp_qiime_mapping_qc_filtered.tsv.quartiles.tsv', sep='\t', index_col=0)
#df_dec = pd.read_csv('../../data/effect-size/emp_qiime_mapping_qc_filtered.tsv.deciles.tsv', sep='\t', index_col=0)
df_qrt['adiv_observed_otus'] = [df_map['adiv_observed_otus'][i] for i in df_qrt.index]
df_qrt.drop('oxygen_mg_per_l', axis=1, inplace=True)
path_adiv = '../../data/effect-size/emp_qiime_mapping_qc_filtered.quartiles.adiv.tsv'
df_qrt.to_csv(path_adiv, sep='\t')<jupyter_output><empty_output><jupyter_text>for bdiv - unweighted<jupyter_code># df_qrt is metadata in quartiles for beta-diversity analysis
df_qrt = pd.read_csv('../../data/effect-size/emp_qiime_mapping_qc_filtered.tsv.quartiles.tsv', sep='\t', index_col=0)
path_bdiv = '../../data/beta-div/emp_90_gg_1k_unweighted_unifrac.txt.pc.first_ten'
df_bdiv = pd.read_csv(path_bdiv, sep='\t', index_col=0, header=None)
df_bdiv.columns = ['PC1', 'PC2', 'PC3', 'PC4', 'PC5', 'PC6', 'PC7', 'PC8', 'PC9', 'PC10']
for col in ['PC1', 'PC2', 'PC3', 'PC4', 'PC5', 'PC6', 'PC7', 'PC8', 'PC9', 'PC10']:
    df_qrt[col] = [df_bdiv[col][i] for i in df_qrt.index]
path_bdiv = '../../data/effect-size/emp_qiime_mapping_qc_filtered.quartiles.bdiv_uw.tsv'
df_qrt.to_csv(path_bdiv, sep='\t')<jupyter_output><empty_output><jupyter_text>for bdiv - weighted<jupyter_code># df_qrt is metadata in quartiles for beta-diversity analysis
df_qrt = pd.read_csv('../../data/effect-size/emp_qiime_mapping_qc_filtered.tsv.quartiles.tsv' , sep='\t', index_col=0)
path_bdiv = '../../data/beta-div/emp_90_gg_1k_weighted_unifrac.txt.pc.first_ten'
df_bdiv = pd.read_csv(path_bdiv, sep='\t', index_col=0, header=None)
df_bdiv.columns = ['PC1', 'PC2', 'PC3', 'PC4', 'PC5', 'PC6', 'PC7', 'PC8', 'PC9', 'PC10']
# df_qrt is metadata in quartiles for beta-diversity analysis
for col in ['PC1', 'PC2', 'PC3', 'PC4', 'PC5', 'PC6', 'PC7', 'PC8', 'PC9', 'PC10']:
    df_qrt[col] = [df_bdiv[col][i] for i in df_qrt.index]
path_bdiv = '../../data/effect-size/emp_qiime_mapping_qc_filtered.quartiles.bdiv_w.tsv'
df_qrt.to_csv(path_bdiv, sep='\t')<jupyter_output><empty_output><jupyter_text>Exploring the cleaned mapping file (output of clean_map_emp.py)<jupyter_code>df_dec.study_id.value_counts()
df_dec.host_scientific_name.value_counts()
df_dec.latitude_deg.value_counts()
df_dec.longitude_deg.value_counts()
df_dec.envo_biome_3.value_counts()
df_dec.empo_3.value_counts()
df_dec.temperature_deg_c.value_counts()
df_dec.ph.value_counts()
df_dec.salinity_psu.value_counts()
#df_dec.oxygen_mg_per_l.value_counts()
df_dec.nitrate_umol_per_l.value_counts()<jupyter_output><empty_output><jupyter_text>Plotting effect size results<jupyter_code>sns.set_style('ticks', {'ytick.color': 'white'})
sns.set_context('poster', font_scale=1.5)
mycolor = sns.xkcd_rgb['light blue']
cols = ['study_id', 'host_scientific_name',
    'latitude_deg', 'longitude_deg',
    'envo_biome_3', 'empo_3', 
    'temperature_deg_c', 'ph', 'salinity_psu',
    'oxygen_mg_per_l', 'nitrate_umol_per_l']<jupyter_output><empty_output><jupyter_text>Alpha-div<jupyter_code>results_q = pd.read_csv('../../data/effect-size/' +
                      'mannwhitneyu.alpha_map.adiv_observed_otus.quartiles.tsv', 
                      sep='\t', index_col=0)
results_d = pd.read_csv('../../data/effect-size/' +
                      'mannwhitneyu.alpha_map.adiv_observed_otus.deciles.tsv', 
                      sep='\t', index_col=0)
myresults_q = results_q.loc[results_q['column_name'].isin(cols)]
myresults_q = myresults_q.sort_values('effect_size', ascending=False)
myresults_d = results_d.loc[results_d['column_name'].isin(cols)]
myresults_d = myresults_d.sort_values('effect_size', ascending=False)
dict_colors = {'study_id':sns.color_palette('Paired', 12)[1], 'empo_3':sns.color_palette('Paired', 12)[3], 
               'envo_biome_3':sns.color_palette('Paired', 12)[5],'latitude_deg':sns.color_palette('Paired', 12)[7], 
               'longitude_deg':sns.color_palette('Paired', 12)[9], 'host_scientific_name':sns.color_palette('Paired', 12)[0], 
               'salinity_psu':sns.color_palette('Paired', 12)[2], 'temperature_deg_c':sns.color_palette('Paired', 12)[4],
               'ph':sns.color_palette('Paired', 12)[6], 'nitrate_umol_per_l':sns.color_palette('Paired', 12)[8], 
               'oxygen_mg_per_l':sns.color_palette('Paired', 12)[10]}
sns.barplot(x='effect_size', y='column_name', data=myresults_d, palette=dict_colors)
plt.xlabel('Mean pairwise effect size')
plt.ylabel('')
plt.tight_layout()
plt.savefig('adiv_effect_size_deciles.pdf')
sns.barplot(x='effect_size', y='column_name', data=myresults_q, palette=dict_colors)
plt.xlabel('Mean pairwise effect size')
plt.ylabel('Factor')
plt.tight_layout()
plt.savefig('adiv_effect_size_quartiles.pdf')<jupyter_output><empty_output><jupyter_text>Beta-div<jupyter_code>results_w = pd.read_csv('../../data/effect-size/' +
                      'emp_90_gg_1k_weighted_unifrac.emp_qiime_mapping_qc_filtered.quartiles.tsv', 
                      sep='\t', index_col=0)
myresults_w = results_w.loc[results_w['column_name'].isin(cols)]
myresults_w = myresults_w.sort_values('effect_size', ascending=False)
sns.barplot(x='effect_size', y='column_name', data=myresults_w, palette=dict_colors)
plt.xlabel('Mean pairwise effect size')
plt.ylabel('Factor')
plt.tight_layout()
plt.savefig('bdiv_weighted_effect_size_quartiles.pdf')
results_uw = pd.read_csv('../../data/effect-size/' +
                      'emp_90_gg_1k_unweighted_unifrac.emp_qiime_mapping_qc_filtered.quartiles.tsv', 
                      sep='\t', index_col=0)
myresults_uw = results_uw.loc[results_uw['column_name'].isin(cols)]
myresults_uw = myresults_uw.sort_values('effect_size', ascending=False)
sns.barplot(x='effect_size', y='column_name', data=myresults_uw, palette=dict_colors)
plt.xlabel('Mean pairwise effect size')
plt.ylabel('Factor')
plt.tight_layout()
plt.savefig('bdiv_unweighted_effect_size_quartiles.pdf')<jupyter_output><empty_output><jupyter_text>Plotting RDA results<jupyter_code>sns.set_style('ticks', {'ytick.color': 'black'})
path_rda = '../../data/effect-size/rda_values.xlsx'
df_rda = pd.read_excel(path_rda)
df_rda.fillna(0, inplace=True)
df_rda
mycolors = ["dark grey", "windows blue", "amber", "faded green", "dusty purple"]
plt.figure(figsize=(10,10))

plt.bar(np.arange(6), df_rda.study_id, color=dict_colors['study_id'])
bottom = df_rda.study_id
plt.bar(np.arange(6), df_rda.empo_3, bottom=bottom, color=dict_colors['empo_3'])
bottom = bottom + df_rda.empo_3
plt.bar(np.arange(6), df_rda.envo_biome_3, bottom=bottom, color=dict_colors['envo_biome_3'])
bottom = bottom + df_rda.envo_biome_3
plt.bar(np.arange(6), df_rda.latitude_deg, bottom=bottom, color=dict_colors['latitude_deg'])
bottom = bottom + df_rda.latitude_deg
plt.bar(np.arange(6), df_rda.longitude_deg, bottom=bottom, color=dict_colors['longitude_deg'])

plt.xticks(np.arange(6), ['alpha w/ study_id', 'alpha w/o study_id', 'unweighted w/ study_id', 'unweighted w/o study_id', 
                          'weighted w/ study_id', 'weighted w/o study_id'], rotation=270)

plt.tight_layout()

plt.savefig('rda.pdf')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16512744735869966, 'fraction_numerical': 0.01994828223125231, 'mean_word_length': 6.087260034904014, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '378235', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3425, 'n_tokens_neox': 3207, 'n_words': 428, 'path': 'code/07-env-effects-corr/effect_size_main.ipynb', 'repo_name': 'justinshaffer/emp', 'size': '8136'}