<jupyter_start><jupyter_text>Integral ANOVA<NAME> | July 6, 2020* Integral analysis for slo-1 cell specific degradation mutants* HKK796 is the control for VG902, VG903 and HKK1165* Need to get posthoc  Library, functions, and variables<jupyter_code># import library
print('importing libraries')
import os, glob, sys
import numpy as np
import pandas as pd
import scipy.integrate as integrate
import statsmodels.api as sm
from statsmodels.formula.api import ols
from statsmodels.stats.anova import anova_lm
from statsmodels.stats import multicomp
import matplotlib.pyplot as plt
import seaborn as sns
# add local library
lib = '/Users/connylin/Dropbox/CA/_Publications/Manuscript RL Alcohol hab model slo1/rankinlab_slo1paper/2-Materials-Methods/local_lib'
if lib not in sys.path:
    sys.path.insert(1, lib)
import etl, stats, graphpack
# global variables
print('set variables')
DATA_DIR = '/Users/connylin/Dropbox/CA/_Publications/Manuscript RL Alcohol hab model slo1/rankinlab_slo1paper/3-Results/Fig5-slo1 cell specific ko/data'
OUTPUT_DIR_NAME = 'Score_Integral'
strain_dirs = glob.glob(DATA_DIR+'/*')
sub_dir = 'Dance_ShaneSpark4'
pvlimit = 0.001
alpha = 0.05
PRJ_TAG = 'integral_diff'
MEASURES = ['RevFreq','RevSpeed','RevDur']
GROUPS = {'HKK796': ['N2', 'N2_400mM', 'HKK796', 'HKK796_400mM'],
            'HKK1165': ['HKK796','HKK796_400mM', 'HKK1165', 'HKK1165_400mM'],
            'VG902': ['HKK796','HKK796_400mM', 'VG902', 'VG902_400mM'],
            'VG903': ['HKK796','HKK796_400mM', 'VG903', 'VG903_400mM']}
pvlimit = 0.001
alpha = 0.05<jupyter_output>set variables<jupyter_text>get datarequire rawdata.csv and mwtdb.csv generated by conver_shanespark_raw2csv.m<jupyter_code># import data
data = pd.read_csv(os.path.join(DATA_DIR, 'rawdata_all.csv'))
data
# calculate integral ---
data_flipped = data.pivot(index='mwtpath', columns='tap', values=MEASURES)
# calculate integral ---
df_output = pd.DataFrame(np.zeros([data_flipped.shape[0], len(MEASURES)]), index=data_flipped.index.values, columns=MEASURES)

for msr in MEASURES: 
    for i in data_flipped.index: 
        df_output.loc[i,msr] = integrate.simps(data_flipped[msr].loc[i].values)
df_output

# report missing values in integral analysis --
print(f'result has {df_output.shape[0]} plates')
print('remove missing plates')
# remove missing values (not sure why dropna() doesn't work)
bol_output = (df_output == 'NaN')
ind_to_drop = df_output.index[bol_output.any(axis=1)]
df_output.drop(index=ind_to_drop,inplace=True)
print(f'len of output {len(df_output)}') 
# ensure data are in the right datatype
for msr in MEASURES: # for each measure
    df_output[msr] = df_output[msr].astype('float64').copy()
df_output
# get group name
mwtpath = pd.Series(df_output.index.values)
db = mwtpath.str.split(pat='/', expand=True)
df_output['groupname'] = db.loc[:, 5].values
# get strain names
# glob for strain folders
strain_dirs = glob.glob(DATA_DIR+'/*')
sub_dir = 'TWR/Dance_ShaneSpark4'
iv = 'groupname'

for strain in GROUPS.keys():
    # create excel graphing output ---
    savename = f'graph_data_{strain}_{PRJ_TAG}.xlsx'
    savepath = os.path.join(DATA_DIR,savename)
    # reports ---
    savename = f'anova_{strain}_{PRJ_TAG}.txt'
    filename = os.path.join(DATA_DIR, savename)
    text_file = open(filename, 'w')
    text_file.write('ANOVA Integral\n')

    # save to excel
    with pd.ExcelWriter(savepath) as writer:
        for msr in MEASURES:
            print(f'\n{msr} ----------------')
            # get data
            i = df_output[iv].isin(GROUPS[strain]).values
            df = df_output.loc[i, [iv, msr]].copy()
            # create summary
            t = df.groupby(iv).agg(['count','mean','sem'])
            # save to excel
            t.to_excel(writer, sheet_name=msr)

            # run anova 
            lm = ols(f'{msr} ~ {iv}', data=df).fit()
            anovaT = anova_lm(lm)
            display(anovaT)
            # report result in APA format
            anova_apa = print_stats_apa(anovaT.columns[3], 
                                        anovaT.loc[iv,'df'],
                                        anovaT.loc[iv,anovaT.columns[3]], 
                                        anovaT.loc[iv,'PR(>F)'], 
                                        pvalue_limit = pvlimit, 
                                        alpha=alpha,
                                        separator='',
                                        show=True)
            output_str = f'{msr}: {anova_apa}\n'
            print(output_str)
            text_file.write(output_str)

            # print swarm plot
            # make graph
            plt.figure()
            sns.swarmplot(x='groupname',y=msr,data=df)
            plt.title(msr)
            # create save file name
            savefname = f'{msr}_swarmplot_{strain}_{PRJ_TAG}.jpeg'
            # construct save path
            savepath = os.path.join(DATA_DIR, savefname)
            # save fig
            plt.savefig(savepath)
            plt.show() 
            plt.close()         
        
    text_file.close()
    print(f'\n{strain_dir} complete')
    print('=============================================')<jupyter_output>RevFreq ----------------
{'fraction_non_alphanumeric': 0.13556710957057577, 'fraction_numerical': 0.026189100712497592, 'mean_word_length': 2.7393808495320373, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '181415', 'license': 'MIT', 'n_tokens_mistral': 1843, 'n_tokens_neox': 1697, 'n_words': 366, 'path': '3-Results/Fig5-slo1 cell specific ko/integral_diff_cellspecific.ipynb', 'repo_name': 'conny-lin/rankinlab_slo1paper', 'size': '5196'}