<jupyter_start><jupyter_text>Quantitative Analysis and Statistical Tests<jupyter_code>import pandas as pd
import statistics
from scipy.stats import f_oneway, bartlett, levene
import numpy as np
from statsmodels.stats.multicomp import pairwise_tukeyhsd
import pingouin
import matplotlib.pyplot as plt
import seaborn as sns

sns.set(style='white', rc={"grid.linewidth": 0.1})
sns.set_context("paper", font_scale=0.9)
%config InlineBackend.figure_format = 'retina'
%matplotlib inline
# load data
ted = pd.read_csv('../data/processed/ted_measures.csv')<jupyter_output><empty_output><jupyter_text>Assign talks to 4 time periods16 years of data split it as follows1. First set - 2006, 2007, 2008, 20092. Second set - 2010, 2011, 2012, 20133. Third set - 2014, 2015, 2016, 20174. Fourth set - 2018, 2019, 2020, 2021<jupyter_code># divide the talks into 4 time periods

def to_tp(x):

    if x in [2006, 2007, 2008, 2009]:
        return 1
    elif x in [2010, 2011, 2012, 2013]:
        return 2
    elif x in [2014, 2015, 2016, 2017]:
        return 3
    else:
        return 4

ted['time_period'] =ted['p_year'].apply(to_tp)
ted['time_period'].value_counts()<jupyter_output><empty_output><jupyter_text>Hypotheses TestingTest different hypotheses.1. Are there statistically significant differences between different time periods for the measures computed? => ANOVA2. If yes, where do they differences lie?    - Can Tukey's HSD be used?    - If not, what else to use?Significance level chosen : 5%<jupyter_code># measures
meas = ['mws', 'fkl', 'mtld', 'pm_i', 'pm_we', 'pm_you', 'lf', 'spm', 'nip']

# significance level
alpha = 0.05
# helper functions

def cnt_outliers(df, feature):

    iqr = stats.iqr(df[feature], interpolation='midpoint')
    out = [point for point in df[feature].values if (point > Q3+1.5*iqr) or (point < Q1-1.5*iqr)]

    return (len(out))

def split_time(df, col):

    periods = [1,2,3,4]
    temp = df[[col, 'time_period']]
    op = (list(temp[temp['time_period'] == 1][col].values),
          list(temp[temp['time_period'] == 2][col].values),
          list(temp[temp['time_period'] == 3][col].values),
          list(temp[temp['time_period'] == 4][col].values)
    )

    return op<jupyter_output><empty_output><jupyter_text>One-way ANOVA<jupyter_code>for m in meas:
    print(m)
    t1, t2, t3, t4 = split_time(ted, m)
    anova_res = f_oneway(t1, t2, t3, t4)
    print(anova_res)
    if (anova_res[1] < 0.05):
        print('The null is rejected. There are statistically significant differences between the different time periods')
    else:
        print('The null is not rejected.')
    print('----------')<jupyter_output>mws
F_onewayResult(statistic=39.87654712833259, pvalue=2.78948136587378e-25)
The null is rejected. There are statistically significant differences between the different time periods
----------
fkl
F_onewayResult(statistic=35.79192647640749, pvalue=9.786620318473051e-23)
The null is rejected. There are statistically significant differences between the different time periods
----------
mtld
F_onewayResult(statistic=83.34752227620514, pvalue=7.22976957448858e-52)
The null is rejected. There are statistically significant differences between the different time periods
----------
pm_i
F_onewayResult(statistic=0.1749226273575349, pvalue=0.9134083836408684)
The null is not rejected.
----------
pm_we
F_onewayResult(statistic=5.299154239615984, pvalue=0.0012116840283249819)
The null is rejected. There are statistically significant differences between the different time periods
----------
pm_you
F_onewayResult(statistic=36.43555803694209, pvalue=3.882466970946186e-23)
The null is rejected. There [...]<jupyter_text>The null is not rejected only in `pm_i`. This indicates that there are no statistically significant differences between the values of `pm_i` across all four time periods.<jupyter_code>meas = ['mws', 'fkl', 'mtld', 'pm_we', 'pm_you', 'lf', 'spm', 'nip']
# check for equal within-group variance
for m in meas:
    print(m)
    t1, t2, t3, t4 = split_time(ted, m)
    stat, p = bartlett(t1, t2, t3, t4)
    alpha = 0.05
    if p < alpha:  # null hypothesis: not equal within-group variances
        print("The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances")
    else:
        print("The null hypothesis cannot be rejected")<jupyter_output>mws
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances
fkl
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances
mtld
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances
pm_we
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances
pm_you
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances
lf
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances
spm
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances
nip
The null hypothesis can be rejected. Therefore, the groups do not have equal within-group variances<jupyter_text>Since the assumption for equal within-group variance does not hold true, Tukey's HSD can't be used. In this case, Games Howell test can be used to identify the differences between the time period groups for the computed measures.  Games Howell Post-hoc Test<jupyter_code>def games_howell(df, ev_qty, show_only_diff = False, alpha=0.05):

    for q in ev_qty:
        pair_diff = pingouin.pairwise_gameshowell(
        df,
        dv = q,
        between = 'time_period'
    ).round(3)

        if (show_only_diff):
            pair_diff = pair_diff[pair_diff['pval'] < alpha]
    
        print(q)
        print(pair_diff)
        print()
games_howell(ted, meas, True)
quant_vars = ['p_year', 'mws', 'fkl', 'mtld', 'pm_i', 'pm_we', 'pm_you', 'nip', 'spm', 'time_period', 'views', 'lf']
ted_regr = ted[quant_vars]
ted_regr.shape
ted_regr.head()
# distribution of descriptive measures

temp = ted_regr.drop(['p_year', 'time_period', 'views'], axis=1)
temp.head()
fig, axes = plt.subplots(2, 3, figsize=(16,8))

# spm
sns.kdeplot(ax=axes[0,0], x='spm', data=temp, alpha=0.5, shade=True)
axes[0,0].set_title("SPM (Median = " + str(temp.spm.median()) + ")")
axes[0,0].set_ylabel("Probability Density")
axes[0,0].set_xlabel("")

# fkl
sns.kdeplot(ax=axes[0,1], x='fkl', data=temp, alpha=0.5, shade=True)
axes[0,1].set_title("FKL (Median = " + str(temp.fkl.median()) + ")")
axes[0,1].set_ylabel("Probability Density")
axes[0,1].set_xlabel("")

# mtld
sns.kdeplot(ax=axes[0,2], x='mtld', data=temp, alpha=0.5, shade=True)
axes[0,2].set_title("MTLD (Median = " + str(round(temp.mtld.median(), 2)) + ")")
axes[0,2].set_ylabel("Probability Density")
axes[0,2].set_xlabel("")

# pronominal measures
sns.kdeplot(ax=axes[1,0], x='pm_i', data=temp, alpha=0.5, shade=True, label='PM_I')
axes[1,0].set_ylabel("Probability Density")
axes[1,0].set_xlabel("")
sns.kdeplot(ax=axes[1,0], x='pm_we', data=temp, alpha=0.5, shade=True, label='PM_We')
axes[1,0].set_ylabel("Probability Density")
axes[1,0].set_xlabel("")
sns.kdeplot(ax=axes[1,0], x='pm_you', data=temp, alpha=0.5, shade=True, label='PM_You')
axes[1,0].set_title("Pronominal Measures")
axes[1,0].set_ylabel("Probability Density")
axes[1,0].set_xlabel("")
axes[1,0].legend(labels=["PM_I", "PM_We", "PM_You"])

# nip
sns.kdeplot(ax=axes[1,1], x='nip', data=temp, alpha=0.5, shade=True)
axes[1,1].set_title("NIP (Median = " + str(temp.nip.median()) + ")")
axes[1,1].set_ylabel("Probability Density")
axes[1,1].set_xlabel("")

# lf
sns.kdeplot(ax=axes[1,2], x='lf', data=temp, alpha=0.5, shade=True)
axes[1,2].set_title("LF (Median = " + str(temp.lf.median()) + ")")
axes[1,2].set_ylabel("Probability Density")
axes[1,2].set_xlabel("")

plt.savefig('../reports/figures/'+"measure_dist"+'.png', bbox_inches='tight')
def year_prog(col, label, title):
    """
    Plot year-wise progression of a measure
    """
    plt.figure(figsize=(5,3))
    ax = sns.lineplot(x='p_year', y=col, data=ted)
    ax.set_xticks(np.arange(16))
    ax.set_xticks([2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019,          2020, 2021])
    ax.axhline(ted[col].mean(), color='r', ls='--')
    plt.xticks(rotation=45)
    plt.xlim(2006, 2021)
    plt.xlabel('\nYear of Publishing Talk')
    plt.ylabel(label)
    plt.savefig('../reports/figures/'+title+'.png', bbox_inches='tight')
year_prog('nip', 'How often does a speaker use a numeric quantity', 'nip')
year_prog('fkl', 'Lexical Diversity', 'fkl')
trial = ted[['p_year', 'cnt_laughs', 'cnt_i', 'cnt_we', 'cnt_you', 'nip_comps_cnt']]
trial.columns = ['Year', 'Count_Laughs', 'Count_I', 'Count_We', 'Count_You', 'Count_NIP']
trial.head()
corr = trial.corr()
mask = np.zeros_like(corr)
mask[np.triu_indices_from(mask)] = True
with sns.axes_style("white"):
    f, ax = plt.subplots(figsize=(7, 5))
    ax = sns.heatmap(corr, mask=mask, vmax=.3, square=True, annot=True, fmt='.2g', linewidths=1, cmap='inferno')
plt.savefig('../reports/figures/'+"corr.png", bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15083070908893473, 'fraction_numerical': 0.059398414594418504, 'mean_word_length': 4.781544256120528, 'pattern_counts': {'":': 1, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '607701', 'license': 'MIT', 'n_tokens_mistral': 3726, 'n_tokens_neox': 3258, 'n_words': 858, 'path': 'notebooks/quant_analysis.ipynb', 'repo_name': 'ry05/TED-Talk-Content-Analysis', 'size': '9209'}