<jupyter_start><jupyter_text>Evaluate w2v results<jupyter_code># Fits the model with the interaction term
# This will also automatically include the main effects for each factor
w2v_model = ols('map_mrr ~ C(clusters)*C(model)', w2v).fit()

# Seeing if the overall model is significant
print(f"Overall model F({w2v_model.df_model: .0f},{w2v_model.df_resid: .0f}) = {w2v_model.fvalue: .3f}, p = {w2v_model.f_pvalue: .10f}")
w2v_model.summary()
# Creates the ANOVA table
res = sm.stats.anova_lm(w2v_model, typ= 1)
anova_table(res)

#We found that the effect of the interaction of cluster and model was insignificant. F(4, 36)=1.92, p > 0.05.
#We redo the analysis, this time testing for the effect of cluster and model in isolation.
# we remove the interaction term and analyse the indepedent variables separately

w2v_model2 = ols('map_mrr ~ C(clusters)+ C(model)', w2v).fit()

print(f"Overall model F({w2v_model2.df_model: .0f},{w2v_model2.df_resid: .0f}) = {w2v_model2.fvalue: .3f}, p = {w2v_model2.f_pvalue: .8f}")
w2v_model2.summary()
res2 = sm.stats.anova_lm(w2v_model2, typ= 1)
anova_table(res2)
# After refitting the ANOVA model on the combination of cluster and model, very weak statistical evidence
# was found that the effect of the regularised models is significant. 
# However, the effect of clusters was significant on the harmonic mean of MAP and MRR, F(2,40)=44.342, p << 0.05
# omega^2 = 0.637
# employ Tukey's HSD to find evidence of effect in model
mc = statsmodels.stats.multicomp.MultiComparison(w2v['map_mrr'], w2v['clusters'])
mc_results = mc.tukeyhsd()
print(mc_results)

# post-hoc analysis revealed that increasing the clusters from 1 to 10, 1 to 25 and 10 to 25 all had a positive effect
# on the MAP_MRR harmonic mean when training an MSE model on word2vec features.<jupyter_output>Multiple Comparison of Means - Tukey HSD,FWER=0.05
===========================================
group1 group2 meandiff lower  upper  reject
-------------------------------------------
  1      10    0.0284  0.0135 0.0434  True 
  1      25    0.055   0.0401 0.0699  True 
  10     25    0.0265  0.0116 0.0414  True 
-------------------------------------------<jupyter_text>Evaluate GloVe<jupyter_code>glove_model = ols('map_mrr ~ C(clusters)*C(model)', glove).fit()

# Seeing if the overall model is significant
print(f"Overall model F({glove_model.df_model: .0f},{glove_model.df_resid: .0f}) = {glove_model.fvalue: .3f}, p = {glove_model.f_pvalue: .10f}")
glove_model.summary()

# the overall interaction combination model was insignificant, F(8, 45) = 2.149, p < 0.05.
# Creates the ANOVA table
res = sm.stats.anova_lm(glove_model, typ= 1)
anova_table(res)

# we tested the combination of clusters and model on the effect of mrr_map harmonic mean.
# the overall 2-way combination ANOVA was insignificant.

glove_model2 = ols('map_mrr ~ C(clusters)', glove).fit()

print(f"Overall model F({glove_model2.df_model: .0f},{glove_model2.df_resid: .0f}) = {glove_model2.fvalue: .3f}, p = {glove_model2.f_pvalue: .8f}")

# we fitted another ANOVA model, this time exploring the effect of clusters along.  
# This was significant, F(2, 42)=9.748, p < 0.05; omega^2 = 0.280
res2 = sm.stats.anova_lm(glove_model2, typ= 1)
anova_table(res2)

mc = statsmodels.stats.multicomp.MultiComparison(glove['map_mrr'], glove['clusters'])
mc_results = mc.tukeyhsd()
print(mc_results)

# Post-hoc analysis revealed that increasing clusters from 1 to 10 had a significantly negative effect
# on the map_mrr harmonic mean, while increasing clusters from 10 to 25 had a positive effect.
# However, increasing the clusters from 1 to 25 had no effect on the results.<jupyter_output>Multiple Comparison of Means - Tukey HSD,FWER=0.05
=============================================
group1 group2 meandiff  lower   upper  reject
---------------------------------------------
  1      10   -0.0332  -0.0519 -0.0145  True 
  1      25   -0.0101  -0.0288  0.0086 False 
  10     25    0.023    0.0043  0.0417  True 
---------------------------------------------<jupyter_text>Evaluate fastText results<jupyter_code>ft_model = ols('map_mrr ~ C(clusters)*C(model)', ft).fit()

# Seeing if the overall model is significant
print(f"Overall model F({ft_model.df_model: .0f},{ft_model.df_resid: .0f}) = {ft_model.fvalue: .3f}, p = {ft_model.f_pvalue: .10f}")
ft_model.summary()

# the interaction combination overall model was significant on the map_nrr, F(8, 36) = 3.556, p<0.05.
# Creates the ANOVA table
res = sm.stats.anova_lm(ft_model, typ= 1)
anova_table(res)

# Inspecting the ANOVA table, however revealed that neither the interaction term nor the model term 
# significant.  We retested a fresh ANOVA, focusing on just the cluster term.
ft_model2 = ols('map_mrr ~ C(clusters)', ft).fit()
print(f"Overall model F({ft_model2.df_model: .0f},{ft_model2.df_resid: .0f}) = {ft_model2.fvalue: .3f}, p = {ft_model2.f_pvalue: .8f}")

res2 = sm.stats.anova_lm(ft_model2, typ= 1)
print (res2)
anova_table(res2)

# There was a significant effect of clusters on the map_mrr score, F(2, 42) = 16.016, p << 0.05., omega^2=0.40
mc = statsmodels.stats.multicomp.MultiComparison(ft['map_mrr'], ft['clusters'])
mc_results = mc.tukeyhsd()
print(mc_results)

# post-hoc analysis reveals that that increasing clusters from 1 to 10, and from 1 to 25 
# has a negative effect on MAP_MRR while increasing clusters from 10 to 25 has a positive negligible effect.<jupyter_output>Multiple Comparison of Means - Tukey HSD,FWER=0.05
=============================================
group1 group2 meandiff  lower   upper  reject
---------------------------------------------
  1      10   -0.0329  -0.0483 -0.0174  True 
  1      25   -0.0291  -0.0445 -0.0136  True 
  10     25    0.0038  -0.0116  0.0192 False 
---------------------------------------------<jupyter_text>Evaluate difference in embeddings using only baseline<jupyter_code>all_embeddings = df.loc[df.model=='Baseline',:]
all_embeddings.head()
#all_embeddings.loc[, ['embeddings','clusters']]
all_model = ols('map_mrr ~ C(embeddings)*C(clusters)', all_embeddings).fit()

# Seeing if the overall model is significant
print(f"Overall model F({all_model.df_model: .0f},{all_model.df_resid: .0f}) = {all_model.fvalue: .3f}, p = {all_model.f_pvalue: .10f}")
all_model.summary()

# Once we could safely ignore the effect of the regularised models, we investigated the interaction of
# embeddings and cluster.s
# The overal ANOVA was significant F(8. 36) = 19.733, p << 0.05
# Creates the ANOVA table
res = sm.stats.anova_lm(all_model, typ= 1)
anova_table(res)
sns.catplot(x="embeddings", y="map_mrr", hue="clusters", kind="point", data=all_embeddings);
# interaction effect is significant.  We decided to Rerun ANOVA investigating simple effect.  That is we
# run three models, checking the influence of the embeddings whilst controlling for 1, 10 and 25 clusters 
# respectively
# embeddings | c = 1
# embeddings | c = 10
# embeddings | c = 25
cl_1 = all_embeddings.loc[all_embeddings.clusters == 1,]
cl_1_model = ols('map_mrr ~ C(embeddings)', cl_1).fit()

# Seeing if the overall model is significant
print(f"Overall model F({cl_1_model.df_model: .0f},{cl_1_model.df_resid: .0f}) = {cl_1_model.fvalue: .3f}, p = {cl_1_model.f_pvalue: .10f}")
cl_1_model.summary()
res = sm.stats.anova_lm(cl_1_model, typ= 1)
anova_table(res)

# the embeddings had a significant effect on single cluster, baseline models, F(2, 12)=52.376, p < 0.05, omega^2 = 0.876
mc = statsmodels.stats.multicomp.MultiComparison(cl_1['map_mrr'], cl_1['embeddings'])
mc_results = mc.tukeyhsd()
print(mc_results)

# post-hoc analysis reveals that fastText yields a higher map_mrr than either GloVe or word2vec and that GloVE
# returns a higher score than word2vec.
cl_10 = all_embeddings.loc[all_embeddings.clusters == 10,]
cl_10_model = ols('map_mrr ~ C(embeddings)', cl_10).fit()

# Seeing if the overall model is significant
print(f"Overall model F({cl_10_model.df_model: .0f},{cl_10_model.df_resid: .0f}) = {cl_10_model.fvalue: .3f}, p = {cl_10_model.f_pvalue: .10f}")
cl_10_model.summary()
res = sm.stats.anova_lm(cl_10_model, typ= 1)
anova_table(res)

# the Embeddings were also significant with respect to a 10-cluster model F(2, 12)=9.471, p < 0.05, omega^2 = 0.530
mc = statsmodels.stats.multicomp.MultiComparison(cl_10['map_mrr'], cl_10['embeddings'])
mc_results = mc.tukeyhsd()
print(mc_results)

# Once again fastText-trained models yield a significantly higher map_mrr than either GloVe or word2vec
# but in this context word2vec vectors has only a marginally stronger effect on map_mrr than GloVe.
cl_25 = all_embeddings.loc[all_embeddings.clusters == 25,]
cl_25_model = ols('map_mrr ~ C(embeddings)', cl_25).fit()

# Seeing if the overall model is significant
print(f"Overall model F({cl_25_model.df_model: .0f},{cl_25_model.df_resid: .0f}) = {cl_25_model.fvalue: .3f}, p = {cl_25_model.f_pvalue: .10f}")
cl_25_model.summary()
res = sm.stats.anova_lm(cl_25_model, typ= 1)
anova_table(res)

# Lastly, embeddings have a medium effect on 25-cluster models too, F(2,12)=11.406, p<0.05, omega^2=0.58
mc = statsmodels.stats.multicomp.MultiComparison(cl_25['map_mrr'], cl_25['embeddings'])
mc_results = mc.tukeyhsd()
print(mc_results)

# we already know from a previous test that the fastText model's performance is inhibited by training more cluster
# Indeed, we only detected that fastText has a bigger influence on GloVe but there was no detectable difference
# between fastText and word2vec and GloVe and word2vec.

# There is strong empirical evidence which suggests that - on our chosen combined dataset and metrics -  
# training a single-cluster, baseline model on fastText is the way to go.  This  challenges 
# Ustalov et al.'s assertion that regularisation has an effect on MSE model performance.  We were not able to
# confirm that even on the same word2vec embeddings they used in their experiment.  However, we do not run 
# exhaustive grid-search tests to tune the lambda regularisation parameter, instead setting to 1., which is 
# their chosen regularisation weight for re-projected regularisation.
# Fu et. al's finding were re-confirmed, despite us chaning the setup in several ways: we recast their original 
# problem as hypernym discovery; evaluated on information retrieval metrics; cross-validated the models on a 
# different dataset.  Cluster size was certainly found to improve performance on word2vec embeddings, 
# although we didn't test larger cluster sizes.<jupyter_output>Multiple Comparison of Means - Tukey HSD,FWER=0.05
============================================
group1 group2 meandiff  lower  upper  reject
--------------------------------------------
  ft   glove  -0.0385  -0.0599 -0.017  True 
  ft    w2v   -0.0198  -0.0413 0.0016 False 
glove   w2v    0.0186  -0.0029 0.0401 False 
--------------------------------------------<jupyter_text>Analyse performance of MSE against baseline<jupyter_code>base_w2v = df.loc[(df.embeddings=='w2v'),]#&(df.clusters==1),]

w2v_naive_model = ols('map_mrr ~ C(model)*C(clusters)', base_w2v).fit()
print_model(w2v_naive_model)
w2v_naive_model.summary()
res = sm.stats.anova_lm(w2v_naive_model, typ= 1)
anova_table(res)
mc = statsmodels.stats.multicomp.MultiComparison(base_w2v['map_mrr'], base_w2v['clusters'])
mc_results = mc.tukeyhsd()
print(mc_results)<jupyter_output>Multiple Comparison of Means - Tukey HSD,FWER=0.05
============================================
group1 group2 meandiff  lower  upper  reject
--------------------------------------------
  1      10    0.0229  -0.0304 0.0761 False 
  1      25    0.0376  -0.0157 0.0908 False 
  10     25    0.0147  -0.0385 0.0679 False 
--------------------------------------------
{'fraction_non_alphanumeric': 0.15920951321755497, 'fraction_numerical': 0.05552228590982976, 'mean_word_length': 3.9097018059638806, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '489935', 'license': 'MIT', 'n_tokens_mistral': 4356, 'n_tokens_neox': 3901, 'n_words': 1205, 'path': 'ANOVA Analysis.ipynb', 'repo_name': 'jfarrugia-uom/hyperstar', 'size': '11689'}