<jupyter_start><jupyter_text>word2vec vs glove vs count @ wikipedia<jupyter_code>d = {'expansions__noise': 0.0, 
     'expansions__decode_handler': 'SignifiedOnlyFeatureHandler', 
     'expansions__k': 3, 
     'expansions__vectors__rep': 0,
     'expansions__vectors__unlabelled_percentage': 100.0,
     'expansions__vectors__unlabelled' : 'wiki',
     'expansions__allow_overlap': 0,
     'labelled': 'amazon_grouped-tagged', 
     'expansions__vectors__algorithm': 'word2vec', 
     'document_features_tr': 'J+N+AN+NN',
     'document_features_ev': 'AN+NN',
     'expansions__vectors__unlabelled_percentage':100,
     'expansions__entries_of':None,
     'expansions__vectors__composer__in': ['Add', 'Mult', 'Right', 'Left']
    }

def _get(d):
    return [x.id for x in Experiment.objects.filter(**d).\
            order_by('expansions__vectors__algorithm', 'expansions__vectors__composer')]

w2v = _get(d)
d['expansions__vectors__algorithm'] =  'glove'
glove = _get(d)
d['expansions__vectors__algorithm'] =  'count_windows'
wins = _get(d)
d['expansions__vectors__algorithm'] =  'count_dependencies'
deps = _get(d)

print('wins', wins, '\nglove', glove, '\nw2v', w2v)
horder = ['deps', 'wins', 'glove', 'word2vec']
xorder = ['Add', 'Mult', 'Left', 'Right']
with sns.color_palette("cubehelix", 4):
    diff_plot_bar(  [deps, wins, glove, w2v], horder,
                    [Experiment.objects.get(id=id).expansions.vectors.composer for id in w2v],
                    xlabel='', hline_at=random_vect_baseline(),
hue_order=horder, order=xorder)
plt.savefig('plot-w2v-vs-glove.pdf', format='pdf', dpi=300, bbox_inches='tight', pad_inches=0.1)

# sanity check
from itertools import chain
for id in chain.from_iterable([wins, glove, w2v]):
    print('id %d, score %.2f'%(id, Results.objects.get(id=id, classifier=CLASSIFIER).accuracy_mean))<jupyter_output>wins [27, 29, 28, 30] 
glove [97, 99, 98, 100] 
w2v [34, 36, 35, 37]
id 27, score 0.44
id 29, score 0.31
id 28, score 0.41
id 30, score 0.31
id 97, score 0.45
id 99, score 0.32
id 98, score 0.39
id 100, score 0.32
id 34, score 0.53
id 36, score 0.35
id 35, score 0.47
id 37, score 0.34<jupyter_text>Are differences significant?<jupyter_code>pairwise_significance_exp_ids(zip(wins,deps), ['expansions__vectors__composer', 
                                              'expansions__vectors__algorithm'])
pairwise_significance_exp_ids(zip(wins,glove), ['expansions__vectors__composer', 
                                              'expansions__vectors__algorithm'])<jupyter_output>Running significance for (27, 97)
Running significance for (29, 99)
Running significance for (28, 98)
Running significance for (30, 100)
{'fraction_non_alphanumeric': 0.13820224719101123, 'fraction_numerical': 0.049812734082397, 'mean_word_length': 4.352705410821644, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '509289', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1104, 'n_tokens_neox': 958, 'n_words': 204, 'path': 'notebooks/w2v_vs_glove_vs_count_NPs.ipynb', 'repo_name': 'mbatchkarov/ExpLosion', 'size': '2670'}