<jupyter_start><jupyter_text>Variation due to `word2vec`'s random initialisation - fairly small at the word analogy task<jupyter_code>%cd ~/NetBeansProjects/ExpLosion/
from itertools import chain
from notebooks.common_imports import *
from gui.output_utils import *
from gui.user_code import pretty_names, pairwise_significance_exp_ids

sns.timeseries.algo.bootstrap = my_bootstrap
sns.categorical.bootstrap = my_bootstrap
def get(corpus='amazon_grouped-tagged', rep=0, avg=False, reorder=False,
       composers=['Add', 'Mult', 'Left', 'Right'], k=[3]):
    query_dict = {
        'expansions__use_similarity': 0, 
          'expansions__neighbour_strategy':'linear',
          'expansions__vectors__dimensionality': 100, 
          'document_features_ev': 'AN+NN', 
          'document_features_tr': 'J+N+AN+NN', 
          'expansions__allow_overlap': False,
          'expansions__entries_of': None,
          'expansions__vectors__algorithm': 'word2vec',
          'expansions__vectors__composer__in': composers, 
          'expansions__vectors__unlabelled': 'wiki',
          'expansions__decode_handler': 'SignifiedOnlyFeatureHandler',
          'expansions__noise': 0,
          'expansions__use_similarity': 0, 
          'expansions__k__in':k,
          'expansions__vectors__unlabelled_percentage': 15,
          'expansions__vectors__rep': rep,
          'expansions__vectors__avg': avg,
          'expansions__vectors__reorder': reorder,
          'labelled':corpus}
    return [foo.id for foo in Experiment.objects.filter(**query_dict)]
ids = list(chain.from_iterable(get(rep=r) for r in [0, 1, 2]))
print(ids)
df = dataframe_from_exp_ids(ids, fields_to_include={'View':'expansions__vectors__rep',
                                                    'Composer': 'expansions__vectors__composer'})
with sns.color_palette("cubehelix", 4):
    g = sns.factorplot(data=df, x='Composer', y='Accuracy', hue='View', 
                   hue_order='0 1 2'.split(), 
                   kind='bar', ci=68, aspect=2);
plt.savefig('plot-w2v_random_init_var.pdf', format='pdf', dpi=300, bbox_inches='tight', pad_inches=0.1)
ids = list(chain.from_iterable(get(rep=r, composers=['Add']) for r in [0, 1, 2]))
ids
# are the differences significant
sign_df, _, _ = get_demsar_params(ids, ['expansions__vectors__rep'])
sign_df<jupyter_output><empty_output><jupyter_text>Repeats on R2 corpusDoes the smaller R2 dataset find differences between repeats on the a sample of approx the same size?  There is a difference up to 4%, but it is not significant<jupyter_code>ids = list(chain.from_iterable(get(corpus='reuters21578/r8-tagged-grouped', rep=r) for r in [0, 1, 2]))
print(ids)
get_demsar_params(ids, ['expansions__vectors__rep'])[0]
ids = get(rep=0, composers=['Add'], k=[3,30]) +\
      get(rep=3, avg=True, composers=['Add'], k=[3,30]) +\
      list(chain.from_iterable(get(rep=i, reorder=True, composers=['Add'], k=[3,30]) for i in [2,3,4,5]))
print(ids)
df = dataframe_from_exp_ids(ids, fields_to_include={'rep':'expansions__vectors__rep',
                                                    'avg':'expansions__vectors__avg',
                                                    'dice':'expansions__vectors__reorder',
                                                    'k': 'expansions__k',
                                                    'Composer': 'expansions__vectors__composer'}).convert_objects(convert_numeric=True)

df['method'] = 'avg3'
df.loc[df.rep==0, 'method'] = 'std'
for i in [2,3,4,5]:
    df.loc[(df.rep==i) & (df.dice==1), 'method'] = 'dice%d'%i
df = df.drop('avg dice rep'.split(), axis=1)
df['Method'] = df.method
with sns.color_palette("cubehelix", 6):
    g = sns.factorplot(data=df, x='k', y='Accuracy', hue='Method', 
                       kind='bar', ci=68, aspect=2);
    g.set(ylim=(.3, None))
plt.savefig('plot-w2v_random_init_boost.pdf', format='pdf', dpi=300, bbox_inches='tight', pad_inches=0.1)
Experiment.objects.get(expansions__k=30, expansions__vectors__composer='Add',
                       expansions__vectors__reorder=False).id
# 55= k=30, 100% wiki
# 75= k=3, 15%wiki
get_ci(55)[1], get_ci(370)[1]<jupyter_output><empty_output><jupyter_text>Are the differences significant?<jupyter_code>ids1 = [i for i in ids if Experiment.objects.get(id=i).expansions.vectors.composer=='Add']
print(ids1)
get_demsar_params(ids, ['expansions__vectors__composer', 
                        'expansions__vectors__rep'])[0]<jupyter_output>[75, 71, 296, 297, 298, 299]<jupyter_text>Compare unigram vectors between multiple runsHow many of the top `n` neighbour of some entries are the same accross multiple runs<jupyter_code>from glob import glob
from discoutils.thesaurus_loader import Vectors as V
from random import sample
from itertools import combinations

pattern = '/lustre/scratch/inf/mmb28/FeatureExtractionToolkit/word2vec_vectors/word2vec-wiki-15perc.unigr.strings.rep*'
files = sorted(glob(pattern))
thes = [V.from_tsv(f) for f in files]
for t in thes:
    t.init_sims(n_neighbors=100)
def dice(n1, n2):
    return 2 * len(set(n1) & set(n2))/ (len(n1) + len(n2))
for i,j in combinations(range(len(thes)), 2):
    print(i, j, dice(thes[i].keys(), thes[j].keys()))
def dice_loop(words, thes, log=False):
    dice_data = []
    for i, j in combinations(range(len(thes)), 2):
        pair_id = '%d & %r'%(i+1, j+1 if j < 3 else 'A')
        if log:
            print('Doing pair', pair_id, flush=True)
        for num, word in enumerate(words):
            n1 = [x[0] for x in thes[i].get_nearest_neighbours(word)]
            n2 = [x[0] for x in thes[j].get_nearest_neighbours(word)]
            if n1 and n2:
                dice_data.append([pair_id, word, dice(n1, n2)])
    return dice_data
sampled_words = sample(list(thes[0].keys()), 5000)
dice_data = dice_loop(sampled_words, thes, log=True)
df3 = pd.DataFrame(dice_data, columns='Views Word Dice'.split())
with sns.axes_style("white"):
    g = sns.FacetGrid(df3, col="Views", col_wrap=3);
    g.map(sns.distplot, 'Dice', kde=True);

for ax in g.axes.flat:
    sparsify_axis_labels(ax)
    ax.set_xlim(0, 1.01)
    ax.set_yticklabels([])

sns.despine(left=True, bottom=True)
plt.savefig('plot-w2v_random_init_neigh_overlap.pdf', format='pdf', dpi=300, bbox_inches='tight', pad_inches=0.1)<jupyter_output><empty_output><jupyter_text>ObservationsNeighbours tend to be quite different over multiple runs, but the overall accuracy of the classification task changes very little Qualitative analysisSee belowSeems to me good neighbours (which seem sensible) tend to be the same across repeated runs<jupyter_code>def multiway_dice(entry, thesauri):
    df = pd.DataFrame(dice_loop([entry], thesauri), columns='Pair Word Dice'.split())
    return df.Dice.mean()
df = compare_neighbours(thes, [0, 1, 2, 4, 5],
                        words=['balkans/N', 'lesbian/J', 'ottawa/N', 'sneaker/N', 'essay/N', 'falsify/V', 'inborn/J'])
df['mw_dice'] = [multiway_dice(feat, thes) for feat in df.index]
df.to_csv('compare_repeated_w2v.csv')
df.sort('mw_dice')
print(pd.DataFrame(df.stack()).to_latex())
df = compare_neighbours(thes, [0, 1, 2, 4, 5])
df.head()
all_feats = set.union(*[set(v.keys()) for v in thes[:3]])
len(all_feats)
from collections import Counter
Counter(sum(f in v for v in thes[:3]) for f in all_feats)
pattern = '/lustre/scratch/inf/mmb28/FeatureExtractionToolkit/word2vec_vectors/word2vec-wiki-100perc.unigr.strings.rep0'
v_avg = V.from_tsv(pattern)
v_avg.init_sims(n_neighbors=10)
len(v_avg)
new_entries = set(v_avg.keys()) - set(thes[0].keys())
old_entries = set(v_avg.keys()) & set(thes[0].keys())
new_nouns = [x for x in new_entries if x.endswith('/N')]
len(new_entries), len(old_entries), len(new_nouns)
new_entries
v_avg.get_nearest_neighbours('measured/J')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14344420187491974, 'fraction_numerical': 0.02439964042635161, 'mean_word_length': 4.0703125, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '507374', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2955, 'n_tokens_neox': 2771, 'n_words': 559, 'path': 'notebooks/w2v_repeats_effect_of_random_initialisation.ipynb', 'repo_name': 'mbatchkarov/ExpLosion', 'size': '7787'}