<jupyter_start><jupyter_text>Compressing Word EmbeddingsDownloadable version of GloVe embedding (with fallback source), as well as downloadable versions of sparsified GloVe embedding from own hosting.Include instructions for Levy test-suite installation, so that any given embedding can be tested.And functions/tools to play with the loaded embedding (of whatever type).  Download Pre-Built EmbeddingsThe following needs to be Pythonized :<jupyter_code>RCL_BASE=('http://redcatlabs.com/downloads/'+
          'deep-learning-workshop/notebooks/data/'+
          'research/ICONIP-2016/')

"""
# http://redcatlabs.com/downloads/deep-learning-workshop/LICENSE

# Files in : ${RCL_BASE} :

# :: These are either as downloaded from GloVe site, or generated by Levy code
# 507206240 Oct 25  2015 2-pretrained-vectors_glove.6B.300d.hkl
# 160569440 May 14 14:57 1-glove-1-billion-and-wiki_window11-lc-36_vectors.2-17.hkl

# :: These are originals - citation desired...
#  53984642 May 15 14:13 sparse.6B.300d_S-21_2n-shuf-noise-after-norm_.2.01_6-75_4000_GPU-sparse_matrix.hkl
# 148011260 May 15 14:13 sparse.6B.300d_S-21_2n-shuf-noise-after-norm_.2.01_6-75_4000_GPU-sparsity_recreate.hkl

#  57907373 May 15 02:53 sparse.6B.300d_S-21_2n-shuf-noise-after-norm_4k_.2.01_1-50_5000_GPU-sparse_matrix.hkl
# 147946219 May 15 02:52 sparse.6B.300d_S-21_2n-shuf-noise-after-norm_4k_.2.01_1-50_5000_GPU-sparsity_recreate.hkl


# Spare?
# 122248980 May  2 13:09 misc/sparse.6B.300d_T-21_3500.1024@0.05-GPU-sparse_matrix.hkl
# 447610336 May  2 13:04 misc/sparse.6B.300d_T-21_3500.1024@0.05-GPU-sparsity_recreate.hkl
#  53312127 May 11 14:10 misc/sparse.6B.300d_S-21_2n-shuf_1024@6.75_2000_GPU-sparse_matrix.hkl
# 148027055 May 11 14:10 misc/sparse.6B.300d_S-21_2n-shuf_1024@6.75_2000_GPU-sparsity_recreate.hkl
#  57054795 May 11 12:09 misc/sparse.6B.300d_S-21_2n-shuf_4096@1.50_2000_GPU-sparse_matrix.hkl
# 147997824 May 11 12:09 misc/sparse.6B.300d_S-21_2n-shuf_4096@1.50_2000_GPU-sparsity_recreate.hkl

"""

import os, requests

def get_embedding_file( hkl ):  
    if not os.path.isfile(os.path.join('data', hkl)):
        # ... requests.get( RCL_BASE + hkl)
        pass<jupyter_output><empty_output><jupyter_text>Download the Omer-Levy Test RegimeSee : https://levyomer.files.wordpress.com/2015/03/improving-distributional-similarity-tacl-2015.pdfTo download the test suite, please run the script ```download-tests.bash```.  Function to test an Embedding .txt File<jupyter_code>import os, subprocess
from __future__ import print_function

def test_embedding_file(vectors_txt, vocab_max=131072 ):
  # Do we need to process VECTORS_FILE->{ VECTORS_WORDS, VECTORS_NPY }?
  # Answer = YES : the .words is required, and is used to create .npy and .vocab
    
  vectors_txt_words = '%s.words' % (vectors_txt,)
  if not os.path.isfile(vectors_txt_words) or os.stat(vectors_txt).st_mtime>os.stat(vectors_txt_words).st_mtime:
    print("Reading %s" % (vectors_txt,))
    # This is just a copy of 'text file' with the vocab_size and embedding_size pre-pended
    #echo "131072 300" > ${VECTORS_WORDS}
    #head -131072 ${VECTORS_FILE} >> ${VECTORS_WORDS}
    with open(vectors_txt) as fin:
      first_line = fin.readline()
      embedding_dim = len(first_line.strip().split()) -1 
      vocab_size = len(fin.readlines()) +1  # Ouch! - read in whole file to find length

      if vocab_size>vocab_max:
        vocab_size=vocab_max
            
    print("Building %s" % (vectors_txt_words,))
    with open(vectors_txt) as fin:
      with open(vectors_txt_words, 'wt') as fout:
        # Write the first line, which, ironically, will be discarded by the omerlevy code
        fout.write("%d %d\n" % (vocab_size, embedding_dim))
                
        # And copy over at most vocab_max lines of the original file 
        for i, line in enumerate(fin.readlines()):
          if i>vocab_size:
            break
          fout.write(line)
    print("Built %s as %d %d-d vectors" % (vectors_txt_words, vocab_size, embedding_dim))
    
  vectors_txt_npy   = '%s.npy' % (vectors_txt_words,)
  vectors_txt_vocab = '%s.vocab' % (vectors_txt_words,)
  if not os.path.isfile(vectors_txt_npy) or os.stat(vectors_txt_words).st_mtime>os.stat(vectors_txt_npy).st_mtime:
    print("Building %s and %s" % (vectors_txt_npy, vectors_txt_vocab, ))
    # Sadly, we can't just invoke this as a python function - need to go via shell...
    subprocess.call([ "python", "../omerlevy/hyperwords/text2numpy.py", vectors_txt_words ])
    print("Built %s and %s" % (vectors_txt_npy, vectors_txt_vocab, ))

   
  def run_word_test(test_str, test_cmd):
    print("  %s" % ((test_str+' '*30)[:30],), end='')
    #subprocess.call(test_cmd)
    try:
      res = subprocess.check_output( test_cmd, stderr=subprocess.STDOUT,)
      print(" : %s" % (res.strip(),))
    except subprocess.CalledProcessError as e:
      print(" : ERROR : %s" % (test_str,))
      print(e)

  def run_word_similarity(test_str, test_set):
    test_cmd = [ 
        "python", "../omerlevy/hyperwords/ws_eval.py", "VECTORS", vectors_txt, 
        "../omerlevy/testsets/ws/%s" % (test_set,) 
    ]
    run_word_test(test_str, test_cmd)

  def run_word_analogy(test_str, test_set):
    #python ../omerlevy/hyperwords/analogy_eval.py VECTORS ${VECTORS_FILE} ../omerlevy/testsets/analogy/google.txt
    test_cmd = [ 
        "python", "../omerlevy/hyperwords/analogy_eval.py", "VECTORS", vectors_txt, 
        "../omerlevy/testsets/analogy/%s" % (test_set,) 
    ]
    run_word_test(test_str, test_cmd)

  if True:
    print("Word Similarity Tests (~5 seconds each)")
    run_word_similarity("WS353 Similarity  ", "ws353_similarity.txt")
    run_word_similarity("WS353 Relatedness ", "ws353_relatedness.txt")
    run_word_similarity("Bruni MEN         ", "bruni_men.txt")
    run_word_similarity("Radinsky M.Turk   ", "radinsky_mturk.txt")
    run_word_similarity("Luoung Rare Words ", "luong_rare.txt")

  if True:
    print("Word Analogy Tests (~60 seconds each)")
    run_word_analogy("Google Analogy    ", "google.txt")
    run_word_analogy("MSR Analogy       ", "msr.txt")

import numpy as np
import hickle<jupyter_output><empty_output><jupyter_text>Choose an embedding to Load<jupyter_code>#embedding_file = '../data/2-pretrained-vectors_glove.6B.300d.hkl'
#embedding_file = '../data/1-glove-1-billion-and-wiki_window11-lc-36_vectors.2-17.hkl'

#embedding_file = '../data/lloyds_normed_8.hkl'

# 1024-d embeddings : sparse and recreated
embedding_file = '../data/sparse.6B.300d_S-21_2n-shuf-noise-after-norm_.2.01_6-75_4000_GPU-sparse_matrix.hkl'
#embedding_file = '../data/sparse.6B.300d_S-21_2n-shuf-noise-after-norm_.2.01_6-75_4000_GPU-sparsity_recreate.hkl'

# 4096-d embeddings : sparse and recreated
#embedding_file = '../data/sparse.6B.300d_S-21_2n-shuf-noise-after-norm_4k_.2.01_1-50_5000_GPU-sparse_matrix.hkl'
#embedding_file = '../data/sparse.6B.300d_S-21_2n-shuf-noise-after-norm_4k_.2.01_1-50_5000_GPU-sparsity_recreate.hkl'

d = hickle.load(embedding_file)
vocab, embedding = d['vocab'], d['embedding']
vocab_orig = d.get('vocab_orig', vocab)

dictionary = dict( (word, i) for i,word in enumerate(vocab) if i<len(embedding) )
dictionary_orig = dict( (word, i) for i,word in enumerate(vocab_orig) if i<len(embedding) )

print("Embedding loaded :", embedding.shape)   # (vocab_size, embedding_dimension)=(rows, columns)
embedding_normed = embedding / np.linalg.norm(embedding, axis=1)[:, np.newaxis]

def save_embedding_for_tests(vocab, embedding, save_filename_txt='../data/tmp.embedding.txt'):
  with open(save_filename_txt, 'wb') as f:
    for l in range(0, embedding.shape[0]):
      f.write("%s %s\n" % (vocab[l], ' '.join([ ('0' if x==0. else ("%.6f" % (x,))) for x in embedding[l, :].tolist() ]), ))
  print("Saved to %s" % (save_filename_txt, ))

def save_embedding_to_hickle(vocab, embedding_save, save_filename_hkl, vocab_orig=None):
  print("About to save to %s" % (save_filename_hkl,))
  d=dict( 
    vocab=vocab, 
    vocab_orig=vocab if vocab_orig is None else vocab,
    embedding=embedding_save,
  )
  hickle.dump(d, save_filename_hkl, mode='w', compression='gzip')
  print("Saved to %s" % (save_filename_hkl,))
  
vocab[0]
entries = [ x for x in embedding[0].tolist() if x!=0.0 ]
len(entries)
#45 

for w in 'the iraq baghdad uk london criminal apple some hypothesis maximal innocuous'.split(' '):
  i=dictionary[w]
  entries = [ x for x in embedding[i].tolist() if x!=0.0 ]
  print("%20s @%6d len=%d" % (w,dictionary_orig[w],len(entries),))

  #               the @     0 len=18
  #              some @    60 len=18
  #            london @   266 len=91
  #                uk @   448 len=82
  #              iraq @   606 len=113
  #          criminal @  1449 len=104
  #             apple @  2046 len=112
  #           baghdad @  2320 len=116
  #        hypothesis @  6957 len=136
  #           maximal @ 27962 len=107
  #         innocuous @ 30111 len=86


if False:
  # Look at per-position best words
  for i in range(0, embedding.shape[1], 10):
    best_words_j = np.argsort( -embedding[:, i ] )[0:10]
    for j in best_words_j:
      print("%4i -> %s" % (i, vocab[j],))
    print('')

if False:
  #i=2000
  values = [x for x in (-np.sort( -embedding[i] )).tolist() if x>0. ]
  print("values: ["+', '.join([ ('%.4f' % (x,)) for x in values ])+']')
  #values: [1.1442, 0.9337, 0.9333, 0.9257, 0.7520, 0.5529, 0.4818, 0.4740, 0.4568, 0.4554, 0.4434, 0.4419, 0.4334, 0.4187, 0.4175, 0.4068, 0.4005, 0.3989, 0.3698, 0.3421, 0.3206, 0.3151, 0.3150, 0.3120, 0.3119, 0.3067, 0.3010, 0.2948, 0.2853, 0.2828, 0.2816, 0.2815, 0.2799, 0.2793, 0.2764, 0.2714, 0.2636, 0.2570, 0.2507, 0.2487, 0.2336, 0.2336, 0.2335, 0.2328, 0.2325, 0.2323, 0.2255, 0.2227, 0.2227, 0.2226, 0.2208, 0.2178, 0.2159, 0.2134, 0.2067, 0.2049, 0.1947, 0.1935, 0.1932, 0.1926, 0.1921, 0.1914, 0.1897, 0.1894, 0.1832, 0.1782, 0.1766, 0.1730, 0.1714, 0.1683, 0.1662, 0.1638, 0.1629, 0.1602, 0.1568, 0.1561, 0.1452, 0.1419, 0.1399, 0.1372, 0.1370, 0.1352, 0.1350, 0.1342, 0.1334, 0.1334, 0.1302, 0.1289, 0.1268, 0.1243, 0.1230, 0.1211, 0.1192, 0.1113, 0.1051]

  print("changes: ["+', '.join([ ('%.1f' % (values[i+1]/values[i]*100.,)) for i in range(0,len(values)-1) ])+']')
  #changes: [81.6, 100.0, 99.2, 81.2, 73.5, 87.1, 98.4, 96.4, 99.7, 97.3, 99.7, 98.1, 96.6, 99.7, 97.4, 98.4, 99.6, 92.7, 92.5, 93.7, 98.3, 100.0, 99.0, 100.0, 98.3, 98.1, 97.9, 96.8, 99.1, 99.6, 100.0, 99.4, 99.8, 99.0, 98.2, 97.1, 97.5, 97.6, 99.2, 93.9, 100.0, 100.0, 99.7, 99.9, 99.9, 97.1, 98.8, 100.0, 100.0, 99.2, 98.6, 99.1, 98.9, 96.9, 99.1, 95.0, 99.4, 99.9, 99.7, 99.8, 99.6, 99.1, 99.8, 96.7, 97.3, 99.1, 98.0, 99.1, 98.2, 98.8, 98.6, 99.4, 98.3, 97.9, 99.5, 93.1, 97.7, 98.6, 98.1, 99.8, 98.7, 99.9, 99.4, 99.4, 100.0, 97.6, 99.0, 98.4, 98.0, 99.0, 98.4, 98.5, 93.4, 94.4]


w='motorcycle'
w_i=dictionary[w]

#top_i =np.argmax(embedding[w_i])
good_i =np.argsort( -embedding[w_i] )

for i in range(0,10):
  best_words_j = np.argsort( -embedding[:, good_i[i] ] )[0:12]
  
  #for j in best_words_j:
  #  print("%s" % (vocab[j],))
  #print('')
  
  print("%s" % (', '.join( [ vocab[j] for j in best_words_j] ), ) )<jupyter_output><empty_output><jupyter_text>Embedding Exploration Utility functions<jupyter_code>def vector_for(w):
  w_i=dictionary[w]
  return embedding[w_i]

def l2_normed(e):
  return e / np.sqrt( np.dot(e,e) )

def cosine(a,b):
  return np.dot(l2_normed(a), l2_normed(b))

def top_senses_for(e):
  good_i = np.argsort( -e )
  for i in range(0,10):
    best_words_j = np.argsort( -embedding[:, good_i[i] ] )[0:12]
    print("%s" % (', '.join( [ vocab[j] for j in best_words_j] ), ) )

def closest_to(e, n=10):
  closest = np.argsort( - np.dot(embedding_normed, l2_normed(e) ) )
  return "%s" % (', '.join( [ vocab[j] for j in closest[0:n] ] ), ) 

def count_positive(e):
  return len( [ x for x in e.tolist() if x>0.0 ] )

def nonzero_positions(e):
    return [ i for (i,x) in enumerate(e.tolist()) if x!=0.0 ]

def nonneg(e):
  return np.maximum(0, e)

def closest_dist(s):
  ab,xy = s.split('=')
  (a,b),(x,y) = ab.split(':'), xy.split(':')
  print( "%s is to %s as %s is to ?%s? " % (a,b,x,y,))
  (a,b,x,y) = map(vector_for, [a,b,x,y])  # Convert to vectors
  print('  x+b-a           = %s' % (closest_to( x + b - a ),))
  print('  [x+b-a]         = %s' % (closest_to( nonneg(x + b - a) ),))
  print('  x+[b-a]         = %s' % (closest_to( x + nonneg(b-a) ),))
  print('  [x-a]+b         = %s' % (closest_to( nonneg(x-a) + b ),))
  print('  [2x-a]+[2b-a]   = %s' % (closest_to( nonneg(2*x-a) + nonneg(2*b-a) ),))
  print('  x+[b-a]+b+[x-a] = %s' % (closest_to( x+nonneg(b-a) + b+nonneg(x-a) ),))


top_senses_for(vector_for('motorbike'))

man   = vector_for('man')
woman = vector_for('woman')
king  = vector_for('king')
queen = vector_for('queen')

#top_senses_for(man)
#top_senses_for(woman)
#top_senses_for(king)
#top_senses_for(queen)

#top_senses_for(man * woman) # Intersection
#top_senses_for(man + woman) # Union
#top_senses_for(man - woman) # ??


closest_to(man)
#man, woman, girl, person, men, teenager, she, friend, he, father, her, boy, someone, mother, him, his, victim, son, who, guy
closest_to(woman)
#woman, man, girl, mother, teenager, daughter, wife, women, her, person, she, girlfriend, friend, men, husband, widow, couple, boy, someone, victim

closest_to(king)
#king, queen, henry, mswati, mongkut, eirik, charles, vajiravudh, thoden, wenceslaus, zvonimir, athelstan, vladislaus, thelred, gojong, prince, jayavarman, kalkaua, sweyn, pomare
closest_to(queen)
#queen, princess, elizabeth, king, margrethe, empress, lady, sister, prince, sirikit, <NAME>, monarch, daughter, duchess, olten, mother, infanta, rania, widow

closest_dist('pound:england=franc:france')


england,pound,america,dollar = map(vector_for, 'england pound america dollar'.split())

curr = england,pound,america,dollar = map(vector_for, 'england pound america dollar'.split())
map(count_positive, curr)
#[84, 126, 94, 134]

map(count_positive, [ england*pound, america*dollar, england*america, pound*dollar])
#[12, 14, 17, 56]

total = england+pound+america+dollar
map(count_positive, [ england*101-100*total, pound*101-100*total, america*101-100*total, dollar*101-100*total])

save_embedding_for_tests(vocab, embedding, save_filename_txt='../data/tmp.embedding.txt')
test_embedding_file('../data/tmp.embedding.txt', vocab_max=131072)
#save_embedding_for_tests(vocab, embedding, save_filename_txt='../data/lloyds_normed_8.txt')
#test_embedding_file('../data/lloyds_normed_8.txt', vocab_max=131072)
len(nonzero_positions(embedding[10202]))
# Let's make a *random* transformation matrix to 'recreate' and approximate 300d dense embedding
A = np.random.normal(loc=0.0, scale=1.0, size=(embedding.shape[1], 300))

embedding_reconstructed = np.dot(embedding, A)        # Project back down to 300d using random matrix
embedding_reconstructed.shape
embedding_reconstructed = embedding/5.0               # Check that scaling is irrelevant (ACTUALLY, THIS CHECK IS BOGUS)
embedding_reconstructed = np.where(embedding>0,1,0)   # Binarized
embedding_reconstructed = np.where(embedding>0,0,1)   # Binarized (inverted) is TERRIBLE
def construct_d(a_i, b_i, c_i, emb):
  a = emb[a_i]
  b = emb[b_i]
  c = emb[c_i]
  d = c+b-a              # This is the standard np() vector constructor
  return np.dot(emb, d.T)   # Return the score of this vs all the embedding vectors

# Let's build an analogy tester right here...
def test_analogies(test_set="msr", emb=embedding, construction_fn=construct_d):
  trials, trials_w,total,total_possible=[],[],0,0
  with open("../omerlevy/testsets/analogy/%s.txt" % test_set) as f:  # 'google' or 'msr'
    for trial in f.readlines():
      t_w = [ w for w in trial.strip().split() ]
      t = [ dictionary.get(w, None) for w in t_w ]
      total+=1
      if None not in t:
        trials_w.append(t_w)
        trials.append(t)
        total_possible+=1
    
  print(trials_w[0], trials[0])

  count, correct = 0, 0
  for i,trial in enumerate(trials):
    if i % 10 >0:  # 10x thinning factor for speed (1 for accuracy)
      continue
    d_score = construction_fn( trial[0], trial[1], trial[2], emb)
    
    # Set the scores for the original vectors to useless values
    d_score[ trial[0] ] = d_score[ trial[1] ] = d_score[ trial[2] ] = -1
    
    # Now find the argmax score:
    d_i = np.argmax( d_score )
    #print( d_score.shape, d_i, i )
    
    if True and False:
      w = trials_w[i]
      print("'%s:%s=%s:%s'?  :: %s" % ( w[0], w[1], w[2], vocab[ d_i ],
                ('WIN' if trial[3]==d_i else "FAIL"),
            ))
    
    count += 1
    if trial[3]==d_i:
      correct += 1
    
    if i % 250==0:
      print("At %d : %.2f%%" % (count, 100.*correct/count))
  print("Local(%s) final : %.2f%% using %d, which is %.2f%% of total" % (test_set, 
    100.*correct/count, count, 100.*correct/count*total_possible/total,))
    
test_analogies(emb=embedding_normed) # , test_set="google")
#NO test_analogies(emb=embedding) # , test_set="google")
def remax(a):
    return a / np.amax(a)

def a_with_b(a, b):
    return remax(a+b)

def a_without_b(a, b):
    c = a.copy()
    c[ b>0 ]=0.
    return remax(c)
    
def closest_dist(s):
  ab,xy = s.split('=')
  (a,b),(x,y) = ab.split(':'), xy.split(':')
  print( "%s is to %s as %s is to ?%s? " % (a,b,x,y,))
  (a,b,x,y) = map(vector_for, [a,b,x,y])  # Convert to vectors
  print('  x+b-a           = %s' % (closest_to( x + b - a ),))
  print('  [x+b-a]         = %s' % (closest_to( nonneg(x + b - a) ),))
  print('  x+[b-a]         = %s' % (closest_to( x + nonneg(b-a) ),))
  print('  [x-a]+b         = %s' % (closest_to( nonneg(x-a) + b ),))
  print('  [2x-a]+[2b-a]   = %s' % (closest_to( nonneg(2*x-a) + nonneg(2*b-a) ),))
  print('  x+[b-a]+b+[x-a] = %s' % (closest_to( x+nonneg(b-a) + b+nonneg(x-a) ),))
  setish1 = a_with_b( a_without_b( x,  a_without_b(a, b) ), a_without_b(b, a))
  print('  setish1         = %s' % (closest_to( setish1 ),))
  setish2 = a_with_b( a_without_b( b,  a_without_b(a, x) ), a_without_b(x, a))
  print('  setish2         = %s' % (closest_to( setish2 ),))

#closest_dist('pound:england=franc:france')
closest_dist('london:england=paris:france')
closest_dist('smallest:smaller=smoothest:smoother')  # Better with set ops
#closest_dist('great:greater=classy:classier')  # No idea
#closest_dist('richest:richer=meanest:meaner')  # No idea
#closest_dist('seem:seems=develop:develops')
closest_dist('few:fewer=friendly:friendlier')

#### All MSR sets with 'fast' in them are bogus
##closest_dist('weak:weakest=fastest:fast') !!!

e=vector_for('england')
l=vector_for('london')
f=vector_for('france')
p=vector_for('paris')
#closest_to( p+e-l )

p_poss1 = a_with_b( a_without_b( p,  a_without_b(l, e) ), a_without_b(e, l))
p_poss2 = a_with_b( a_without_b( e,  a_without_b(l, p) ), a_without_b(p, l))
print( closest_to( p_poss1 ) )
print( closest_to( p_poss2 ) )
def construct_d_setish(a_i, b_i, c_i, emb):
  a = emb[a_i]
  b = emb[b_i]
  c = emb[c_i]
  #d = c + (b-a)              # This is the standard np() vector constructor
  #d = a_with_b( a_without_b( c,  a_without_b(a, b) ), a_without_b(b, a))  # This is set-ish  v1
  #d = a_with_b( a_without_b( b,  a_without_b(a, c) ), a_without_b(c, a))  # This is set-ish  v2
  d = a_with_b(a_with_b( c,  a_without_b(b, a) ), a_with_b( b, a_without_b(c, a)))  # This is set-ish  v3
  d += c + (b-a)
  return np.dot(emb, d.T)   # Return the score of this vs all the embedding vectors

test_analogies(emb=embedding_normed, construction_fn=construct_d_setish, test_set="google")
# =0,=1, A/B,A+B,B/A, A/C,A+C,C/A 
a = np.array( [0.,1., 1.,1.,0., 1.,1.,0.])
b = np.array( [0.,1., 0.,1.,1., 0.,0.,0.])
c = np.array( [0.,1., 0.,0.,0., 0.,1.,1.])
d = np.array( [0.,1., 0.,0.,1., 0.,0.,1.]) # Should this de-emphasise 'a', for instance?
d0 = c + (b-a)
d1 = a_with_b( a_without_b( c,  a_without_b(a, b) ), a_without_b(b, a))  # This is set-ish  v1
d2 = a_with_b( a_without_b( b,  a_without_b(a, c) ), a_without_b(c, a))  # This is set-ish  v2
d3 = a_with_b(a_with_b( c,  a_without_b(b, a) ), a_with_b( b, a_without_b(c, a)))  # This is set-ish  v3
d0,d3, d1,d2
embedding_reconstructed_normed = embedding_reconstructed / np.linalg.norm(embedding_reconstructed, axis=1)[:, np.newaxis]
save_embedding_for_tests(vocab, embedding_reconstructed_normed, save_filename_txt='../data/tmp.embedding.txt')
test_embedding_file('../data/tmp.embedding.txt', vocab_max=131072)
vocab[ 0:10 ]
embedding[0, 0:100]
embedding_reconstructed[0, 0:100]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16970553822152887, 'fraction_numerical': 0.07912441497659907, 'mean_word_length': 3.7826999300536257, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '377705', 'license': 'MIT', 'n_tokens_mistral': 9763, 'n_tokens_neox': 8688, 'n_words': 1796, 'path': 'notebooks/explore-embeddings.ipynb', 'repo_name': 'mdda/compressing-word-embeddings', 'size': '20516'}