<jupyter_start><jupyter_text>GloVe implementation speed tests (section 2.1)<jupyter_code>__author__ = '<NAME> and <NAME>'<jupyter_output><empty_output><jupyter_text>This notebook requires a compiled version of [the official GloVe code release](https://nlp.stanford.edu/projects/glove/) to be in the directory `official-glove/build`.<jupyter_code>import numpy as np
import os
import timeit
import random
import pandas as pd
import subprocess
import sys
from utils import build_weighted_matrix

# GloVe implementations:
from nonvectorized_glove import GloVeModel
from tf_mittens import Mittens
from vector_glove import VectorGlove<jupyter_output><empty_output><jupyter_text>Random corporaWe build simulated corpora from a Zipfian distribution of words, and we build matrices from them using the same methods used for our empirical corpora.<jupyter_code>def generate_corpus(n_words=1000000):
    """Returns a string of integers with a Zipfian distribution."""
    # For `n_words` at 1000 or more, these settings tend 
    # to return matrices that about the sparsity we see in
    # our empirical matrices.
    words = [str(i) for i in np.random.zipf(1.7, n_words)]
    corpus = " ".join(words)
    return corpus     
def generate_corpus_and_matrix(vocab_size, window_size):
    """Creates a corpus and associated matrix. This helps
    ensure parity between tests with the official distribution,
    where we start by reading in a corpus, and tests with the
    Python implementations, where we start with a matrix.    
    """
    # Setting `n_words` this way is an attempt to ensure
    # that we get a matrix of the right size we want. For
    # large vocabularies, really big corpora are needed.
    n_words = np.min([vocab_size * 10000, int(5e8)])
    corpus = generate_corpus(n_words)    
    tokenizer = lambda x: x.split(' ')
    X = build_weighted_matrix(
        [corpus], 
        tokenizing_func=tokenizer, 
        vocab_size=vocab_size, 
        window_size=window_size)
    return X.values, corpus<jupyter_output><empty_output><jupyter_text>Experiments<jupyter_code>n = 50
xmax = 100
alpha = 0.75
max_iter = 10
eta = 0.01
tol = 1e-4
window_size = 10<jupyter_output><empty_output><jupyter_text>Official GloVe distribution<jupyter_code>def official_glove_experiment(corpus, vocab_size, verbose=False):
    BUILDDIR = 'official-glove/build'    
    CORPUS_FILE = 'official-glove/speed-test-corpus.txt'
    VOCAB_FILE = 'official-glove/vocab.txt'
    COOCCUR_FILE = 'official-glove/cooccurrence.bin'
    SHUFFLE_FILE = 'official-glove/cooccurrence.shuf.bin'
    VECTORS_FILE = 'official-glove/vectors'
    
    with open(CORPUS_FILE, 'wt') as f:
        f.write(corpus)                
    
    VERBOSE = 0
    MEMORY = 4.0
    NUM_THREADS = 1
        
    vocab_cmd = [
        '{}/vocab_count'.format(BUILDDIR),
        '-max-vocab', str(vocab_size),
        '-min-count', '1',
        '< {} > {}'.format(CORPUS_FILE, VOCAB_FILE)]    
    
    cooccur_cmd = [
        '{}/cooccur'.format(BUILDDIR), 
        '-memory', str(MEMORY),
        '-verbose', '0',
        '-vocab-file', VOCAB_FILE,
        '-window-size', str(window_size),
        '< {} > {}'.format(CORPUS_FILE, COOCCUR_FILE)]  
    
    shuffle_cmd = [
        '{}/shuffle'.format(BUILDDIR),
        '-memory', str(MEMORY), 
        '-verbose', '0',
        '< {} > {}'.format(COOCCUR_FILE, SHUFFLE_FILE)]
    
    glove_cmd = [
        '{}/glove'.format(BUILDDIR), 
        '-save-file', VECTORS_FILE, 
        '-threads', str(NUM_THREADS), 
        '-input-file', SHUFFLE_FILE,
        '-x-max', str(xmax),
        '-iter', str(max_iter),
        '-vector-size', str(n),
        '-binary', '0',
        '-vocab-file', VOCAB_FILE]
    
    for cmd in [vocab_cmd, cooccur_cmd, shuffle_cmd]:        
        x = subprocess.run(" ".join(cmd), shell=True, check=True, stdout=subprocess.PIPE)
        if verbose:
            print("="*70)
            print(" ".join(cmd) + ";")
            print(x)
        
    def run_test():
         subprocess.run(glove_cmd)
                        
    secs = timeit.timeit(run_test, number=1)
    
    VECTORS_FILE = VECTORS_FILE + ".txt"
            
    X = pd.read_csv(VECTORS_FILE, delim_whitespace=True, index_col=0).values
      
    for f in [CORPUS_FILE, VOCAB_FILE, COOCCUR_FILE, SHUFFLE_FILE, VECTORS_FILE]:
        os.remove(f)
                
    return secs, X<jupyter_output><empty_output><jupyter_text>Vectorized Tensorflow<jupyter_code>def vectorized_tensorflow_experiment(X):
    model = Mittens(
        n=n, 
        xmax=xmax, 
        alpha=alpha, 
        max_iter=max_iter, 
        eta=eta, 
        tol=tol,
        display_progress=0)
    
    def run_test():    
        model.fit(X)

    return timeit.timeit(run_test, number=1)<jupyter_output><empty_output><jupyter_text>Non-vectorized TensorflowAdapted Grady Simon (https://github.com/GradySimon/tensorflow-glove)<jupyter_code>def nonvectorized_tensorflow_experiment(X):    
    model = GloVeModel(
        n=n, 
        alpha=alpha, 
        xmax=xmax,
        eta=eta,
        max_iter=max_iter)
    
    def run_test():
        model.fit(X)
        
    return timeit.timeit(run_test, number=1)<jupyter_output><empty_output><jupyter_text>Vectorized Numpy<jupyter_code>def vectorized_numpy_experiment(X):
    model = VectorGlove(
        n=n, 
        xmax=xmax, 
        alpha=alpha, 
        max_iter=max_iter,
        learning_rate=eta, 
        display_progress=False)
    
    def run_test():
        model.fit(X)
        
    return timeit.timeit(run_test, number=1)<jupyter_output><empty_output><jupyter_text>Experiment runs<jupyter_code>def timing_experiment(
        n_tests=5, 
        vocab_sizes=(5000, 10000, 20000),
        funcs=(vectorized_numpy_experiment,
               nonvectorized_tensorflow_experiment,
               vectorized_tensorflow_experiment,
               official_glove_experiment)):               
    data = []            
    for vocab_size in vocab_sizes:   
        print("Vocab size: {:,}".format(vocab_size))
        for t in range(1, n_tests+1):
            X, corpus = generate_corpus_and_matrix(vocab_size, window_size) 
            print("\tX has vocab size {:,} and {:,} non-0 entries".format(
                    X.shape[0], np.count_nonzero(X)))
            for func in funcs:
                print("\t", t, func.__name__)                
                if func.__name__ == 'official_glove_experiment':
                    secs, X = func(corpus, vocab_size)
                else:                
                    secs = func(X)
                experiment_name = func.__name__.replace("_experiment", "")
                data.append({
                    'test_num': t, 
                    'iterations': max_iter,
                    'vocab_size': X.shape[0],
                    'model': experiment_name, 
                    'seconds': secs})
                
        pd.DataFrame(data).to_csv("tmp-speed-interim_to{}.csv".format(vocab_size))

    df = pd.DataFrame(data)     
    return df
def summarize(data, digits=2):
    results = data.groupby(['model', 'vocab_size']).apply(
        lambda x: x['seconds'].sum() / x['iterations'].sum())
    results = results.to_frame().rename(columns={0: 'mean seconds per iteration'})
    return results.round(digits)<jupyter_output><empty_output><jupyter_text>CPU<jupyter_code>cpu_data = timing_experiment()
cpu_data_20K = timing_experiment(
     funcs=(vectorized_numpy_experiment,
               nonvectorized_tensorflow_experiment,
               vectorized_tensorflow_experiment,
               official_glove_experiment),
    vocab_sizes=(20000,))
cpu_data.to_csv("results/speed-tests-cpu.csv")
summarize(cpu_data)<jupyter_output><empty_output><jupyter_text>GPU<jupyter_code>gpu_data = timing_experiment(
     funcs=(vectorized_tensorflow_experiment,
            nonvectorized_tensorflow_experiment))
gpu_data.to_csv("results/speed-tests-gpu.csv")
summarize(gpu_data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11150043799274184, 'fraction_numerical': 0.010511825804029533, 'mean_word_length': 2.672794117647059, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 50, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '211133', 'license': 'Apache-2.0', 'n_tokens_mistral': 2790, 'n_tokens_neox': 2652, 'n_words': 501, 'path': 'Papers/Mittens/speed_tests.ipynb', 'repo_name': 'markgraves/roamresearch', 'size': '8009'}