<jupyter_start><jupyter_text>Word Embeddings in RedisWe will use a local Redis server running on localhost:6379<jupyter_code>import redis
import time
import numpy 
import plotly
import pickle
import struct
import numpy as np
from tqdm import tqdm_notebook as tqdm<jupyter_output><empty_output><jupyter_text>Dummy EmbeddingsFor testing purposes we will use randomly generated numpy arrays as dummy embbeddings.<jupyter_code>def embeddings(n=1000, dim=512):
    """
    Yield n tuples of random numpy arrays of *dim* length indexed by *n*
    """
    idx = 0
    while idx < n:
        yield (str(idx), numpy.random.rand(dim))
        idx += 1<jupyter_output><empty_output><jupyter_text>Conversion FunctionsSince we can't just save a NumPy array into the database, we will convert numpy arrays<jupyter_code>def adapt_array(array):
    """
    Adapt numpy array for saving into redisDB

    :param numpy.array array: NumPy array to encode
    :return: encoded NumPy array
    """
    array = array.astype(np.float32)
    h = array.shape[0]

    shape = struct.pack('>I',h)
    encoded = shape + array.tobytes()

    return encoded


def convert_array(encoded):
    """

    :param BLOG encoded: encoded NumPy array
    :return: One steaming hot NumPy array
    :rtype: numpy.array
    """
    
    h = struct.unpack('>I',encoded[:4])
    array = np.frombuffer(encoded, dtype=np.float32, offset=4).reshape(h)
    
    return array

db = redis.Redis(host='localhost', port=6379, db=0)

%%time
for key, emb in embeddings():
    arr = adapt_array(emb)
    db.set(key, arr)
%%time
for key, _ in embeddings():
    obj = db.get(key)
    emb = convert_array(obj)
    assert(type(emb) is numpy.ndarray)<jupyter_output>CPU times: user 65.6 ms, sys: 18.7 ms, total: 84.3 ms
Wall time: 98.2 ms<jupyter_text>Sample some dataTo test the I/O we will write and read some data from the database. This may take a while.<jupyter_code>write_times = []
read_times = []
counts = [500, 1000, 2000, 3000, 4000, 5000, 50000, 100000, 1000000, 10000000]

for c in counts:
    db.flushall()
    
    start_time_write = time.time()
    for key, emb in tqdm(embeddings(c), total=c):
        arr = adapt_array(emb)
        db.set(key, arr)
    write_times.append(time.time() - start_time_write)
    
    start_time_read = time.time()
    for key, _ in embeddings(c):
        obj = db.get(key)
        emb = convert_array(obj)
        assert(type(emb) is numpy.ndarray)
    read_times.append(time.time() - start_time_read)
    
print('DONE')
# save times for later plotting

with open('./collected_times/redis-write-times.pickle', 'wb') as f:
    pickle.dump(write_times, f)
    
with open('./collected_times/redis-read-times.pickle', 'wb') as f:
    pickle.dump(read_times, f)<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>plotly.offline.init_notebook_mode(connected=True)
trace = plotly.graph_objs.Scatter(
    y = write_times,
    x = counts,
    mode = 'lines+markers'
)
layout = plotly.graph_objs.Layout(title="Redis Write Times",
                yaxis=dict(title='Time in Seconds'),
                xaxis=dict(title='Embedding Count'))
data = [trace]
fig = plotly.graph_objs.Figure(data=data, layout=layout)
plotly.offline.iplot(fig, filename='jupyter-scatter-write')
plotly.offline.init_notebook_mode(connected=True)
trace = plotly.graph_objs.Scatter(
    y = read_times,
    x = counts,
    mode = 'lines+markers'
)
layout = plotly.graph_objs.Layout(title="Redis Read Times",
                yaxis=dict(title='Time in Seconds'),
                xaxis=dict(title='Embedding Count'))
data = [trace]
fig = plotly.graph_objs.Figure(data=data, layout=layout)
plotly.offline.iplot(fig, filename='jupyter-scatter-read')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11840342887757835, 'fraction_numerical': 0.02303777122957407, 'mean_word_length': 3.8118556701030926, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '414763', 'license': 'MIT', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1278, 'n_words': 317, 'path': 'Word_Embeddings_Redis.ipynb', 'repo_name': 'krokodilj/word_embedding_storage', 'size': '3733'}