<jupyter_start><jupyter_text>Glove or Spacy encodingsTakes text inputs of varying lengths and outputs glove encodings of the same length1. Tokenize with Spacy2. Get word embeddings (start with Spacy's, then use Glove if needed)3. Let torch handle uneven text lengths with `pad_sequence`?<jupyter_code>#!/usr/local/anaconda3/envs/spike_basicoV5/bin/python -m spacy download en_core_web_sm
#!/usr/local/anaconda3/envs/spike_basicoV5/bin/python -m spacy download en_core_web_lg
%load_ext autoreload
%autoreload 2
import src
import spacy
import numpy as np
nlp = spacy.load('en_core_web_sm', )

doc = nlp("Apple is looking at buying U.K. startup for $1 billion")
for token in doc:
    print(token.text)
nlp = spacy.load("en_core_web_lg")
tokens = nlp("dog cat banana afskfsd")
for token in tokens:
    print(token.text, token.has_vector, token.vector_norm, token.is_oov)
example_list = ["hiya, what's up?",
               "another description of stuff",
               "and a final one! Yay!"]<jupyter_output><empty_output><jupyter_text>+ index_item_matrix (items x largo_maximo_glosas)+ embedding_matrix  (npalabras + 1 x nlatentes)<jupyter_code>encodings_of_all_sentences = src.encode_sequence(example_list, nlp, n_of_dims=300)
encodings_of_all_sentences.shape
index_item_matrix, embedding_matrix, unique_tokens_d = src.get_embedding_and_index_item_matrix(example_list,
                                                                                               nlp, n_of_embedding_dims=300)<jupyter_output><empty_output><jupyter_text>Pre-process Movie Data<jupyter_code>project_id = "spike-sandbox"

query = """
SELECT movie_id, movie_name, category_0
FROM EVIC.movies
"""

import pandas as pd

movies = pd.read_gbq(query, project_id=project_id)
movies['full_string'] = movies['movie_name'] + ' ' + movies.category_0
movies.sort_values('movie_id', inplace=True)
index_item_matrix, embedding_matrix, unique_tokens_d  = src.get_embedding_and_index_item_matrix(
                                                        movies.full_string.values, nlp, n_of_embedding_dims=300)

movies.full_string.values
export_d = {'index_item_matrix': index_item_matrix,
           'embedding_matrix': embedding_matrix,
           'unique_tokens_d': unique_tokens_d}

import pickle
with open('data/index_item_and_embedding_matrix_dict.pickle', 'wb') as handle:
    pickle.dump(export_d, handle, protocol=pickle.HIGHEST_PROTOCOL)
for text, ind in unique_tokens_d.items():
    print(text, ind)
unique_tokens_d<jupyter_output><empty_output><jupyter_text>Do Padding with pytorch instead?<jupyter_code>from torch.nn.utils.rnn import pad_sequence
import torch
a = torch.ones(25, 300)
b = torch.ones(22, 300)
c = torch.ones(15, 300)
pad_sequence([a, b, c]).size()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0968208092485549, 'fraction_numerical': 0.01300578034682081, 'mean_word_length': 4.472332015810276, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '283980', 'license': 'MIT', 'n_tokens_mistral': 983, 'n_tokens_neox': 948, 'n_words': 201, 'path': 'produce_encodings.ipynb', 'repo_name': 'SpikeLab-CL/encodings_for_cold_start', 'size': '2768'}