<jupyter_start><jupyter_text>Load model<jupyter_code># make sure models.py is in the working directory
model = torch.load('infersent.allnli.pickle')
# torch.load(..pickle) will use GPU/Cuda by default. If you are on CPU:
# model = torch.load('infersent.allnli.pickle', map_location=lambda storage, loc: storage)
# On CPU, setting the right number of threads with "torch.set_num_threads(k)" may improve performance
model.set_glove_path(GLOVE_PATH)
model.build_vocab_k_words(K=100000)<jupyter_output>Vocab size : 100000<jupyter_text>Load sentences<jupyter_code># Load some sentences
sentences = []
with open('samples.txt') as f:
    for line in f:
        sentences.append(line.strip())
print(len(sentences))
sentences[:5]<jupyter_output><empty_output><jupyter_text>Encode sentences<jupyter_code># gpu mode : ~1000 sentences/s
# cpu mode : ~75 sentences/s
embeddings = model.encode(sentences, bsize=128, tokenize=False, verbose=True)
print('nb sentences encoded : {0}'.format(len(embeddings)))<jupyter_output>Nb words kept : 128201/130068 (98.56 %)
Speed : 1170.65 sentences/s (gpu mode, bsize=128)
nb sentences encoded : 9815<jupyter_text>Visualization<jupyter_code>np.linalg.norm(model.encode(['the cat eats.']))
def cosine(u, v):
    return np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))
cosine(model.encode(['the cat eats.'])[0], model.encode(['the cat drinks.'])[0])
idx = randint(0, len(sentences))
_, _ = model.visualize(sentences[idx])
my_sent = 'The cat is drinking milk.'
_, _ = model.visualize(my_sent)
model.build_vocab_k_words(500000) # getting 500K words vocab
my_sent = 'barack-obama is the former president of the United-States.'
_, _ = model.visualize(my_sent)<jupyter_output>Vocab size : 500000
{'fraction_non_alphanumeric': 0.13736903376018628, 'fraction_numerical': 0.04074505238649592, 'mean_word_length': 6.314893617021276, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '130597', 'license': 'MIT', 'n_tokens_mistral': 678, 'n_tokens_neox': 621, 'n_words': 138, 'path': 'data_analysis/pythonCode/InferSent/encoder/demo.ipynb', 'repo_name': 'norberte/Statistical-Consulting', 'size': '1718'}