<jupyter_start><jupyter_text>Sentence Similarity<jupyter_code>%load_ext autoreload
%autoreload 2
%reload_ext autoreload

%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from pathlib import Path

import scipy.spatial as ss
from sentence_transformers import SentenceTransformer

from importlib.util import find_spec
if find_spec("similarity_abstract_search") is None:
    import sys
    sys.path.append('..')
    
from medsearch.datasets.dataset import SemanticCorpusDataset
from medsearch.models.tfidf_model import TfidfModel  
from medsearch import utils
data = SemanticCorpusDataset(5000)
df = data.load_one_batch()
df
df.reset_index().to_json(str(data.data_dirname().parent/'paper_data.json'), orient='records')
# utils.load_json(str(data.data_dirname().parent/'paper_data.json'))<jupyter_output><empty_output><jupyter_text>Semantic Search<jupyter_code>embedder = SentenceTransformer('bert-base-nli-mean-tokens')
corpus = ['A man is eating food.',
          'A man is eating a piece of bread.',
          'The girl is carrying a baby.',
          'A man is riding a horse.',
          'A woman is playing violin.',
          'Two men pushed carts through the woods.',
          'A man is riding a white horse on an enclosed ground.',
          'A monkey is playing drums.',
          'A cheetah is running behind its prey.'
          ]
corpus_embeddings = embedder.encode(corpus)
queries = ['A man is eating pasta.', 
    'Someone in a gorilla costume is playing a set of drums.', 
    'A cheetah chases prey on across a field.']
query_embeddings = embedder.encode(queries)
closest_n = 5
for query, query_embedding in zip(queries, query_embeddings):
    distances = ss.distance.cdist([query_embedding], corpus_embeddings, "cosine")[0]

    results = zip(range(len(distances)), distances)
    results = sorted(results, key=lambda x: x[1])

    print("\n\n======================\n\n")
    print("Query:", query)
    print("\nTop 5 most similar sentences in corpus:")

    for idx, distance in results[0:closest_n]:
        print(corpus[idx].strip(), "(Score: %.4f)" % (1-distance))<jupyter_output>======================


Query: A man is eating pasta.

Top 5 most similar sentences in corpus:
A man is eating a piece of bread. (Score: 0.8480)
A man is eating food. (Score: 0.7759)
Two men pushed carts through the woods. (Score: 0.2095)
A monkey is playing drums. (Score: 0.1945)
A man is riding a white horse on an enclosed ground. (Score: 0.1586)


======================


Query: Someone in a gorilla costume is playing a set of drums.

Top 5 most similar sentences in corpus:
A monkey is playing drums. (Score: 0.7985)
A cheetah is running behind its prey. (Score: 0.2860)
The girl is carrying a baby. (Score: 0.2351)
A man is riding a horse. (Score: 0.2023)
A man is riding a white horse on an enclosed ground. (Score: 0.1963)


======================


Query: A cheetah chases prey on across a field.

Top 5 most similar sentences in corpus:
A cheetah is running behind its prey. (Score: 0.9007)
Two men pushed carts through the woods. (Score: 0.3662)
A monkey is playing drums. (Score: 0.[...]<jupyter_text>TD-IDF<jupyter_code>corpus = [f'{t} . {a} ' for t, a in zip(df.title, df.paperAbstract)]
model = TfidfModel()
%%time
X, V = model.fit(corpus)
X.shape
IX=model.svmSimilarity(X)<jupyter_output>100%|██████████| 64/64 [00:00<00:00, 528.33it/s]<jupyter_text>Search<jupyter_code>paper_dict  = utils.load_json(str(data.data_dirname().parent/'paper_data.json'))
search_dict = utils.load_json(str(data.data_dirname().parent/'search.json'))
sim_dict    = utils.load_json(str(data.data_dirname().parent/'sim_vecs.json'))

paper_dict[3]['title']

qparts = ('breast cancer screening').strip().split() 
n = len(paper_dict)

scores =[]
for i, sd in enumerate(search_dict):
    score = sum(sd.get(q, 0) for q in qparts)
    if score ==0: continue
    score += 1.0*(n-i)/n
    scores.append((score, paper_dict[i]))
scores.sort(reverse=True, key=lambda x: x[0])
papers = [x[1] for x in scores if x[0]>0]
if len(papers)>40:
    papers = papers[:40]
papers[0]
paper_dict[0]['id']
len(sim_dict)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14258235152680934, 'fraction_numerical': 0.026689107958643904, 'mean_word_length': 4.367741935483871, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '357695', 'license': 'MIT', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1381, 'n_words': 430, 'path': 'MedSemanticSearch/Notebooks/02_Sentence_Similarity.ipynb', 'repo_name': 'DanielhCarranza/info-retrieval-semantic-scholar', 'size': '4159'}