<jupyter_start><jupyter_text>preqs`start bert-en`<jupyter_code>import pandas as pd
dfjson = pd.read_json(f'/pi/stack/crawlers/langcrs/all_zh.json')
dfjson.head()
# quotes['text'].to_list()
dfjson_ja = pd.read_json(f'/pi/stack/crawlers/langcrs/all_ja.json')
dfjson_id = pd.read_json(f'/pi/stack/crawlers/langcrs/all_id.json')
set_ja=set(dfjson_ja['text'].to_list())
set_id=set(dfjson_id['text'].to_list())
print(len(set_ja), len(set_id), set_ja-set_id)
new_df = pd.merge(dfjson_ja, dfjson_id,  how='left', left_on=['text','chapter'], right_on = ['text','chapter'])
new_df.head()
import json
def search_in(text, lang):
    with open(f'/pi/stack/crawlers/langcrs/all_{lang}.json') as json_file:
        sents=json.load(json_file)
        return [sent for sent in sents if sent['text']==text]
def search_in_list(text, langs):
    rs={}
    for lang in langs:
        rs[lang]=search_in(text, lang)
    return rs

search_in('both of us', 'ja')
search_in_list('I write a letter.', ['ja', 'fa', 'id'])
import pandas as pd
from bert_serving.client import BertClient

# Bert Client must be running locally
bc = BertClient()
def train(quotes):
    embeddings = bc.encode(quotes.text.to_list())
    quotes['EMBEDDINGS'] = embeddings.tolist()

    # Persist to pickle
    quotes.to_pickle('data/embedded_corpus.pkl')
train(dfjson)
def load_quotes_and_embeddings(file):
    quotes = pd.read_pickle(file)

    # change dtype in place for memory efficiency
    quotes['EMBEDDINGS'] = quotes['EMBEDDINGS'].apply(
        lambda arr: np.array(arr, dtype='float32')
    )

    quote_embeddings = np.stack(quotes.EMBEDDINGS.values)

    # reduce memory footprint by dropping column
    quotes.drop('EMBEDDINGS', axis='columns')

    # normalize embeddings for cosine distance
    embedding_sums = quote_embeddings.sum(axis=1)
    normed_embeddings = quote_embeddings / embedding_sums[:, np.newaxis]
    return quotes, normed_embeddings

quotes, embeddings = load_quotes_and_embeddings('data/embedded_corpus.pkl')
def create_index(embeddings):
    import faiss
    """
    Create an index over the quote embeddings for fast similarity search.
    """
    dim = embeddings.shape[1]
    index = faiss.IndexFlatL2(dim)
    index.add(embeddings)
    return index

index = create_index(embeddings)
def search(text):
    text_embedding = bc.encode([text])
    normalized_text_embedding = text_embedding / text_embedding.sum()

    _, idx = index.search(normalized_text_embedding, 5)

    relevant_quotes = quotes.iloc[idx.flatten()]['text'].values
    relevant_chapters = quotes.iloc[idx.flatten()]['chapter'].values
    return relevant_quotes, relevant_chapters
# bc = BertClient()
text="I dreamed a dream."

relevant_quotes, relevant_chapters=search(text)
for q in range(5):
    print('>'+relevant_quotes[q])
    print(relevant_chapters[q])
text="I write a letter."

relevant_quotes, relevant_chapters=search(text)
for q in range(5):
    print('>'+relevant_quotes[q])
    print(relevant_chapters[q])
x,y=[1,2]
print(y)<jupyter_output>2
{'fraction_non_alphanumeric': 0.13583527067419462, 'fraction_numerical': 0.0036532713384257723, 'mean_word_length': 4.905882352941177, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '499340', 'license': 'Apache-2.0', 'n_tokens_mistral': 1097, 'n_tokens_neox': 1067, 'n_words': 207, 'path': 'spacy-2.2/procs-faiss-corpus.ipynb', 'repo_name': 'samlet/stack', 'size': '3011'}