get_ipython().run_line_magic('load_ext', 'autoreload')
get_ipython().run_line_magic('autoreload', '2')

import sys
import pkg_resources 

try: # When on Colab, let's install pyserini, Pytorch, and Faiss
    import google.colab
    repo_path = 'dspy'
    get_ipython().system('git -C $repo_path pull origin || git clone https://github.com/stanfordnlp/dspy $repo_path')
    get_ipython().run_line_magic('cd', '$repo_path')
    get_ipython().system('pip install -e .')
    if not "pyserini" in {pkg.key for pkg in pkg_resources.working_set}:
        get_ipython().system('pip install pyserini')
    if not "torch" in {pkg.key for pkg in pkg_resources.working_set}:
        get_ipython().system('pip install torch')
    if not "faiss-cpu" in {pkg.key for pkg in pkg_resources.working_set}:
        get_ipython().system('pip install faiss-cpu')
except:
    repo_path = '.'
    if not "dspy-ai" in {pkg.key for pkg in pkg_resources.working_set}:
        get_ipython().system('pip install -U pip')
        get_ipython().system('pip install dspy-ai')

if repo_path not in sys.path:
    sys.path.append(repo_path)

import dspy



pys_ret_prebuilt = dspy.Pyserini(index='beir-v1.0.0-nfcorpus.contriever-msmarco', query_encoder='facebook/contriever-msmarco', id_field='_id', text_fields=['title', 'text'])

dspy.settings.configure(rm=pys_ret_prebuilt)

example_question = "How Curry Can Kill Cancer Cells"

retrieve = dspy.Retrieve(k=3)
topK_passages = retrieve(example_question).passages

print(f"Top {retrieve.k} passages for question: {example_question} \n", '-' * 30, '\n')

for idx, passage in enumerate(topK_passages):
    print(f'{idx+1}]', passage, '\n')



get_ipython().system('wget https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nfcorpus.zip -P collections')
get_ipython().system('unzip collections/nfcorpus.zip -d collections')



get_ipython().system('python -m pyserini.encode    input   --corpus collections/nfcorpus/corpus.jsonl            --fields title text    output  --embeddings indexes/faiss.nfcorpus.contriever-msmarco            --to-faiss    encoder --encoder facebook/contriever-msmarco            --device cuda:0            --pooling mean            --fields title text')



from datasets import load_dataset

dataset = load_dataset(path='json', data_files='collections/nfcorpus/corpus.jsonl', split='train')

pys_ret_local = 
dspy.Pyserini(index='indexes/faiss.nfcorpus.contriever-msmarco', query_encoder='facebook/contriever-msmarco', dataset=dataset, id_field='_id', text_fields=['title', 'text'])