<jupyter_start><jupyter_text>© 2020 NokiaLicensed under the BSD 3 Clause licenseSPDX-License-Identifier: BSD-3-Clause  Setup<jupyter_code>%load_ext autoreload
%autoreload 2

import os
import json
import time
import numpy as np
import sys

from codesearch.encoders import BasicEncoder
from codesearch import embedding_pretraining
from codesearch.embedding_pretraining import train_fasttext_model_from_snippets, load_fasttext_model
from codesearch.utils import SaveableFunction
from codesearch.data import load_snippet_collection, EVAL_DATASETS, SNIPPET_COLLECTIONS, eval_datasets_from_regex
from codesearch.ncs.ncs_embedder import TfidfCodeEmbedder, NcsEmbedder
from codesearch.evaluation import evaluate_and_dump 
from codesearch.embedding_retrieval import EmbeddingRetrievalModel
start = time.time()<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Read configuration parameters from environment variables (when this notebook is run as a script).<jupyter_code>fast_text_checkpoint = os.environ.get("fast_text_checkpoint", None)
model_filename = os.environ.get("model_filename", None)

snippets_collection = os.environ.get("snippets_collection", "so-ds-feb20")
train_snippets_collection = os.environ.get("train_snippets_collection", "so-ds-feb20")
valid_dataset = os.environ.get("valid_dataset", None)
test_dataset = os.environ.get("test_dataset", "conala-curated-0.5-test")

text_overrides = json.loads(os.environ.get("text_overrides", "{}"))
code_overrides = json.loads(os.environ.get("code_overrides", "{}"))
fast_text_overrides = json.loads(os.environ.get("fast_text_overrides", "{}"))
zip_fn_name = os.environ.get("zip_fn", "zip_descr_end")
output_dir = os.environ.get("output_dir", ".")
model_filename, fast_text_checkpoint
snippets_collection
text_overrides, code_overrides, fast_text_overrides, zip_fn_name<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>if valid_dataset and valid_dataset not in EVAL_DATASETS and valid_dataset not in SNIPPET_COLLECTIONS:
    raise ValueError()
test_datasets = eval_datasets_from_regex(test_dataset)
snippets = load_snippet_collection(snippets_collection)
train_snippets = load_snippet_collection(train_snippets_collection) 

train_snippets[-1]
f=open('../../processed_search_1.json')
train_snippets=json.load(f)
snippets=train_snippets
snippets[0]
train_snippets<jupyter_output><empty_output><jupyter_text>Train or load embedding model<jupyter_code>enc = BasicEncoder(text_preprocessing_params=text_overrides, code_preprocessing_params=code_overrides)
zip_fn = getattr(sys.modules[embedding_pretraining.__name__], zip_fn_name)
model = train_fasttext_model_from_snippets(train_snippets, enc, zip_fn, fast_text_overrides, "./", save=True)
if fast_text_checkpoint:
    model, enc = load_fasttext_model(fast_text_checkpoint)
    print("Loaded fast text checkpoint")
    
else:
    enc = BasicEncoder(text_preprocessing_params=text_overrides, code_preprocessing_params=code_overrides)
    zip_fn = getattr(sys.modules[embedding_pretraining.__name__], zip_fn_name)
    model = train_fasttext_model_from_snippets(train_snippets, enc, zip_fn, fast_text_overrides, "./", save=False)<jupyter_output><empty_output><jupyter_text>Unsupervised retrieval baseline A first baseline that computes a representation a snippet representation as a tfidf weighted average of their embeddings and a query representation by averaging all terms.  Embedding code & queries<jupyter_code>tfidf_model = TfidfCodeEmbedder.create_tfidf_model(enc, model, snippets)
embedder = NcsEmbedder(model, enc, tfidf_model)<jupyter_output>100%|██████████| 12137/12137 [00:16<00:00, 754.58it/s]<jupyter_text>Create retrieval model<jupyter_code>retrieval_model = EmbeddingRetrievalModel(embedder)
retrieval_model.add_snippets(snippets)
if model_filename: embedder.save(model_filename)
embedder.save('best_ncs_embedder')<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>sample_queries = ["train a tensorflow model", "plot a bar chart", "merge two dataframes", "sort a list", "read a pandas dataframe from a file", "plot an image"]
config = {"text": text_overrides, "code": code_overrides, "fasttext": fast_text_overrides}
evaluate_and_dump(
    retrieval_model, 
    config, 
    output_dir, 
    valid_dataset, 
    test_datasets,
    sample_queries=sample_queries
)
duration = time.time() - start
f"Running the notebook took {duration} seconds"
give_your_query=['sort a list']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09359059658460855, 'fraction_numerical': 0.009314703925482368, 'mean_word_length': 8.657387580299785, 'pattern_counts': {'":': 3, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '29793', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1563, 'n_tokens_neox': 1505, 'n_words': 286, 'path': 'ncs/ncs.ipynb', 'repo_name': 'pritam004/Neural-Code-Search', 'size': '4509'}