<jupyter_start><jupyter_text>Autocomplete  Preparation<jupyter_code># prepare the required modules
!pip install seaborn
"""Run these line to load the available datasets"""
import json

# load data test pure
with open('test_autocomplete_pure.json', 'r') as fp:
    testwords = json.load(fp)
    
# load data test with typo
with open('test_autocomplete_typo.json', 'r') as fp:
    test_1typo = json.load(fp)
# method to populate experiment result
def getAutoCompleteData(url, file, datatest, method, dataname):
    import requests
    from time import time
    start = time()
    results = {}
    
    for word in datatest:
        start_in = time()
        response = requests.get(url+word.strip())
        results[word] = [method, dataname, response.json(), len(response.json()), len(word), time()-start_in]
        
    print((time()-start)/len(datatest), 'seconds')
    with open(file, 'w') as fp:
        json.dump(results, fp)
    return results<jupyter_output><empty_output><jupyter_text>Analysis<jupyter_code># testing fast-autocomplete
url = 'http://130.216.216.55/autocomplete?query='
results_pure_fa = getAutoCompleteData(url, 'results_pure_fa.json', testwords, 'fast', '0 typo')
# testing SciGraph
url = 'http://130.216.216.55/autocomplete_sc?query='
results_pure_sc = getAutoCompleteData(url, 'results_pure_sc.json', testwords, 'SciGraph', '0 typo')
# testing fast-autocomplete
url = 'http://172.16.58.3/autocomplete?query='
results_1typo_fa = getAutoCompleteData(url, 'results_1typo_fa.json', test_1typo, 'fast', '1 typo')
# testing SciGraph
url = 'http://172.16.58.3/autocomplete_sc?query='
results_1typo_sc = getAutoCompleteData(url, 'results_1typo_sc.json', test_1typo, 'SciGraph', '1 typo')<jupyter_output>0.8379882657527924 seconds<jupyter_text>Autocomplete Analysis<jupyter_code># Pure datatest fast-autocomplete vs SciGraph

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_theme(style="darkgrid")


data = list(results_1typo_fa.values()) + list(results_pure_fa.values()) + list(results_pure_sc.values()) + list(results_1typo_sc.values())
df = pd.DataFrame(data, columns = ['method', 'dataname', 'returns', '# of return', '# of characters in a query', 'time (second)'])
df
df['method & datatest'] = df['method'] + ' + ' + df['dataname']
df
sns.set_style("whitegrid")
df['method + datatest'] = df['method'] + ' + ' + df['dataname']
sns.relplot(x="# of characters in a query", y="time (second)", hue="method + datatest", kind="line", data=df, height=5, aspect=6/5);
plt.savefig("execution_time.pdf", dpi=300)
plt.savefig("execution_time.png", dpi=300)
sns.set_style("whitegrid")
sns.relplot(x="# of characters in a query", y="# of return", hue="method + datatest", kind="line", data=df, height=10, aspect=12/10);
plt.savefig("return_number.pdf", dpi=300)
plt.savefig("return_number.png", dpi=300)<jupyter_output><empty_output><jupyter_text>Query performance  Preparation<jupyter_code># load datatest
import json

with open('query_datatest.json', 'r') as fp:
    datatests = json.load(fp)
    
with open('author_datatest.json', 'r') as fp:
    datatests_author = json.load(fp)
# get dataset id with provided query
# using fuzzy search, for fields of:
# "organisms.primary.species.originalName",
# "item.keywords.keyword",
# "item.name",
# "item.description",
# "item.readme.description",
# "anatomy.organ.name"

query = {
    "size": 50,
    "from": 0,
    "query": {
        "multi_match": {
          "fields":  [ "organisms.primary.species.originalName",
                       "item.keywords.keyword",
                       "item.name",
                       "item.description",
                       "item.readme.description",
                       "anatomy.organ.name",
                       "contributors.last.name",
                       "contributors.first.name",
                       "contributors.name",
                     ],
          "query": "query keywords",
          "fuzziness": "AUTO"
            }
          },
        "_source":{
            "includes":["pennsieve.identifier"],
        }
  }

url = 'https://scicrunch.org/api/1/elastic/SPARC_PortalDatasets_pr/_search?api_key=<KEY>'

def getESDataset(keyword):
    query['query']['multi_match']['query'] = keyword
    response = requests.post(url=url, json=query)
    dsList = []
    for hit in response.json()['hits']['hits']:
        try:
            dsList += [hit['_source']['pennsieve']['identifier']]
        except:
            pass
    return dsList
def getResults(datatest, approach):
    result = {}
    for term_num, d1 in datatest.items():
        result[term_num] = {}
        for typo, d2 in d1.items():
            result[term_num][typo] = {}
            print('\n getting the result of %s - %s using %s'%(term_num, typo, approach))
            for count, keyword in enumerate(d2):
                if keyword not in result[term_num][typo]:
                    if approach == 'AQUA':
                        result[term_num][typo][keyword] = getESDataset(getSpellCorrection(keyword))
                    elif approach == 'ES':
                        result[term_num][typo][keyword] = getESDataset(keyword)
                if count%20 == 0: print(count, end=' ')
    return result<jupyter_output><empty_output><jupyter_text>Get AQUA result<jupyter_code>from symspellpy import SymSpell, Verbosity

def __loadSpellChecker():
    """
    Loading spelling checker n-gram model to symspellpy
    """
    from symspellpy import SymSpell, Verbosity
    max_edit_distance_dictionary = 3
    prefix_length = 6
    sym_spell = SymSpell(max_edit_distance_dictionary, prefix_length)
    sym_spell.load_pickle('_spell_model_ds')
    return sym_spell



# get symspellpy pipeline
sym_spell = __loadSpellChecker()

def getSpellCorrection(query):
    return sym_spell.word_segmentation(query).corrected_string<jupyter_output><empty_output><jupyter_text>Keyword as query<jupyter_code># get results using AQUA where the key is keywords
aqua_result = getResults(datatests, 'AQUA')
# save to json
with open('search_aqua_result.json', 'w') as fp:
    json.dump(aqua_result, fp)
# open json
with open('search_aqua_result.json', 'r') as fp:
    aqua_result = json.load(fp)<jupyter_output><empty_output><jupyter_text>Author as query<jupyter_code># get results using AQUA where the key is author
aqua_result_author = getResults(datatests_author, 'AQUA')
# save to json
with open('search_aqua_result_author.json', 'w') as fp:
    json.dump(aqua_result_author, fp)
# open json
with open('search_aqua_result_author.json', 'r') as fp:
    aqua_result_author = json.load(fp)<jupyter_output><empty_output><jupyter_text>Get elasticsearch result  Keyword as query<jupyter_code># get results using ES where the key is keywords
es_result = getResults(datatests, 'ES')
# save to json
with open('search_es_result.json', 'w') as fp:
    json.dump(es_result, fp)
# open json
with open('search_es_result.json', 'r') as fp:
    es_result = json.load(fp)<jupyter_output><empty_output><jupyter_text>Author as query<jupyter_code># get results using ES where the key is author
es_result_author = getResults(datatests_author, 'ES')
# save to json
with open('search_es_result_author.json', 'w') as fp:
    json.dump(es_result_author, fp)
with open('search_es_result_author.json', 'r') as fp:
    es_result_author = json.load(fp)<jupyter_output><empty_output><jupyter_text>Performance analysis<jupyter_code># change result to binary
import copy
def result2binary(datatest, result):
    result = copy.deepcopy(result)
    for term_num, d1 in result.items():
        for typo, d2 in d1.items():
            for keyword in d2:
                for count, idx in enumerate(d2[keyword]):
                    if idx in datatest[term_num][typo][keyword]:
                        d2[keyword][count] = 1
                    else:
                        d2[keyword][count] = 0
    return result
r_aqua = result2binary(datatests, aqua_result)
r_a_aqua = result2binary(datatests_author, aqua_result_author)
r_es = result2binary(datatests, es_result)
r_a_es = result2binary(datatests_author, es_result_author)
import pandas as pd
def compareMAP(results, names, originalData):
    # create dataframe
    numOfTerm = ['1 term', '2 terms', '3 terms']
    cols = pd.MultiIndex.from_product([numOfTerm, ['AQUA', 'ES']])
    index = pd.Index(['0 typo', '1 del', '1 insert', '1 replace', 'no space', 'no space 1 typo', 'no space 2 typo', 'no space 3 typo', '3 typo'])
    df = pd.DataFrame(columns=cols, index=index)
    df.index.name = 'Typo'
    
    # calculate map

    for term_num, d1 in results[0].items():
        for typo in d1:
            tot_ap = 0
            for keyword in results[0][term_num][typo]:
                value = 0
                for i in range(len( results[0][term_num][typo][keyword])):
                    if  results[0][term_num][typo][keyword][i] == 1:
                        value += sum( results[0][term_num][typo][keyword][:i+1])/(i+1)
                ap = 0 if sum( results[0][term_num][typo][keyword])==0 else value / len(originalData[str(term_num)][typo][keyword])
                tot_ap += ap
            tot_ap2 = 0
            for keyword in results[1][term_num][typo]:
                value = 0
                for i in range(len(results[1][term_num][typo][keyword])):
                    if results[1][term_num][typo][keyword][i] == 1:
                        value += sum(results[1][term_num][typo][keyword][:i+1])/(i+1)
                ap = 0 if sum(results[1][term_num][typo][keyword])==0 else value / len(originalData[str(term_num)][typo][keyword])
                tot_ap2 += ap

#             print(term_num, ';', typo, ';', tot_ap/len(results[0][term_num][typo]), ';', tot_ap2/len(results[1][term_num][typo]))
            plural = 's' if int(term_num)>1 else ''
            df.loc[typo][term_num + ' term' + plural][names[0]] = tot_ap/len(results[0][term_num][typo])
            df.loc[typo][term_num + ' term' + plural][names[1]] = tot_ap2/len(results[1][term_num][typo])
    return df
## Comparison of AQUA VS ES to keyword queries
df = compareMAP([r_aqua, r_es], ['AQUA', 'ES'], datatests)
df
print(df.to_latex())
from tabulate import tabulate
print(tabulate(df, tablefmt="pipe", headers="keys"))
## Comparison of AQUA VS ES to author queries
df = compareMAP([r_a_aqua, r_a_es], ['AQUA', 'ES'], datatests_author)
df = df.dropna(axis=1, how='all')
df = df.dropna(how='all')
df
print(df.to_latex())
print(tabulate(df, tablefmt="pipe", headers="keys"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14046918035900846, 'fraction_numerical': 0.01576597967518283, 'mean_word_length': 3.529032258064516, 'pattern_counts': {'":': 9, '<': 41, '<?xml version=': 0, '>': 42, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '339906', 'license': 'MIT', 'n_tokens_mistral': 3672, 'n_tokens_neox': 3532, 'n_words': 771, 'path': 'experiment/Experiment.ipynb', 'repo_name': 'napakalas/aqua', 'size': '10562'}