<jupyter_start><jupyter_text>Use the neural network index: ```trained_model``` to search for an index<jupyter_code>def binarySearch(alist, item, predicted_index, max_abs_error):
    first = max(predicted_index - max_abs_error - 1,0)
    last = predicted_index + max_abs_error
    found = False
    midpoint = 0

    while first<=last and not found:
        midpoint = (first + last)//2
        if alist.iloc[midpoint,0] == item[0]:
            found = True
        else:
            if item[0] < alist.iloc[midpoint,0]:
                last = midpoint-1
            else:
                first = midpoint+1

    return found, midpoint
data=pd.concat([train_data,pd.Series(train_label[0])],axis=1)
data['predicted_index']=-1
data['found']=False
data['time_to_find']=0.0
data['found_at']=-1


import time
from tqdm import tqdm
max_abs_error = int(max(abs(error_in_index_moe)))
i=15

for i in tqdm(data.index):
    sample_timestamp_ps = train_data.iloc[i]
    strt = time.time()
    predicted_index = trained_model.predict(sample_timestamp_ps)[0]
    # print("predicted_index %s for timestamp (key) %s"%(predicted_index,sample_timestamp))
    # print(max_abs_error)
    found, index = binarySearch(data,sample_timestamp_ps,int(predicted_index),max_abs_error)
    end = time.time()
    data['predicted_index'].iloc[i]=predicted_index
    data['found'].iloc[i]=found
    data['time_to_find'].iloc[i]=end-strt
    data['found_at'].iloc[i]=index

#print("Found at %s"%index) if found else print("not found")
#int(max(abs(error_in_index_moe)))
data[:300].describe()
pd.DataFrame(np.asarray(train_data)).describe()
from keras.models import load_model

trained_model.save('my_moe_web.h5')  # creates a HDF5 file 'my_model.h5'
trained_model.save_weights('my_moe_weights_web.h5')
# del model  # deletes the existing model

# returns a compiled model
# identical to the previous one
#model = load_model('my_model.h5')
!ls -ltrh<jupyter_output>total 2032
-rw-r--r--@ 1 Sonai  staff   1.0K 22 Jan 07:07 LICENSE
-rw-r--r--@ 1 Sonai  staff   2.1K 22 Jan 07:07 README.md
drwxr-xr-x@ 4 Sonai  staff   128B 22 Jan 07:07 [34mdata[m[m
-rw-r--r--@ 1 Sonai  staff    81B 22 Jan 07:07 requirements.txt
-rw-r--r--@ 1 Sonai  staff    11K 24 Jan 07:49 mmoe.py
-rw-r--r--@ 1 Sonai  staff   8.6K 26 Jan 06:52 census_income_demo.py
-rw-r--r--@ 1 Sonai  staff   4.0K 26 Jan 07:28 synthetic_demo.py
-rw-r--r--@ 1 Sonai  staff   5.0K 26 Jan 07:32 weblog.py
-rw-r--r--@ 1 Sonai  staff   872B 27 Jan 07:14 kerasplt.py
-rw-r--r--@ 1 Sonai  staff   5.2K 28 Jan 19:07 web.py
drwxr-xr-x@ 5 Sonai  staff   160B 28 Jan 19:08 [34m__pycache__[m[m
-rw-r--r--@ 1 Sonai  staff    51K 29 Jan 05:45 my_moe_web.h5
-rw-r--r--@ 1 Sonai  staff    25K 29 Jan 05:45 my_moe_weights_web.h5
-rw-r--r--@ 1 Sonai  staff   388K 29 Jan 06:29 my_btree_web.pickle
-rw-r--r--@ 1 Sonai  staff   145K 29 Jan 06:29 Untitled.ipynb
-rw-r--r--@ 1 Sonai  staff   332K 29 Jan 06:30 my_dict_web.pic[...]<jupyter_text>Size of the persisted model weights is 25K. Whereas size of the persisted btree index is 388K. Moreover the size of the model weights will not grow with increase in number ofdata points.<jupyter_code>!ls -lthr ../sdsc-http.txt
n_df = pd.DataFrame(index=train_data['pk'].values, data=train_data['pk'].index)
n_df.to_dict()[0]

from BTrees.OOBTree import OOBTree
t = OOBTree()
t.update(n_df.to_dict()[0])
import pickle,sys
sys.setrecursionlimit(30000)
with open('my_btree_web.pickle', 'wb+') as f:
    pickle.dump(t,f)
import pickle,sys
sys.setrecursionlimit(30000)
with open('my_dict_web.pickle', 'wb+') as f:
    pickle.dump(train_data.to_dict()['pk'],f)
import BTrees.check
BTrees.check.display(t)
type(t)
import time
from tqdm import tqdm
n_df['found']=False
n_df['time_to_find']=0.0
n_df['found_at']=-1
max_abs_error = int(max(abs(error_in_index_moe)))
i=15

for i in tqdm(n_df.index):
    strt=time.time()
    found_at=t.get(i)
    end=time.time()
    n_df['found'].loc[i]=True# if found_at else False
    n_df['found_at'].loc[i]=found_at
    n_df['time_to_find'].loc[i]=end-strt
n_df.iloc[:300].describe()
train_data['pk'].index
n_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14330823415108088, 'fraction_numerical': 0.051493806169540926, 'mean_word_length': 3.884934756820878, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '431009', 'license': 'MIT', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1639, 'n_words': 373, 'path': 'MOE_vs_BTREE.ipynb', 'repo_name': 'ArnabRaxit/keras-mmoe', 'size': '4117'}