<jupyter_start><jupyter_text>Centralized Preproc<jupyter_code>query = ({}, {'lines':1, 'time':1})
backend.plot_query("imr_preprocess", query, title="Centralized Preprocess",
                   ylabel='time (sec)', xlabel='#entries',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Centralized W2V Train<jupyter_code>backend.plot_query("imr_w2v_train", query, '-x', title="W2V Centralized Train",
                   ylabel='time (sec)', xlabel='#entries',label="exec. time")

### alternative implementation ###
# res = backend.query("imr_w2v_train", query)
# res = zip(*res)[1:]

# import matplotlib.pyplot as plt
# plt.plot(res[0],res[1], alpha=0.3)
# plt.scatter(res[0],res[1], marker='D',s=50, color='red')


plt.show()<jupyter_output><empty_output><jupyter_text>Centralized W2V Vectorize<jupyter_code>backend.plot_query("imr_w2v_vectorize", query, title="W2V Centralized Vectorization",
                   ylabel='time (sec)', xlabel='# documents',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Centralized Classifier Train<jupyter_code>backend.plot_query("imr_train", query, title="Train Centralized Classifier",
                   ylabel='time (sec)', xlabel='# documents',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Centralized Classification<jupyter_code>backend.plot_query("imr_predict", query, title="Predict with Centralized Classifier",
                   ylabel='time (sec)', xlabel='# documents',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Spark W2V Train<jupyter_code>query = ({}, {'lines':1, 'time':1})
backend.plot_query("imr_w2v_train_spark", query, title="Spark W2V Train",
                   ylabel='time (sec)', xlabel='#entries',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Spark W2V Vectorize<jupyter_code>backend.plot_query("imr_w2v_vectorize_spark", query, title="Spark W2V Vectorization",
                   ylabel='time (sec)', xlabel='# documents',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Spark Classifier Train<jupyter_code>backend.plot_query("imr_lr_train_spark", query, title="Train Spark Classifier",
                   ylabel='time (sec)', xlabel='# documents',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Spark Classification<jupyter_code>backend.plot_query("imr_classify_spark", query, title="Predict with Spark Classifier",
                   ylabel='time (sec)', xlabel='# documents',
                   label="exec. time")<jupyter_output><empty_output><jupyter_text>Compararative  Classifier train<jupyter_code>backend.plot_query("imr_train", query, title="Classifier Training",
                   ylabel='time (sec)', xlabel='# documents',
                   label="Centralized", show_plot=False)
backend.plot_query("imr_lr_train_spark", query,
                   ylabel='time (sec)', xlabel='# documents',
                   label="Spark")<jupyter_output><empty_output><jupyter_text>Classify<jupyter_code>backend.plot_query("imr_predict", query, title="Predict using Model",
                   ylabel='time (sec)', xlabel='# documents',
                   label="Centralized", show_plot=False)
backend.plot_query("imr_classify_spark", query,
                   ylabel='time (sec)', xlabel='# documents',
                   label="Spark")<jupyter_output><empty_output><jupyter_text>W2V train<jupyter_code>query = ({}, {'lines':1, 'time':1})
backend.plot_query("imr_w2v_train", query, '-x', title="W2V Train", show_plot=False,
                   ylabel='time (sec)', xlabel='#entries',label="Centralized")
backend.plot_query("imr_w2v_train_spark", query,
                   ylabel='time (sec)', xlabel='#entries',
                   label="Spark")<jupyter_output><empty_output><jupyter_text>W2V Vectorize<jupyter_code>backend.plot_query("imr_w2v_vectorize", query, title="W2V Vectorization",
                   ylabel='Centralized', xlabel='# documents', show_plot=False,
                   label="Centralized")


backend.plot_query("imr_w2v_vectorize_spark", query, 
                   ylabel='time (sec)', xlabel='# documents',
                   label="Spark")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14920856610800745, 'fraction_numerical': 0.008379888268156424, 'mean_word_length': 3.9907084785133566, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '663264', 'license': 'Apache-2.0', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1372, 'n_words': 196, 'path': 'asap-tools/monitoring/IMR_Workflow.ipynb', 'repo_name': 'project-asap/Profiler', 'size': '4296'}