<jupyter_start><jupyter_text>TPCH<jupyter_code>tpch_ds = QueriesDataset('tpch:uniform2')
tpch_df = tpch_ds.load()
tpch_schema = tpch_ds.schema()
tpch_df = tpch_df.query('runtime < 500')
tpch_df['bucket'] = tpch_df['runtime'].apply(np.log2).apply(np.round).apply(lambda x: max(0, x)).astype(int)
tpch_df.groupby('bucket').count().mean()
# tpch_df = tpch_df.groupby('bucket').head(1500)
from collections import Counter

tpch_estimaor = QueryEstimater(tpch_schema, checkpoint_path)
tpch_queries, tpch_labels = list(tpch_df['query']), list(tpch_df['runtime'].apply(lambda x: max(x, 1)))
label_counts = Counter(np.round(np.log2(tpch_labels)))
print(label_counts, len(label_counts))
tpch_predictions = tpch_estimaor.predict_batch(tpch_queries, lowest=1)
# Rows - prediction, Columns - actual
plot_log2_confusion(tpch_predictions, tpch_labels)
plt.plot(tpch_predictions, 'crimson', alpha=0.7)
plt.plot(tpch_labels,  'blue', alpha=0.4)
plt.legend(['predicted', 'real'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16145307769929365, 'fraction_numerical': 0.018163471241170535, 'mean_word_length': 9.553191489361701, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '416867', 'license': 'MIT', 'n_tokens_mistral': 409, 'n_tokens_neox': 404, 'n_words': 52, 'path': 'dqo/estimator/greq/v2/evaluate.ipynb', 'repo_name': 'danield137/deep_query_optimzation', 'size': '991'}