<jupyter_start><jupyter_text>Binarizer<jupyter_code>import scipy.stats
import numpy as np
X = scipy.stats.norm(0, 1).rvs((200000, 500))
y = np.mean(X, axis=1)
binarizer = Binarizer(32)
%%timeit n=5
binarizer.fit(X)
%%timeit n=5
df_bins = binarizer.transform(X)
%%timeit n=5
df_bins = binarizer.transform(X)
%%timeit n=5
DecisionTree(max_depth=6, binarizer=binarizer).fit(df_bins, MSELoss.point_stats(y), MSELoss(min_samples_leaf=1), binarize=False)
%%timeit n=5
DecisionTree(max_depth=6, binarizer=binarizer).fit(df_bins, MSELoss.point_stats(y), MSELoss(min_samples_leaf=1), binarize=False)
%%time
benchmark_tree()
%%time
benchmark_tree()
%%time
benchmark_tree(max_depth=1)
%load_ext line_profiler
%lprun -f DecisionTree.eval_stats benchmark_tree()
%lprun -f DecisionTree.fit benchmark_tree()<jupyter_output><empty_output><jupyter_text>Uplift<jupyter_code>from sgp.tests.test_uplift import make_dataset
X, tr, y = make_dataset(100000)
f_names = ['f1', 'f2', 'f3', 'f4', 'f5']
from sgp.trees.loss import MSEUpliftLoss

tree = DecisionTree(max_depth=3, n_bins=64, verbose=False)
tree_model = tree.fit(X, MSEUpliftLoss.point_stats(y, tr), MSEUpliftLoss())
print(tree_model.pretty_str(f_names))
from sgp.trees.loss import SigUpliftLoss
tree = DecisionTree(max_depth=3, n_bins=64, verbose=False)
tree_model = tree.fit(X, SigUpliftLoss.point_stats(y, tr), SigUpliftLoss())
print(tree_model.pretty_str(f_names))
#tree = DecisionTree(max_depth=6, n_bins=64, verbose=False)
forest = RandomForest(max_depth=6, n_trees=100, n_bins=128, n_jobs=5, verbose=False)
forest_model = forest.fit(X, MSEUpliftLoss.point_stats(y, tr), MSEUpliftLoss())
y_pred = forest_model.predict(X)
plt.plot(X[:, 0], y_pred, 'o')
from sgp.trees.binarizer import binarization_plot
bins_values, uplift_values, stds_values = binarization_plot(X[:, 0], tr, y, n_bins=64)

plt.plot(bins_values, uplift_values)
plt.fill_between(bins_values, uplift_values - 3 * stds_values, uplift_values + 3 * stds_values, alpha=0.1)
plt.show()
from sgp.trees.loss import uplift_at_k

random_y_pred = y_pred.copy()
np.random.shuffle(random_y_pred)
plt.plot(uplift_at_k(random_y_pred, y, tr)[::1000])

plt.plot(uplift_at_k(y_pred, y, tr)[::1000])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14343707713125844, 'fraction_numerical': 0.03067207938655841, 'mean_word_length': 8.857777777777779, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '459449', 'license': 'MIT', 'n_tokens_mistral': 1008, 'n_tokens_neox': 926, 'n_words': 154, 'path': 'notebooks/TestTrees.ipynb', 'repo_name': 'marnikitta/sgp', 'size': '2217'}