<jupyter_start><jupyter_text>Histogram / Density plot of nr of trees per ensemble<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from arboreto.core import *
from arboretum.core import XGB_KWARGS
XGB_KWARGS
meta_df = pd.read_csv('/home/tmo/work/kuleuven/papers/arboreto/meta_out_macosko_40k.tsv', sep='\t')
meta_df.head()
df = macosko_df

all_zeros = df.loc[:, (df == 0).all()]
set(all_zeros.columns)
macosko_df = pd.read_csv('/media/tmo/data/work/datasets/macosko/in/macosko_40k.tsv.gz', sep='\t')<jupyter_output><empty_output><jupyter_text>* the targets with 5000 estimators are the all-zero columns! * these targets should have been filtered out before proceeding the analysis<jupyter_code>clean_df = meta_df[meta_df['n_estimators'] < 2000]
fig, ax = plt.subplots()
clean_df.n_estimators.plot.hist(bins=1000, ax=ax, figsize=(12,8))
ax.set_xlabel('Number of estimators')
plt.show()
fig, ax = plt.subplots(figsize=(12, 6))
sns.kdeplot(clean_df.n_estimators, bw=10, shade=True, ax=ax)
plt.savefig('macosko_n_estimators_kde.svg')
plt.show()
fig, ax = plt.subplots(figsize=(12, 4))
sns.distplot(clean_df.n_estimators, ax=ax)
plt.savefig('macosko_n_estimators_distplot.svg')
plt.show()
fig, ax = plt.subplots(figsize=(12,4))
sns.kdeplot(clean_df.n_estimators, shade=True, ax=ax)
plt.show()
clean_df.n_estimators.sum()
len(clean_df.n_estimators)
clean_df.n_estimators.sum() / (len(clean_df.n_estimators) * 1000)
clean_df
nonzero_df = pd.DataFrame(macosko_df.astype(bool).sum(axis=0)).reset_index()
nonzero_df.columns = ['target', 'n_nonzero']
nonzero_df.head()
fig, ax = plt.subplots()
clean_df.n_estimators.plot.hist(bins=1000, ax=ax, figsize=(12,8))
ax.set_yscale('log')
ax.set_xlabel('Number of estimators')
plt.show()
fig, ax = plt.subplots()
nonzero_df.merge(clean_df[['target', 'n_estimators']]).plot.scatter(ax=ax, x='n_nonzero', y='n_estimators', figsize=(12,8), alpha=.05)

ax.set_xscale('log')
#ax.set_yscale('log')
plt.show()
nonzero_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15896671634376552, 'fraction_numerical': 0.02384500745156483, 'mean_word_length': 7.426778242677824, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '214653', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 896, 'n_tokens_neox': 848, 'n_words': 135, 'path': 'notebooks/analysis/01_macosko_nr_trees_plot.ipynb', 'repo_name': 'gordian-biotechnology/arboreto', 'size': '2013'}