<jupyter_start><jupyter_text>Boundary Issues<jupyter_code>import os, sys
cwd = os.getcwd()
# sys.path.insert(0, f"{cwd}/../")
sys.path.insert(0, "/home/emmanuel/code/rbig")

# RBIG Packages
from rbig.data import ToyData
from rbig.information.kde import KDESklearn

from typing import Iterable, Optional, Dict, NamedTuple, Tuple, Union
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin

import numpy as np
from scipy import stats

# Plot Functions
import seaborn as sns
import matplotlib.pyplot as plt
plt.style.use(["seaborn-paper"])

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>def plot_dist(X_samples: np.ndarray, bins=100, *args, **kwargs):
    fig, ax = plt.subplots()
    sns.distplot(
        X_samples, 
        bins=bins, 
        ax=ax, 
        kde_kws={
            'linewidth': 7, 
            'color': 'black',
        },
        hist_kws={
            'color': 'green'
        }
    )
    return fig, ax
seed = 123
n_samples = 10_000
a = 4

# initialize data distribution
data_dist = stats.gamma(a=a)

# get some samples
X = data_dist.rvs(size=(n_samples, 1), random_state=seed)

# plot distribution
plot_dist(X, bins=100);<jupyter_output><empty_output><jupyter_text>CDF EstimationFor the CDF estimation, I will use the quantile method.<jupyter_code># number of quantiles
n_quantiles = 1_000
n_quantiles = max(1, min(n_quantiles, n_samples))

# calculate the reference values (support), [0, 1]
references = np.linspace(0, 1, n_quantiles, endpoint=True)

# calculate quantiles
quantiles = np.percentile(X, references * 100)<jupyter_output>0.16885335848364358 18.275140139054425<jupyter_text>Plot CDF Distribution<jupyter_code># plot distribution
plot_dist(quantiles, nbins=100);<jupyter_output><empty_output><jupyter_text>It looks really close to the original distribution. I think we can be happy with this.  Plot - CDF Function<jupyter_code># CDF plot
fig, ax = plt.subplots()
ax.plot(quantiles, references, linestyle='--', color='black')
ax.set_xlabel(r'Support Points')
ax.set_ylabel(r'Empirical CDF')
plt.show()<jupyter_output><empty_output><jupyter_text>This also looks OK. But what happens if we find some points that are outside of our distribution? In other words, what happens if I have points that lay outside of my support?<jupyter_code>print('Data Boundaries:', X.min(), X.max())
print('Quantile Boundaries:', quantiles.min(), quantiles.max())<jupyter_output>Data Boundaries: 0.16885335848364358 18.275140139054425
Quantile Boundaries: 0.16885335848364358 18.275140139054425<jupyter_text>So we can extend the boundary (the support) to ensure that we can accommadate points that lie outside of the region.  Method I - Extend the Quantiles Boundaries<jupyter_code>support_extension = 10

domain = np.abs(quantiles.max() - quantiles.min())

domain_ext = (10 / 100) * domain

lower_bound = quantiles.min() - domain_ext
upper_bound = quantiles.max() + domain_ext

print('New Boundaries:', lower_bound, upper_bound)
new_quantiles = np.hstack([lower_bound, quantiles, upper_bound])

# testing
assert lower_bound == new_quantiles.min()
assert upper_bound == new_quantiles.max()
assert np.ndim(new_quantiles) == 1<jupyter_output><empty_output><jupyter_text>Interpolation We will now interpolate to ensure that the support gets extended.<jupyter_code>new_references = np.interp(new_quantiles, quantiles, references)
print(new_references.min(), new_references.max())
# CDF plot
fig, ax = plt.subplots()
ax.plot(new_quantiles, new_references, linestyle='--', color='red', label='Extended')
ax.plot(quantiles, references, linestyle='--', color='black', label='Old')
ax.legend(fontsize=12)
ax.set_xlabel(r'Support Points')
ax.set_ylabel(r'Empirical CDF')
plt.show()<jupyter_output><empty_output><jupyter_text>PDF Estimation<jupyter_code>quantiles.min(), quantiles.max()
plt.hist(hist[0], bins=100);
hpdf, hbins = np.histogram(X.squeeze(), bins=100, density=True)
plt.plot(hbins[:-1], hpdf)

plt.plot(quantiles)
hist = np.histogram(X.squeeze(), bins=100, density=True)

# plt.hist(hist[0], bins=100);

hist_clf = stats.rv_histogram(hist)

plot_dist(hist_clf.pdf(X));

# plot distribution
plot_dist(data_dist.pdf(X), nbins=100);
hist, bin_edges = np.histogram(X.squeeze(), bins='auto')

hist_clf = stats.rv_histogram((hist, bin_edges))
# plot distribution
plot_dist(hist_clf.pdf(X), nbins=100);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12533998186763373, 'fraction_numerical': 0.038757932910244784, 'mean_word_length': 5.863141524105754, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '110197', 'license': 'MIT', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1569, 'n_words': 359, 'path': 'old_notebooks/demo_7.2_boundary_issues.ipynb', 'repo_name': 'IPL-UV/rb', 'size': '4412'}