<jupyter_start><jupyter_text>Cheat Sheet<jupyter_code>import numpy as np
import pandas as pd

from pystatplottools.pdf_env.loading_figure_mode import loading_figure_mode
fma, plt = loading_figure_mode(develop=True) # develop=False will export the generated figures as pngs into "./data/RectangleData"
plt.style.use('seaborn-dark-palette')<jupyter_output><empty_output><jupyter_text>pystatplottools.pytorch_data_generation - In Memory Dataset for a Custom Dataset  Custom Data Generator We start by defining a data generator for rectangles with a random unique color.<jupyter_code>from pystatplottools.pytorch_data_generation.data_generation.datageneratorbaseclass import DataGeneratorBaseClass


class RectangleGenerator(DataGeneratorBaseClass):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.inp_size = kwargs.pop("dim")
        self.tar_size = 2

        self.h, self.w = self.inp_size
        self.rectangle_hs = np.arange(1, self.h)
        self.rectangle_ws = np.arange(1, self.w)

    def sampler(self):
        rect_h = np.random.choice(self.rectangle_hs)
        rect_w = np.random.choice(self.rectangle_ws)
        sample = np.zeros((self.h, self.w))

        left_idx_h = np.random.randint(0, self.h - rect_h + 1, 1)[0]
        left_idx_w = np.random.randint(0, self.w - rect_w + 1, 1)[0]

        sample[left_idx_h:left_idx_h + rect_h, left_idx_w:left_idx_w + rect_w] = np.random.rand() * (rect_h / (left_idx_w + 1)) / self.h
        return sample, np.array([rect_h, rect_w])


class BatchRectangleGenerator(RectangleGenerator):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.batch_size = kwargs.pop("batch_size")

    def sampler(self):
        rect_hs = np.random.choice(self.rectangle_hs, self.batch_size)
        rect_ws = np.random.choice(self.rectangle_ws, self.batch_size)
        batch = np.zeros((self.batch_size, self.h, self.w))

        left_idx_hs, left_idx_ws = np.zeros(self.batch_size, dtype=np.int), np.zeros(self.batch_size, dtype=np.int)
        for rect_h in self.rectangle_hs:
            left_idx_hs[rect_hs == rect_h] = np.random.randint(0, self.h - rect_h + 1, np.sum(rect_hs == rect_h))
        for rect_w in self.rectangle_ws:
            left_idx_ws[rect_ws == rect_w] = np.random.randint(0, self.w - rect_w + 1, np.sum(rect_ws == rect_w))

        for idx, (sample, rect_h, rect_w, left_idx_h, left_idx_w) in enumerate(zip(batch, rect_hs, rect_ws, left_idx_hs, left_idx_ws)):
            batch[idx, left_idx_h:left_idx_h + rect_h, left_idx_w:left_idx_w + rect_w] = np.random.rand() * (rect_h / (left_idx_w + 1)) / self.h

        return batch, np.stack([rect_hs, rect_ws], axis=1)


def data_generator_factory(data_generator_name="RectangleGenerator"):
    if data_generator_name == "BatchRectangleGenerator":
        return BatchRectangleGenerator
    else:
        return RectangleGenerator<jupyter_output><empty_output><jupyter_text>Preparing and Loading an In-Memory Dataset The following two functions prepare and load the a generated dataset for the rectangle generator.<jupyter_code>def prepare_rectangle_data_in_memory_dataset():
    rect_data_dir = "RectangleData"
    root = "./data/RectangleData"

    from pystatplottools.utils.utils import set_up_directories
    data_root, results_root = set_up_directories(data_dir=rect_data_dir, results_dir=rect_data_dir)

    ''' Data generation with storage of a permament file '''

    data_generator_args = {
        # RectangleGenerator Args
        "dim": [10, 12]
    }
    # Prepare in memory dataset
    from pystatplottools.pytorch_data_generation.data_generation.datagenerationroutines \
    import prepare_in_memory_dataset
    from examples.rectangle_data_generator import data_generator_factory

    prepare_in_memory_dataset(
        root=root,
        batch_size=89,
        n=50000,
        data_generator_args=data_generator_args,
        data_generator_name="BatchRectangleGenerator",
        data_generator_factory=data_generator_factory
    )


def load_rectangle_data_memory_dataset():
    # Load in memory dataset
    from pystatplottools.pytorch_data_generation.data_generation.datagenerationroutines \
    import load_in_memory_dataset
    from examples.rectangle_data_generator import data_generator_factory
    data_loader = load_in_memory_dataset(
        root="./data/RectangleData", batch_size=89, data_generator_factory=data_generator_factory, slices=None, shuffle=True,
        num_workers=0, rebuild=False
        # sample_data_generator_name="RectangleGenerator"  # optional: for a generation of new samples
    )
    return data_loader
directory = "./results/RectangleData"
prepare_rectangle_data_in_memory_dataset()
data_loader = load_rectangle_data_memory_dataset()

# Iterate over dataset
for batch_idx, batch in enumerate(data_loader):
    data, target = batch<jupyter_output>Random seed is set by np.random.seed()<jupyter_text>pystatplottools.visualization - Inspection of the dataset<jupyter_code>from pystatplottools.visualization import sample_visualization

config_dim = (10, 12) # Dimension of the generated samples

# Random sample
config, label = data_loader.dataset.get_random_sample()
# Random batch with size 108
batch, batch_label = data_loader.dataset.get_random_batch(108)
# Single Sample
sample_visualization.fd_im_single_sample(sample=config, label=label, config_dim=config_dim, minmax=(0, 1),
                                         fma=fma, filename="single_sample", directory=directory, figsize=(4, 4));
# Batch with labels
sample_visualization.fd_im_batch(batch, batch_labels=batch_label, num_samples=24, dim=(4, 6), config_dim=config_dim,
                                 ab=(batch[:24].cpu().numpy().min(), batch[:24].cpu().numpy().max()), fma=fma, filename="batch", directory=directory, width=3.0);
# Batch grid
sample_visualization.fd_im_batch_grid(batch, config_dim=config_dim, ab=(0, 1),
                                      fma=fma, filename="batch_grid", directory=directory);<jupyter_output><empty_output><jupyter_text>pystatplottools.expectation_values - Expectation Values of the Rectangle Dataset<jupyter_code>from examples.mock_data import load_characteristics
data = load_characteristics()
data<jupyter_output><empty_output><jupyter_text>The data dataframe contains some characteristics for each sample:- 'height' and 'width' refer to the the height and width of the rectangle- 'dx' and 'dy' are the distances in pixels in the horizontal and the vertical direction counted from the left and the top of the sample- 'mean' is the mean pixel color of the sample- 'color' refers to the color of the rectangleThe data is sorted based on the area 'A' of the rectangle.  Expectation Values for each Rectangle Area 'A'<jupyter_code>from pystatplottools.expectation_values.expectation_value import ExpectationValue

ep = ExpectationValue(data=data)

# Computes for the given columns the respective expectation values - The expectation values are computed seperately for each area 'A'
ep.compute_expectation_value(columns=["mean", "dx", "dy", "height", "width"],
                             exp_values=["mean", "max", "min", "secondmoment", "fourthmoment"])

expectation_values = ep.expectation_values
expectation_values<jupyter_output><empty_output><jupyter_text>Expectation Values for all Rectangle Areas 'A'<jupyter_code># Expectation values over entire dataset - drop 'A' dependence and compute the same for the entire dataset - summarized as one dataframe df
from pystatplottools.utils.utils import drop_index_level
ep_all_data = ExpectationValue(data=drop_index_level(data=data, single_key="df"))
ep_all_data.compute_expectation_value(columns=["mean", "dx", "dy", "height", "width"],
                                      exp_values=["mean", "max", "min", "secondmoment", "fourthmoment"])
expectation_values_all_data = ep_all_data.expectation_values
expectation_values_all_data.transpose()<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code># Visualization of the average pixel color as a function the rectangle area 'A'
fig, ax = fma.newfig(1.0)

rectangle_areas = expectation_values.index.values
mean_all_data = expectation_values_all_data["mean"]["mean"].iloc[0]
ax.plot([rectangle_areas[0], rectangle_areas[-1]], [mean_all_data, mean_all_data], label="Weighted Mean")

ax.plot(rectangle_areas, expectation_values["mean"]["mean"], label="Mean")
ax.plot(rectangle_areas, expectation_values["mean"]["min"], color="C{}".format(1), ls="-.", label="Min")
ax.plot(rectangle_areas, expectation_values["mean"]["max"], color="C{}".format(1), ls="--", label="Max")

ax.legend()
ax.set_xlabel("Rectangle Area")
ax.set_ylabel("Average Pixel Color")
ax.set_xscale("log")
ax.set_yscale("log")
plt.tight_layout()

fma.savefig(directory, "image_mean_over_rectangle_area")<jupyter_output><empty_output><jupyter_text>pystatplottools.distributions - Computation of (Probability) Distributions  distributions.marginal_distribution - Marginal Distributions / 1D Distributions<jupyter_code>from pystatplottools.distributions.marginal_distribution import MarginalDistribution

marginal_distribution = MarginalDistribution(data=data)

range_min, range_max = marginal_distribution.extract_min_max_range_values(["height", "width", "dx", "dy", "mean",
                                                                           "color"])
marginal_distribution.compute(
    axes_indices=["height", "width", "dx", "dy", "mean", "color"],
    range_min=range_min,
    range_max=range_max,
    nbins=[10, 12, 11, 9, 10, 12],
    statistic='probability',
    bin_scales=['linear', 'linear', 'linear', 'linear', 'logarithmic', 'logarithmic']
)

linearized_marginal_distribution = marginal_distribution.linearize(
    order_by_bin=True,
    bin_alignment="center"
)
linearized_marginal_distribution<jupyter_output><empty_output><jupyter_text>Visualization as Histograms<jupyter_code>from pystatplottools.utils.bins_and_alignment import revert_align_bins

areas = [5, 42, 88]

fig, axes = fma.newfig(1.8, nrows=6, ncols=3, ratio=1.0, figsize=(11, 11))
for i, area in enumerate(areas):
    for j, (observable, bin_scale) in enumerate(zip(["height", "width", "dx", "dy", "mean", "color"],
                                                    ['linear', 'linear', 'linear', 'linear', 'logarithmic',
                                                     'logarithmic'])):
        binedges = revert_align_bins(
            data_range=linearized_marginal_distribution.loc[observable]["bin"].values,
            bin_alignment="center",
            bin_scale=bin_scale
        )
        if bin_scale == "logarithmic":
            width = 0.6 * (binedges[1:] - binedges[:-1])
            axes[j][i].set_xscale("log")
        else:
            width = 0.9 * (binedges[1:] - binedges[:-1])

        axes[j][i].bar(
            x=linearized_marginal_distribution.loc[observable]["bin"].values,
            height=linearized_marginal_distribution.loc[observable][area].values,
            width=width
        )
        axes[j][i].set_xlabel(observable)
        from pystatplottools.visualization.utils import add_fancy_legend_box
        add_fancy_legend_box(ax=axes[j][i], name=float(area))

for j, observable in enumerate(["height", "width", "dx", "dy", "mean", "color"]):
    axes[j][0].set_ylabel("P(" + observable + ")")
        
# ax.set_xlabel("Rectangle Area")
# ax.set_ylabel("Average Pixel Color")

plt.tight_layout()

fma.savefig(directory, "marginal_distributions")<jupyter_output><empty_output><jupyter_text>distributions.joint_distribution - Joint Distributions<jupyter_code>from pystatplottools.distributions.joint_distribution import JointDistribution

joint_distribution = JointDistribution(data=data)

range_min, range_max = joint_distribution.extract_min_max_range_values(["dx", "dy", "mean"])

joint_distribution.compute(
    axes_indices=["dx", "dy", "mean"],
    range_min=range_min,
    range_max=range_max,
    nbins=[11, 9, 10],
    statistic="probability"
)

# Internally a multidimensional histogram is computed with np.histogram() for each rectangle area.
# The histograms can be accessed via: joint_distribution.distribution or linearized.

# Transforms joint_distribution into a linear list of mid boundaries for the different bins
# and the respective statistics for the values
linearized_joint_distribution = joint_distribution.linearize(
    output_statistics_name="prob",
    dataframes_as_columns=False,
    bin_alignment="center"
)
linearized_joint_distribution<jupyter_output><empty_output><jupyter_text>pystatplottools.plotting.contour2D - Visualization of a 2D Joint Distribution<jupyter_code># Preparation - Merge all areas
from pystatplottools.utils.utils import drop_index_level
data_all = drop_index_level(data=data, single_key="df")
from pystatplottools.distributions.joint_distribution import JointDistribution

joint_distribution = JointDistribution(data=data_all)

range_min, range_max = list(map(list, data_all[["dx", "dy"]].agg(["min", "max"]).values))
joint_distribution.compute(
    axes_indices=["dx", "dy"],
    range_min=range_min,
    range_max=range_max,
    nbins=[11, 9],
    statistic='probability'
)

linearized_joint_distribution = joint_distribution.linearize(output_statistics_name="prob")

# Contour plot
fig, ax = fma.newfig(1.4)

from pystatplottools.plotting.contour2D import Contour2D

contour2D = Contour2D(
    ax=ax,
    data=linearized_joint_distribution.loc["df"],
    x="dx",  # possibility to rescale x and y axis or perform other operation for x axis
    # like computing a mass difference
    y="dy",
    z_index="prob"
)

contour2D.set_ax_labels(x_label="dx", y_label="dy")
cf = contour2D.pcolormesh(
    cbar_scale="Lin",
    lev_num=40,
    cmap="PiYG"
)
contour2D.add_colorbar(fig=fig, cf=cf, z_label="Probability")

plt.tight_layout()
fma.savefig(directory, "probability_colormesh")<jupyter_output><empty_output><jupyter_text>distributions.binned_statistics - Computation of Binned Statistics Computing and Visualizing mean for different values of (dx, height)<jupyter_code># Preparation - Merge all areas
from pystatplottools.utils.utils import drop_index_level
data_all = drop_index_level(data=data, single_key="df")

from pystatplottools.distributions.binned_statistics import BinnedStatistics

binned_statistics = BinnedStatistics(data=data_all)

range_min, range_max = list(map(list, data_all[["dx", "height"]].agg(["min", "max"]).values))
binned_statistics.compute(
    axes_indices=["dx", "height"],
    columns=["mean", "color"],
    range_min=range_min,
    range_max=range_max,
    nbins=[11, 8],
    statistic='mean'
)

linearized_statistics = binned_statistics.linearize(
    output_statistics_names=["mean", "color"],
    dataframes_as_columns=False,
    bin_alignment="center"
)
linearized_statistics<jupyter_output><empty_output><jupyter_text>Visualization of Binned Statistics with a Logarithmic Colorbar<jupyter_code># Contour plot
fig, ax = fma.newfig(1.4, figsize=(10, 7), ratio=0.5, nrows=1, ncols=2)

# Preparation
from pystatplottools.plotting.contour2D import Contour2D

contour2D_mean = Contour2D(
    ax=ax[0],
    data=linearized_statistics.loc["df"],
    x="dx",
    y="height",
    z_index="mean"
)

contour2D_color = Contour2D(
    ax=ax[1],
    data=linearized_statistics.loc["df"],
    x="dx",
    y="height",
    z_index="color"
)

contour2D_mean.set_ax_labels(x_label="dx", y_label="Height")

contour2D_color.set_ax_labels(x_label="dx")
contour2D_color.set_ax_label_visibility(ax[1], y_ticks_visible=False)

lev_min = linearized_statistics[["mean", "color"]].values.min()
lev_max = linearized_statistics[["mean", "color"]].values.max()
norm, levs = Contour2D.get_log_norm_and_levs(lev_min=lev_min, lev_max=lev_max, lev_num=40)

contour2D_mean.contourf(
    norm=norm,
    levs=levs,
    cmap="PiYG"
)
contour2D_mean.add_fancy_box("Average Pixel Color")

cf = contour2D_color.contourf(
    norm=norm,
    levs=levs,
    cmap="PiYG"
)
contour2D_color.add_fancy_box("Rectangle Color")

fig.subplots_adjust(wspace=0.1, right=0.9)
cbar_ax = fig.add_axes([0.92, 0.05, 0.012, 0.8])

Contour2D.add_colorbar(
    fig=fig, cf=cf,
    z_label="<Color>",
    cax=cbar_ax,
    z_ticks=[1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1],
    z_tick_labels=['$10^{-4}$', '$3\\times10^{-4}$', '$10^{-3}$', '$3\\times10^{-3}$', '$10^{-2}$',
                   '$3\\times10^{-2}$', '$10^{-1}$', '$3\\times10^{-1}$']
)

fma.savefig(directory, "mean_colormesh")<jupyter_output><empty_output><jupyter_text>distributions.sparse_joint_distribution - Computation of Joint Distributions in High Dimensions and Marginalization Preparation<jupyter_code># Downsizing dataset samples - Mean value over 2x2 blocks (from 10 x 12 to 5 x 6)
sample_data = data_loader.dataset.data.cpu().numpy().reshape(len(data_loader.dataset), -1)
sample_data = sample_data.reshape(-1, 5, 2, 6, 2).mean(axis=(2, 4)).reshape(len(data_loader.dataset), -1)
import pandas as pd
from pystatplottools.utils.utils import add_index_level
sample_data = add_index_level(data=pd.DataFrame(sample_data), key="df")
sample_data # 30 pixels per sample<jupyter_output><empty_output><jupyter_text>Computation of a Sparse Joint Distribution in Pixel Space with 10 Bins in each Dimension'.<jupyter_code>from pystatplottools.distributions.sparse_joint_distribution import SparseJointDistribution

spare_joint_distribution = SparseJointDistribution(data=sample_data)
range_min, range_max = spare_joint_distribution.extract_min_max_range_values()

spare_joint_distribution.compute(
    statistic="probability",
    transform='lin',
    nbins=[10 for _ in range(len(sample_data.columns))],
    range_min=range_min,
    range_max=range_max,
    bin_scales='linear',
    one_dim_bin_index=False
)

sparse_distribution = spare_joint_distribution.distribution

linearized_sparse_distribution, bin_information = spare_joint_distribution.linearize(order_by_bin=True)

# Data is stored by a vector (bin_0, ..., bin_29) consisting of the bin index in each dimension
linearized_sparse_distribution<jupyter_output><empty_output><jupyter_text>Possiblity to represent this as a joint distribution with respective bins<jupyter_code>linearized_joint_distribution = spare_joint_distribution.linearized_sparse_distribution_to_linearized_joint_distribution(
    linearized_sparse_distribution=linearized_sparse_distribution,
    bin_information=bin_information,
    bin_alignment="center"
)
linearized_joint_distribution<jupyter_output><empty_output><jupyter_text>Marginalization - Also works for distributions.joint_distribution<jupyter_code>partly_marginalized_distribution = spare_joint_distribution.marginalize(
    initial_axes_indices=list(range(30)), remaining_axes_indices=list(range(5)), data=linearized_joint_distribution)

partly_marginalized_distribution<jupyter_output><empty_output><jupyter_text>Verify if the computed marginal distribution is correct<jupyter_code># Verify marginal distribution

part_spare_joint_distribution = SparseJointDistribution(data=sample_data[list(range(5))])
part_range_min, part_range_max = part_spare_joint_distribution.extract_min_max_range_values()

part_sparse_distribution = part_spare_joint_distribution.compute(
    statistic="probability",
    transform='lin',
    nbins=[10 for _ in range(5)],
    range_min=part_range_min,
    range_max=part_range_max,
    bin_scales='linear'
)

linearized_part_sparse_distribution, part_bin_information = part_spare_joint_distribution.linearize(order_by_bin=True)

linearized_part_joint_distribution = part_spare_joint_distribution.linearized_sparse_distribution_to_linearized_joint_distribution(
    linearized_sparse_distribution=linearized_part_sparse_distribution,
    bin_information=part_bin_information,
    bin_alignment="center"
)

# assert np.sum(partly_marginalized_distribution.values - linearized_part_joint_distribution.values) < 1e-10, \
#     "Distributions do not coincide"

linearized_part_joint_distribution
# Check whether partly_marginalized_distribution and linearized_part_joint_distribution are the same
np.sum(partly_marginalized_distribution.values - linearized_part_joint_distribution.values)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11023504909253198, 'fraction_numerical': 0.012397104036497074, 'mean_word_length': 5.288431555971313, 'pattern_counts': {'":': 3, '<': 78, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '94358', 'license': 'MIT', 'n_tokens_mistral': 6890, 'n_tokens_neox': 6543, 'n_words': 1234, 'path': 'examples/cheat_sheet.ipynb', 'repo_name': 'statphysandml/pystatplottools', 'size': '20166'}