<jupyter_start><jupyter_text>Processor Data AnalysisThis document contains debugging and exploratory information based on data collected as part of the CODEX processor CLI (saved as ```processor_data.json```) and is often helpful for the following:- Tracking large (and possibly erroneous) drift compensations- Viewing best focal plane selections for each region and tile (often useful for finding bad parts of a sample)- Diagnosing over/under saturation in individual channels- Examining execution times by operation  Parameters<jupyter_code># Parameters
processor_data_path = None
# Validation
assert processor_data_path is not None, 'Must set parameter for path to processor data json file'<jupyter_output><empty_output><jupyter_text>Load Processor Data<jupyter_code>%matplotlib inline
from cytokit.ops import op, best_focus, deconvolution, drift_compensation, tile_summary, tile_generator
from cytokit.exec import read_processor_data
from IPython import display
import plotnine as pn
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import json

# Set expected keys for operations dynamically based on class names (to avoid
# future refactoring synchronization problems)
drift_comp_op = op.CytokitOp.get_op_for_class(drift_compensation.CytokitDriftCompensator)
best_focus_op = op.CytokitOp.get_op_for_class(best_focus.CytokitFocalPlaneSelector)
tile_summary_op = op.CytokitOp.get_op_for_class(tile_summary.CytokitTileSummary)
decon_op = op.CytokitOp.get_op_for_class(deconvolution.CytokitDeconvolution)
tile_gen_op = op.CytokitOp.get_op_for_class(tile_generator.CytokitTileGenerator)

# Load processor data as a dict of data frames, separated by operation type
data = read_processor_data(processor_data_path)

# Explicitly ignore tile generator stats
if tile_gen_op in data:
    del data[tile_gen_op]
# Display the head rows for each operation type
for k in data:
    display.display(data[k].head().style.set_caption(k))<jupyter_output><empty_output><jupyter_text>Drift Compensation Summary<jupyter_code>df = data[drift_comp_op].copy()
dims = ['z', 'y', 'x'][-len(df['translation'].iloc[0]):]
df = df.groupby(['region_index', 'target_cycle'])['translation'].first().reset_index()
for i, dim in enumerate(dims):
    df['delta_' + dim] = df['translation'].apply(lambda v: v[i])

sns.heatmap(
    df.set_index(['region_index', 'target_cycle']).drop('translation', axis=1),
    annot=True, cmap='viridis',
    yticklabels=1, cbar=False
)
plt.gcf().set_size_inches(5, 4 * df['region_index'].nunique())
plt.gca().set_ylabel('Region Index - Target Cycle Index')
plt.gca().set_title('Drift Compensation Translation by Region/Cycle')<jupyter_output><empty_output><jupyter_text>Focal Plane Summary<jupyter_code>df = data[best_focus_op].copy()
(
    pn.ggplot(
        df.assign(
            z=df['best_z'].apply(lambda v: '{}'.format(v)),
            region_index=df['region_index'].apply(lambda v: 'Region {}'.format(v))
        ), 
        pn.aes(x='tile_x', y='tile_y', fill='z', width=.95, height=.95)
    ) +
    pn.scale_y_reverse() +
    pn.scale_fill_brewer(palette='YlOrRd', guide=pn.guide_legend(title='Best Z (0-Based)')) +
    pn.facet_wrap('~region_index', ncol=2) +
    pn.geom_tile() + 
    pn.xlab('Tile X Coordinate') +
    pn.ylab('Tile Y Coordinate') +
    pn.ggtitle('Best Focal Plane by Region/Tile') +
    pn.theme(figure_size=(12, 4 * (df['region_index'].nunique() + 1) // 2))
)<jupyter_output><empty_output><jupyter_text>Tile Summary Statistics<jupyter_code>df = data[tile_summary_op].copy()
(
    pn.ggplot(
        df.assign(
            region=df['region_index'].apply(lambda v: 'Region {}'.format(v)),
            channel=pd.Categorical(
                df['channel'],
                categories=df.groupby('channel')['mean'].mean().sort_values().index, ordered=True
            )
        ), 
        pn.aes(x='channel', y='mean')
    ) + 
    pn.geom_boxplot(outlier_size=0, alpha=.5) +
    pn.geom_jitter(pn.aes(color='tile_index'), width=.1) +
    pn.scale_y_log10() +
    pn.scale_color_continuous(guide=pn.guide_legend(title='Tile Index')) +
    pn.facet_wrap('~region_index', ncol=2) +
    pn.theme(figure_size=(24, 6 * (df['region_index'].nunique() + 1) // 2), axis_text_x=pn.element_text(rotation=90)) +
    pn.ggtitle('Mean Intensity Distributions Across Tiles') +
    pn.xlab('Channel Name') +
    pn.ylab('Mean Intensity')
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12919310970081596, 'fraction_numerical': 0.00657298277425204, 'mean_word_length': 5.11218836565097, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '188136', 'license': 'Apache-2.0', 'n_tokens_mistral': 1566, 'n_tokens_neox': 1475, 'n_words': 313, 'path': 'python/notebooks/analysis/processor_data_analysis.ipynb', 'repo_name': 'lcontento/cytokit', 'size': '4412'}