<jupyter_start><jupyter_text>Try higher resolution data and write directly into cooler<jupyter_code># try 10kb ...
clr = cooler.Cooler('./test.mcool::/resolutions/10000')
# generate bins table with weights=1, and NaN for bad bins ...
bins_oe = clr.bins()[:]
_bad_mask = bins_oe["weight"].isna()
bins_oe["weight"] = 1.
bins_oe.loc[_bad_mask,"weight"] = np.nan
# re-calculate full expected (cis + trans) at higher resolution
expected_df = obs_over_exp_cooler.expected_full(
    clr,
    view_df=hg38_arms,
    smooth_cis=False,
    aggregate_trans=True,
    expected_column_name="expected",
    nproc=4,
)
# setup a generator (lazy) of obs/exp pixels
oe_pixels_stream = obs_over_exp_cooler.obs_over_exp_generator(
    clr,
    expected_df,
    view_df=hg38_arms,
    expected_column_name="expected",
    oe_column_name='oe',
    chunksize=10_000_000
)
# write oe_pixels_stream into cooler - with custom column "oe" (can do "count":float for higlass)
cooler.create_cooler(
    cool_uri = "fun.cool",
    bins = bins_oe,
    pixels = oe_pixels_stream,
    columns=["oe"],
    dtypes={"oe":np.float64},
)
# plot observed and stitched obs/exp side by side directly from the new cooler
istart, iend = 23_000, 25_000
obs = clr.matrix()[istart:iend, istart:iend]
obs_exp = cooler.Cooler("fun.cool").matrix(field="oe")[istart:iend, istart:iend]

f,axs = plt.subplots(1,2,figsize=(14,10))
img = axs[0].imshow(
    obs,
    interpolation="none",
    cmap="YlOrRd",
    norm=colors.LogNorm(vmin=0.00005,vmax=0.01)
)
plt.colorbar(img,ax=axs[0],orientation="horizontal")

# make sure zeros are displayed as "lowest" obs/exp according to the colormap
cm = plt.cm.get_cmap("coolwarm")
cm.set_under(cm(0))

img = axs[1].imshow(
    obs_exp+10**-8,
    interpolation="none",
    cmap=cm,
    norm=colors.LogNorm(vmin=0.4,vmax=2.5)
    # add color to the "under" - to avoid adding "floor" to obs_exp
    # 
)
plt.colorbar(img,ax=axs[1],orientation="horizontal")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14104515474378487, 'fraction_numerical': 0.030948756976154235, 'mean_word_length': 4.7492711370262395, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '508202', 'license': 'MIT', 'n_tokens_mistral': 816, 'n_tokens_neox': 756, 'n_words': 149, 'path': 'cooltools/sandbox/observed_over_expected_example.ipynb', 'repo_name': 'mirnylab/cooltools', 'size': '1971'}