<jupyter_start><jupyter_text>Part D: Set shape of NPZ files for optimum annotator easeCrop array into overlapping x and y cropsSplit an array up into smaller crops that overlap one anotherCheck crop sizeFirst set parameters to check that crop size and overlap are appropriateload up huge array, pick the xy overlapping and processes these (not written to disk) take these re-shaped array and apply a slicing step to each apply the xy crops and write these files to disk. Makes a log of the settings used so that reconstruction is possible. * note that caliban toolbox must first be installed to your python env*<jupyter_code># import modules
import glob
import os
import path
import ntpath
import stat
import sys
import copy

import numpy as np
import pandas as pd

# Image processing tools
import mrcfile

import skimage
import skimage.io
from imageio import imread, volread, imwrite, volwrite
from ipywidgets import fixed, interactive


#import caliban_toolbox.pre_annotation.data_loader
#from caliban_toolbox import reshape_data
#from caliban_toolbox.figure_eight_functions import create_figure_eight_job, download_figure_eight_output
#from caliban_toolbox.utils import widget_utils, plot_utils, data_utils, io_utils

from bokeh.palettes import Viridis, Greys
from bokeh.io import export_png

import xarray as xr

import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline

from skimage import filters, img_as_uint
perm_mod = stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG

import bebi103

import bokeh
bokeh.io.output_notebook()

import panel as pn
pn.extension()

import colorcet

import holoviews as hv
hv.extension('bokeh')
# size of crop
crop_size = (480, 480)

# fraction of crop that will overlap with adjacent crops
overlap_frac = 0.1

# index of channel in npz to visualize
channels_idx = 1

# where crops will get saved
save_dir = "/example_data/timelapse/HeLa_by_image/crop_dir"
# we next add in dimensions of size 1 for any missing dims in input array to bring it up to standard size
X_expanded = data_utils.pad_xr_dims(input_xr=channel_xr)
y_expanded = data_utils.pad_xr_dims(input_xr=y_data)
# we'll test these parameters to see how the crops look
cropped_stack_test, cropped_y_test, log_data_test = reshape_data.crop_multichannel_data(X_data=X_expanded, 
                                                                                        y_data=y_expanded, 
                                                                                        crop_size=crop_size,
                                                                                        overlap_frac=overlap_frac,
                                                                                        test_parameters=True)<jupyter_output><empty_output><jupyter_text>Next we'll add in line overlays for easier visualization<jupyter_code># extract information about test crop dimensions
row_starts, row_ends  = log_data_test["row_starts"], log_data_test["row_ends"]
col_starts, col_ends = log_data_test["col_starts"], log_data_test["col_ends"]

# get a single crop from the stack of cropped images
example_crop = cropped_stack_test[0, 0, 3, 0, :, :, channels_idx]

# load the first image from the xarray stack, which is the one that the crops were generated from
example_img = copy.deepcopy(X_expanded[0, 0, 0, 0, ...])
example_img_overlay = plot_utils.overlay_grid_lines(overlay_img=example_img[:, :, channels_idx], 
                                                    row_starts=row_starts, row_ends=row_ends, col_starts=col_starts, 
                                                    col_ends=col_ends)
plt.figure(figsize = (10, 10))
plt.imshow(example_img_overlay[:, :])
# generate an overlay of a single crop showing where borders are located
img_crop_overlay = plot_utils.overlay_crop_overlap(img_crop=example_crop.values, row_starts=row_starts,
                                                   row_ends=row_ends, col_starts=col_starts, col_ends=col_ends)
plt.figure(figsize = (10, 10))
plt.imshow(img_crop_overlay)
# once the parameters above look good, we'll crop the images
X_cropped, y_cropped, log_data = reshape_data.crop_multichannel_data(X_data=X_expanded, y_data=y_expanded, 
                                                                     crop_size=crop_size, 
                                                                     overlap_frac=overlap_frac)<jupyter_output><empty_output><jupyter_text>now we need to crop in Z  write cropped subvolumes and log to disk<jupyter_code># save cropped data into npz files for upload 
io_utils.save_npzs_for_caliban(X_data=X_cropped, y_data=y_cropped, original_data=channel_data, save_dir=save_dir,
                                   log_data=log_data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.05518394648829431, 'fraction_numerical': 0.006061872909698997, 'mean_word_length': 2.941515650741351, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '364532', 'license': 'MIT', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1368, 'n_words': 430, 'path': 'subvolumes.ipynb', 'repo_name': 'HSapers/image_segmentation', 'size': '4784'}