<jupyter_start><jupyter_text>Note that apparently taking chunked avg does not result in exact same values as taking avg without chunking<jupyter_code>import os
import h5py
import sima
import numpy as np
import matplotlib.pyplot as plt
import tifffile as tiff

fdir = r'D:\bruker_data\test_small_session\vj_ofc_imageactivate_001_2020903-001_small'
fname = 'vj_ofc_imageactivate_001_2020903-001_small'
chunk_size = 2000.0
analyze_mc = True

# define file paths
raw_fpath = os.path.join(fdir, fname + '.h5')
sima_mc_fpath = os.path.join(fdir, fname)+ '_sima_mc.h5'

if analyze_mc:
    data_analyze_fpath = sima_mc_fpath
else:
    data_analyze_fpath = raw_fpath

""" 
generate sima_mc.h5 from .sima folder

note: main_process code applies bidi offset correction to sequences file in .sima folder

"""
if analyze_mc:
    if os.path.exists(os.path.join(fdir, fname+'_mc.sima')) and not os.path.exists(sima_mc_fpath):
        dataset = sima.ImagingDataset.load(os.path.join(fdir, fname+'_mc.sima'))
        dataset.sequences[0].export(sima_mc_fpath, fmt='HDF5', fill_gaps=True, channel_names=None)
# define h5 obj and figure out frame indices for each chunk

h5 = h5py.File(data_analyze_fpath,'r')
h5_obj = h5[list(h5.keys())[0]]
num_frames = h5_obj.shape[0]
if len(h5_obj.shape)==5: # for h5's that have not been squeezed coming out of sima analysis
    y_dim = h5_obj.shape[2]; x_dim = h5_obj.shape[3]
else:
    y_dim = h5_obj.shape[-2]; x_dim = h5_obj.shape[-1]

n_chunks = int(np.ceil(num_frames/chunk_size))
chunked_frame_idx = np.array_split(np.arange(num_frames), n_chunks) # split frame indices into chunks
proj_names = ['mean', 'std', 'max']
dict_projections = {}
for proj_name in proj_names:
    dict_projections[proj_name] = np.empty([n_chunks, y_dim, x_dim])
for chunk_idx, frame_idx in enumerate(chunked_frame_idx):
    print('projecting from frame {} to {}'.format(frame_idx[0],frame_idx[-1]))
    chunk_data = np.array(np.squeeze( h5_obj[frame_idx,...] )).astype('uint16') # np.array loads all data into memory

    dict_projections['mean'][chunk_idx,...] = np.mean(chunk_data, axis=0)
    dict_projections['std'][chunk_idx,...] = np.std(chunk_data, axis=0)
    dict_projections['max'][chunk_idx,...] = np.max(chunk_data, axis=0)
h5.close()
fig, axs = plt.subplots(1,3,figsize=(10,5))

for idx, proj_name in enumerate(proj_names):
    
    # take mean across chunked avges
    dict_projections['all_frame_'+proj_name] = np.squeeze(np.mean(dict_projections[proj_name], axis=0))
    
    axs[idx].imshow(dict_projections['all_frame_'+proj_name])
    axs[idx].set_title(proj_name)
    tiff.imwrite(os.path.join(fdir, '{}_img.tif'.format(proj_name)), dict_projections['all_frame_'+proj_name])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13242846661775495, 'fraction_numerical': 0.02641232575201761, 'mean_word_length': 5.431603773584905, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '651649', 'license': 'MIT', 'n_tokens_mistral': 1094, 'n_tokens_neox': 1055, 'n_words': 208, 'path': 'napeca/post_sima_processing.ipynb', 'repo_name': 'Alex-de-Lecea/NAPE_imaging_analysis', 'size': '2726'}