<jupyter_start><jupyter_text>Vim2<jupyter_code>%%time
import tables

f = tables.open_file('../data/crcns-vim2/Stimuli.mat')
Xv = f.get_node('/sv')[:]
Xt = f.get_node('/st')[:]
f.close()
h5file.close()
%%time
h5file = tables.open_file("../data/crcns-vim2/derived/Stimuli.mat", mode="w", title="Stimulus file")
h5file.create_array('/', 'sv', Xv, "Validation stimulus")
h5file.create_array('/', 'st', Xt, "Training stimulus")
h5file.flush()
h5file.close()
%%time
import tables

f = tables.open_file('../data/crcns-vim2/derived/Stimuli.mat')
Xv = f.get_node('/sv')[:]
Xt = f.get_node('/st')[:]
f.close()
%%time
import numpy as np
import tables

def get_max_r2(all_responses):
    all_responses = (all_responses - all_responses.mean(2, keepdims=True)) / all_responses.std(2, keepdims=True)

    p_r_bar = (np.nanmean(all_responses, axis=1) ** 2).mean(1)
    bar_p_r = (np.nanmean(all_responses ** 2, axis=1)).mean(1)
    p_r_bar

    N = (~np.isnan(all_responses).any(axis=2)).sum(axis=1)
    p_mu = 1 / (N - 1) * (N * p_r_bar - bar_p_r)
    max_r2 = p_mu / p_r_bar
    
    return max_r2

for subject in [1, 2, 3]:
    print(f"Subject {subject}")
    f = tables.open_file(f'../data/crcns-vim2/VoxelResponses_subject{subject}.mat')
    Yv = f.get_node('/rv')[:]
    Yt = f.get_node('/rt')[:]
    Ya = f.get_node('/rva')[:]
    max_r2 = get_max_r2(Ya)
    
    nodes = f.list_nodes('/roi')
    rois = {}
    for node in nodes:
        rois[node.name] = node[:]
    f.close()
    
    mask = (~np.isnan(Yv)).all(1) & (~np.isnan(Yt)).all(1)
    
    h5file = tables.open_file(f'../data/crcns-vim2/derived/VoxelResponses_subject{subject}.mat', mode="w", title="Response file")
    h5file.create_array('/', 'rv', Yv, "Validation responses")
    h5file.create_array('/', 'rt', Yt, "Training responses")
    h5file.create_array('/', 'maxr2', max_r2, "Max R^2 for this dataset")
    h5file.create_array('/', 'mask', mask, "Master mask")
    groups = h5file.create_group('/', 'roi')
    for key, node in rois.items():
        h5file.create_array(groups, key, node, "ROI")
    h5file.flush()
    h5file.close()
Yt.shape
%%time

import tables
f = tables.open_file(f'../data/crcns-vim2/derived/VoxelResponses_subject{subject}.mat')
Yv = f.get_node('/rv')[:]
Yt = f.get_node('/rt')[:]
nodes = f.list_nodes('/roi')
rois = {}
for node in nodes:
    rois[node.name] = node[:]
f.close()<jupyter_output>Wall time: 1.74 s<jupyter_text>pvc-1<jupyter_code>import matplotlib
import matplotlib.image
import numpy as np
import os
import tables

root = '../data/crcns-ringach-data'
movie_info = {}
h5file = tables.open_file(f'../data/crcns-ringach-data/derived/movies.h5', 'w')

for i in range(30):
    for j in range(4):
        print(i, j)
        root_ = os.path.join(root, "movie_frames", f"movie{j:03}_{i:03}.images")
        with open(os.path.join(root_, 'nframes'), 'r') as f:
            nframes = int(f.read())
        
            ims = []
            for frame in range(nframes):
                im_name = f'movie{j:03}_{i:03}_{frame:03}.jpeg'
                the_im = matplotlib.image.imread(os.path.join(root_, im_name))
        
                assert the_im.shape[0] == 240
                the_im = the_im.reshape((120, 2, 160, 2, 3)).mean(3).mean(1)
                the_im = the_im[8:, 24:136, :].transpose((2, 0, 1))
                ims.append(the_im.astype(np.uint8))
        
            m = np.stack(ims, axis=0)
            h5file.create_array('/', f'movie{j:03}_{i:03}', m, "Movie")

h5file.close()
h5file.close()
the_im = matplotlib.image.imread(os.path.join(root_, im_name))
plt.imshow(the_im)
the_im = the_im.reshape((120, 2, 160, 2, 3)).mean(3).mean(1).astype(np.uint8)
#the_im = the_im[8:, 24:136, :].astype(np.uint8)
plt.imshow(the_im)
import matplotlib.pyplot as plt
h5file = tables.open_file(f'../data/crcns-ringach-data/derived/movies.h5', 'r')
plt.imshow(h5file.get_node('/movie000_000')[:][10, :, :, :].transpose((1, 2, 0)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19433914581753853, 'fraction_numerical': 0.03335860500379075, 'mean_word_length': 3.392896781354051, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '146869', 'license': 'MIT', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1641, 'n_words': 273, 'path': 'notebooks/Derive datasets.ipynb', 'repo_name': 'ShahabBakht/brain-scorer', 'size': '3957'}