<jupyter_start><jupyter_text>this makes sure that I can recover the PCAed vgg features.<jupyter_code>import os
from itertools import product
import numpy as np
import h5py
from tang_jcompneuro.io import load_split_dataset, load_split_dataset_idx, load_split_dataset_pretrained_pca_params
from tang_jcompneuro import dir_dictionary
from tang_jcompneuro.cnn_pretrained import blob_corresponding_info
blob_names = list(blob_corresponding_info['vgg19'].keys())
# dataset = load_split_dataset('MkA_Shape', 'all', True, slice(None), last_val=False, suffix='vgg19/legacy/conv3_1')
# len(dataset)
# data_idx = load_split_dataset_idx('MkA_Shape', 'all', True, last_val=False)
# len(data_idx)
# print(dataset[0].shape)
# print(dataset[1].shape)
# print(dataset[2].shape)
# print(dataset[3].shape)
# print(dataset[4].shape)
# print(dataset[5].shape)
# print(data_idx[0].shape)
# print(data_idx[1].shape)
# print(data_idx[2].shape)
# load original VGG features
def load_original_features(layer_name):
    file_in = os.path.join(dir_dictionary['features'], 'cnn_feature_extraction.hdf5')
    layer_idx = blob_names.index(layer_name)
    key_to_extract = '/'.join(['vgg19', 'legacy', str(layer_idx)])
    with h5py.File(file_in, 'r') as f:
        feature_this = f[f'Shape_9500/{key_to_extract}'][...].astype(np.float64, copy=False)
    print(feature_this.shape)
    return feature_this
# load_original_features('conv3_1')
# pca_mean.shape
# pca_components.shape
def check_one_layer(layer_name, seed):
    
    suffix=f'vgg19/legacy/{layer_name}'
    dataset = load_split_dataset('MkA_Shape', 'all', True, slice(None), last_val=False, suffix=suffix, seed=seed)
    dataset = (dataset[0], dataset[2], dataset[4])
    data_idx = load_split_dataset_idx('MkA_Shape', 'all', True, last_val=False, seed=seed)
    assert len(data_idx) == len(dataset) == 3
    pca_mean, pca_components = load_split_dataset_pretrained_pca_params('MkA_Shape', 'all', suffix=suffix, seed=seed)
    features_original = load_original_features(layer_name)
    features_original = features_original.reshape(features_original.shape[0], -1)
    
    # ok. let's check
    for (dataset_out_this, dataset_idx_this) in zip(dataset, data_idx):
        print(dataset_idx_this.shape)
        assert (dataset_out_this.shape[0],) == dataset_idx_this.shape
        
        dataset_out_this_debug = (features_original[dataset_idx_this] - pca_mean)@pca_components.T
        assert dataset_out_this_debug.shape == dataset_out_this.shape
        assert np.allclose(dataset_out_this_debug, dataset_out_this)
        print(np.array_equal(dataset_out_this_debug, dataset_out_this))
def check_all():
    for layer_name, seed in product(['conv2_1', 'conv3_1', 'conv4_1'], range(2)):
        print(layer_name, seed)
        check_one_layer(layer_name, seed)
check_all()<jupyter_output>conv2_1 0
(9500, 128, 8, 8)
(6080,)
True
(1520,)
True
(1900,)
True
conv2_1 1
(9500, 128, 8, 8)
(6080,)
True
(1520,)
True
(1900,)
True
conv3_1 0
(9500, 256, 2, 2)
(6080,)
True
(1520,)
True
(1900,)
True
conv3_1 1
(9500, 256, 2, 2)
(6080,)
True
(1520,)
True
(1900,)
True
conv4_1 0
(9500, 512, 1, 1)
(6080,)
True
(1520,)
True
(1900,)
True
conv4_1 1
(9500, 512, 1, 1)
(6080,)
True
(1520,)
True
(1900,)
True
{'fraction_non_alphanumeric': 0.1347407637379696, 'fraction_numerical': 0.058366966780502946, 'mean_word_length': 5.72651356993737, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '504951', 'license': 'Apache-2.0', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1215, 'n_words': 237, 'path': 'results_ipynb/debug/cnn_pretrained/check_pca.ipynb', 'repo_name': 'leelabcnbc/tang_jcompneuro_revision', 'size': '3221'}