<jupyter_start><jupyter_text>The purpose of this notebook is to run a simple factor analysis.Author: <NAME><jupyter_code>import matplotlib.pyplot as plt
import numpy as np

from sklearn.decomposition import FactorAnalysis, PCA
import scipy.io as sio
import csv
import pandas as pd
import os
import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Data Exploration<jupyter_code>path = '/data/bayrakrg/power+xifra/physio_patterns'
beh = path + '/behavioral_data_375_subjects_unadj.txt'
table = pd.read_csv(beh, delimiter=',')
table
family = path + '/restricted_part_ver2.xlsx'
df = pd.read_excel(family, sheet_name=None)
df<jupyter_output><empty_output><jupyter_text>Initial Analyses<jupyter_code># correlation
# path to percent variance explained maps
path_to_files = path + '/percent_variance_map_RVHR'
files = os.listdir(path_to_files)

# initialise data of lists.
tag, day, phase = [], [], []
for i, filename in enumerate(files):
    parts = filename.split('_')
    tag.append(parts[0])
    day.append(parts[2])
    phase.append(parts[3])

# Create a dict
data = {'tag': tag, 'day': day, 'phase': phase}

# Create DataFrame
df = pd.DataFrame(data)

# compare within day
# REST1 LR to RL, REST2 LR to RL

# plot



# compare across days
# Average within day scans and compare REST1 to REST2

# plot


# permutation test
# prep for PCA
# load brain mask
bm = '/data/MNI152/MNI152_T1_2mm_brain_mask_filled.nii.gz'
bm_inds = np.where(bm==1);

# path to percent variance explained maps
files = path + '/percent_variance_map_RVHR/'

# find unique subject ids
ids = sorted(df.tag.unique())

scans = ['REST1_LR', 'REST1_RL', 'REST2_LR', 'REST2_RL']

all_subjects = []

for tag in ids:
    mats, mean_mats = [], []
    for scan in scans:
#         if tag == '100206':
        filename = files + tag + '_rfMRI_' + scan + '_RVHR.mat'
        mat = sio.loadmat(filename)
        mats.append(mat['pctvar_vec_RVHR'])
    mean_mats = np.mean(np.nan_to_num(mats[0]), axis=0)
    all_subjects.append(mean_mats)

all_subject = np.array(all_subjects)
# Look at the data
plt.figure(figsize=(10,5))
plt.imshow(all_subject[:375, :1000])
plt.colorbar()
plt.show()

# Why does this look so odd?
# run PCA    
pca = PCA(n_components=0.99, svd_solver='full')
ve_pca = pca.fit_transform(all_subjects)

print('Original data shape:', all_subject.shape)
print('PCA data shape:', ve_pca.shape)
print('PCA explained by variance ratio:', pca.explained_variance_ratio_[:3])
"""
Plot the low dim representation of the bold data
"""
# Setting plotting parameters
alpha_val = .4

# Plot
n_plots = 3 
f, axes = plt.subplots(1, n_plots, figsize=(14,5))
st=f.suptitle("Figure 3.2. Scatter plots comparing PCA dimensions ", fontsize="x-large")

# plot data
axes[0].scatter(ve_pca[:, 0], ve_pca[:, 1], 
                alpha=alpha_val, marker='.', s=100, color = 'k')
axes[1].scatter(ve_pca[:, 1], ve_pca[:, 2], 
                alpha=alpha_val, marker='.', s=100, color = 'k')
axes[2].scatter(ve_pca[:, 2], ve_pca[:, 3], 
                alpha=alpha_val, marker='.', s=100, color = 'k')

axes[0].set_title('PCA Dimensions\n1 x 2')
axes[1].set_title('PCA Dimensions\n1 x 3')
axes[2].set_title('PCA Dimensions\n2 x 3')

# modifications that are common to all plots 
for i in range(n_plots): 
    axes[i].axis('equal')
    axes[i].set_xticks([])
    axes[i].set_yticks([])

f.tight_layout()
st.set_y(0.95)
f.subplots_adjust(top=0.75)
# CCA
# Factor Analysis<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14113597246127366, 'fraction_numerical': 0.02495697074010327, 'mean_word_length': 4.5792, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '203360', 'license': 'MIT', 'n_tokens_mistral': 1398, 'n_tokens_neox': 1294, 'n_words': 318, 'path': 'physio_patterns_analyses.ipynb', 'repo_name': 'rgbayrak/100DaysOfCode', 'size': '3494'}