<jupyter_start><jupyter_text>Neural NetworkThis script uses a neural network to model the relationship between brains and cognition.<jupyter_code>import glob
import os

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Get Data<jupyter_code>def get_subject_from_path(path):
    normalized_path = os.path.normpath(path)
    path_components = normalized_path.split(os.sep)
    return path_components[-2][4:]
# Search for all functional connectivity files and read them
fc_path = '/imaging3/owenlab/bpho/python_power_fc'
fc_paths = glob.glob(fc_path + '/**/power_fc.npy', recursive=True)

fcs = {}
for path in fc_paths:
    subject_id = get_subject_from_path(path)
    # print(subject_id)
    subject_fc = np.load(path)
    fcs[subject_id] = subject_fc[np.triu_indices(264, k=1)]
print("Number of functional connectivity:", len(fcs))
print("Number of features (connections):", fcs["NDARAP912JK3"].shape[0])<jupyter_output>Number of functional connectivity: 722
Number of features (connections): 34716<jupyter_text>Get Labels<jupyter_code>wisc_label_path = "/imaging3/owenlab/bpho/Biobank Labels/Subjects_with_WISC.csv"
wisc_labels = pd.read_csv(wisc_label_path, index_col='assessment WISC,EID')
# display(wisc_labels)
subject_ids = wisc_labels.index
# measure_text = "WISC_FW_Scaled"
# measure_col = f'assessment WISC,{measure_text}'
measure_text = "Age"
measure_col = 'assessment Basic_Demos,Age'

fc_matrices = []
measures = []

for subject_id in subject_ids:
    if subject_id not in fcs:
        continue
    
    fc_matrices.append(fcs[subject_id])
    measures.append(wisc_labels.at[subject_id, measure_col])
    
print("Num subjects with measures:", len(measures))
X = np.array(fc_matrices)
y = np.array(measures)

print("X shape:", X.shape, "y shape:", y.shape)<jupyter_output>X shape: (577, 34716) y shape: (577,)<jupyter_text>Dense Neural Network<jupyter_code>input_layer = layers.Input((34716,))
layer_1 = layers.Dense(5000, activation='relu')(input_layer)
layer_2 = layers.Dense(5000, activation='relu')(layer_1)
layer_3 = layers.Dense(1000, activation='relu')(layer_2)
output_layer = layers.Dense(1, activation=None)(layer_3)
model = keras.models.Model(input_layer, output_layer)
model.summary()
model.compile(loss='mse', optimizer='adam', metrics=['accuracy'])
history = model.fit(X, y, batch_size=32, epochs=10, 
          validation_split=0.2, shuffle=True)
# Plot training & validation accuracy values
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper left')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1331806282722513, 'fraction_numerical': 0.020287958115183247, 'mean_word_length': 6.818414322250639, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '368757', 'license': 'MIT', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1052, 'n_words': 214, 'path': 'src/Unused (Archive)/Neural Network.ipynb', 'repo_name': 'Brian-Pho/MSc_Research-Project', 'size': '3056'}