<jupyter_start><jupyter_text>RetrieveThe fMRI time series are taken from https://paris-saclay-cds.github.io/autism_challenge/. You have to follow their instructions on how to retrieve the relavant fMRI data for the MSDL atlas.Check out their GitHub repository at: https://github.com/ramp-kits/autism/.  Parse The following code is heavily based on the code provided by the competition's organizers.<jupyter_code>import numpy as np
import pandas as pd
import os
autism_data = '/home/makism/Temp/autism/'
def parse_dataset():
    _target_column_name = 'asd'
    _prediction_label_names = [0, 1]
    
    subject_id = pd.read_csv(os.path.join(autism_data, 'data', 'train.csv'), header=None)
    # read the list of the subjects
    df_participants = pd.read_csv(os.path.join(autism_data, 'data', 'participants.csv'), index_col=0)
    df_participants.columns = ['participants_' + col for col in df_participants.columns]
    
    # load the structural and functional MRI data
    df_anatomy = pd.read_csv(os.path.join(autism_data, 'data', 'anatomy.csv'), index_col=0)
    df_anatomy.columns = ['anatomy_' + col for col in df_anatomy.columns]
    df_fmri = pd.read_csv(os.path.join(autism_data, 'data', 'fmri_filename.csv'), index_col=0)
    df_fmri.columns = ['fmri_' + col for col in df_fmri.columns]
    
    # load the QC for structural and functional MRI data
    df_anatomy_qc = pd.read_csv(os.path.join(autism_data, 'data', 'anatomy_qc.csv'), index_col=0)
    df_fmri_qc = pd.read_csv(os.path.join(autism_data, 'data', 'fmri_qc.csv'), index_col=0)
    
    # rename the columns for the QC to have distinct names
    df_anatomy_qc = df_anatomy_qc.rename(columns={"select": "anatomy_select"})
    df_fmri_qc = df_fmri_qc.rename(columns={"select": "fmri_select"})

    X = pd.concat([df_participants, df_anatomy, df_anatomy_qc, df_fmri, df_fmri_qc], axis=1)
    X = X.loc[subject_id[0]]
    y = X['participants_asd']
    y = y.rename(columns={'participants_asd': _target_column_name})
    X = X.drop('participants_asd', axis=1)

    return X, y.values
data, labels = parse_dataset()
fmri_data = data[[col for col in data.columns if col.startswith('fmri')]]
fmri_msdl_filenames = fmri_data['fmri_msdl']
fmri = np.array([pd.read_csv(autism_data + "/" + subject_filename, header=None).values 
                 for subject_filename in fmri_msdl_filenames])
anatomy = data[[col for col in data.columns if col.startswith('anatomy')]]
anatomy = anatomy.drop(columns='anatomy_select')<jupyter_output><empty_output><jupyter_text>Store files<jupyter_code>np.save('data/autism_fmri_ts.npy', fmri)
np.save('data/autism_anatomy.npy', anatomy)
np.save('data/autism_labels.npy', labels)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12416604892512972, 'fraction_numerical': 0.0037064492216456633, 'mean_word_length': 4.893013100436681, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '405258', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1029, 'n_tokens_neox': 964, 'n_words': 211, 'path': 'tutorials/0 - Retrieve and parse fMRI.ipynb', 'repo_name': 'wmvanvliet/dyconnmap', 'size': '2698'}