<jupyter_start><jupyter_text>Metadata Organization  Imports<jupyter_code>import pandas as pd 
import numpy as np
import os.path
import glob
import pathlib
import functools
import time
#from pycaret.classification import *
import re<jupyter_output><empty_output><jupyter_text>Function to find all the regressor file paths<jupyter_code>def timer(func):
    """Print the runtime of the decorated function"""
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        print(f'Calling {func.__name__!r}')
        startTime = time.perf_counter()
        value = func(*args, **kwargs)
        endTime = time.perf_counter()
        runTime = endTime - startTime
        print(f'Finished {func.__name__!r} in {runTime:.4f} secs')
        return value
    return wrapper<jupyter_output><empty_output><jupyter_text>Function to find all the BOLD NII file paths<jupyter_code>@timer
def find_paths(relDataFolder, subj, sess, func, patt):
    paths = list(pathlib.Path(relDataFolder).glob(
                        os.path.join(subj, sess, func, patt)
                    )
                )
                        
    return paths<jupyter_output><empty_output><jupyter_text>Find all the regressor file paths<jupyter_code>regressor_paths = find_paths(relDataFolder='../data/preprocessed',
                            subj='sub-*',
                            sess='ses-*',
                            func='func',
                            patt="*confounds_regressors.tsv")
regressor_paths<jupyter_output>Calling 'find_paths'
Finished 'find_paths' in 0.0228 secs<jupyter_text>Find all the BOLD NII file paths<jupyter_code>nii_paths = find_paths(relDataFolder='../data/preprocessed',
                        subj='sub-*',
                        sess='ses-*',
                        func='func',
                        patt="*MNI152NLin2009cAsym_desc-preproc_bold.nii.gz")
nii_paths<jupyter_output>Calling 'find_paths'
Finished 'find_paths' in 0.0193 secs<jupyter_text>Read the participants.tsv file to find summaries of the subjects<jupyter_code>participant_info_df = pd.read_csv(
        '../data/participants.tsv',
        sep='\t'
    )
participant_info_df<jupyter_output><empty_output><jupyter_text>Get a mapping Dataframe of subject and which session is the sleep deprived one<jupyter_code>@timer
def map_sleepdep(participant_info):
    df = pd.DataFrame(participant_info.loc[:,['participant_id', 'Sl_cond']])
    df.replace('sub-', '', inplace=True, regex=True)
    return df.rename(columns={'participant_id':'subject', 'Sl_cond':'sleepdep_session'})

sleepdep_map = map_sleepdep(participant_info_df)
sleepdep_map<jupyter_output>Calling 'map_sleepdep'
Finished 'map_sleepdep' in 0.0026 secs<jupyter_text>Get Dataframe of subject, session, task, path<jupyter_code>def get_bids_components(paths):
    components_list = []
    for path in paths:
        filename = path.stem
        matches = re.search(
            '[a-z0-9]+\-([a-z0-9]+)_[a-z0-9]+\-([a-z0-9]+)_[a-z0-9]+\-([a-z0-9]+)', 
            filename
        )
        subject = matches.group(1)
        session = matches.group(2)
        task = matches.group(3)
        components_list.append([subject, session, task, path.__str__(), 0])
    df = pd.DataFrame(components_list, 
                        columns=['subject', 'session', 'task', 'path', 'sleepdep']
                     )
    return df

components_df = get_bids_components(nii_paths)
components_df<jupyter_output><empty_output><jupyter_text>Combine logically sleepdep_map and components_df into 1 dataframe<jupyter_code>final_df = components_df.merge(sleepdep_map, how='left')<jupyter_output><empty_output><jupyter_text>Response column 'sleepdep' imputed from 'session' 'sleepdep_session'<jupyter_code>for i in range(len(final_df)):
    if int(final_df['session'].iloc[i]) == int(final_df['sleepdep_session'].iloc[i]):
        final_df['sleepdep'].iloc[i] = 1
final_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11980627071119042, 'fraction_numerical': 0.010451185317359163, 'mean_word_length': 3.350332594235033, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '242759', 'license': 'Apache-2.0', 'n_tokens_mistral': 1347, 'n_tokens_neox': 1295, 'n_words': 233, 'path': '14-individualExperimentation/Tony/feature-extraction.ipynb', 'repo_name': 'DataKnightsAI/Sleep-Deprivation-Classification-using-MRI-Scans', 'size': '3923'}