<jupyter_start><jupyter_text>Evaluate completion of fMRIPrep on datasets<jupyter_code>import os.path as op
from glob import glob

import pandas as pd
# Constants
project_dir = "/home/data/nbc/misc-projects/Salo_PowerReplication/"

dsets = {
    "dset-camcan": [
        "{sub}.html",
        "{sub}/func/{sub}_task-movie_echo-1_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-movie_echo-2_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-movie_echo-3_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-movie_echo-4_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-movie_echo-5_space-scanner_desc-partialPreproc_bold.nii.gz",
    ],
    "dset-cambridge": [
        "{sub}.html",
        "{sub}/func/{sub}_task-rest_echo-1_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-rest_echo-2_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-rest_echo-3_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-rest_echo-4_space-scanner_desc-partialPreproc_bold.nii.gz",
    ],
    "dset-dupre": [
        "{sub}.html",
        "{sub}/func/{sub}_task-rest_run-1_echo-1_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-rest_run-1_echo-2_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-rest_run-1_echo-3_space-scanner_desc-partialPreproc_bold.nii.gz",
    ],
    "dset-dalenberg": [
        "{sub}.html",
        "{sub}/func/{sub}_task-images_echo-1_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-images_echo-2_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-images_echo-3_space-scanner_desc-partialPreproc_bold.nii.gz",
    ],
    "dset-cohen": [
        "{sub}.html",
        "{sub}/func/{sub}_task-bilateralfingertapping_echo-1_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-bilateralfingertapping_echo-2_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-bilateralfingertapping_echo-3_space-scanner_desc-partialPreproc_bold.nii.gz",
        "{sub}/func/{sub}_task-bilateralfingertapping_echo-4_space-scanner_desc-partialPreproc_bold.nii.gz",
    ],
}
# Find failed subjects
for dset, target_files in dsets.items():
    print(dset)
    dset_dir = op.join(project_dir, dset)
    deriv_dir = op.join(dset_dir, "derivatives/fmriprep/")
    if not op.isdir(deriv_dir):
        print("\tDataset not yet processed.")
        continue

    participants_file = op.join(dset_dir, "participants.tsv")
    participants_df = pd.read_table(participants_file)
    subject_list = participants_df["participant_id"].tolist()
    failed_subjects = []
    for sub in subject_list:
        tfs = [op.join(deriv_dir, temp.format(sub=sub)) for temp in target_files]
        if not all(op.isfile(tf) for tf in tfs):
            failed_subjects.append(sub)

    if failed_subjects:
        print("\t" + "\n\t".join(failed_subjects))
    
    if dset == "dset-camcan":
        camcan_failed = failed_subjects
# Find the missing files
for dset, target_files in dsets.items():
    print(dset)
    dset_dir = op.join(project_dir, dset)
    deriv_dir = op.join(dset_dir, "derivatives/fmriprep/")
    if not op.isdir(deriv_dir):
        print("\tDataset not yet processed.")
        continue

    participants_file = op.join(dset_dir, "participants.tsv")
    participants_df = pd.read_table(participants_file)
    subject_list = participants_df["participant_id"].tolist()
    failed_subjects = []
    for sub in subject_list:
        tfs = [op.join(deriv_dir, temp.format(sub=sub)) for temp in target_files]
        for tf in tfs:
            if not op.isfile(tf):
                print("\t{}".format(op.basename(tf)))<jupyter_output>dset-camcan
	sub-CC221585.html
	sub-CC221585_task-movie_echo-1_space-scanner_desc-partialPreproc_bold.nii.gz
	sub-CC221585_task-movie_echo-2_space-scanner_desc-partialPreproc_bold.nii.gz
	sub-CC221585_task-movie_echo-3_space-scanner_desc-partialPreproc_bold.nii.gz
	sub-CC221585_task-movie_echo-4_space-scanner_desc-partialPreproc_bold.nii.gz
	sub-CC221585_task-movie_echo-5_space-scanner_desc-partialPreproc_bold.nii.gz
dset-cambridge
dset-dupre
dset-dalenberg
dset-cohen<jupyter_text>Clean up the working directory for CamCAN<jupyter_code>raise Exception()
participants_file = op.join(project_dir, "dset-camcan", "participants.tsv")
participants_df = pd.read_table(participants_file)
len(camcan_failed)
subs = participants_df["participant_id"].tolist()
camcan_successful = sorted(list(set(subs) - set(camcan_failed)))
len(camcan_successful)
idx = [str(subs.index(f) + 1) for f in camcan_failed]
print(",".join(idx))
from shutil import rmtree

wf_dir = "/scratch/nbc/tsalo006/Salo_PowerReplication/dset-camcan/fmriprep-20.2.1/fmriprep_wf"
for sub in camcan_successful:
    sub_name = sub.split("-")[1]
    sub_dir = f"single_subject_{sub_name}_wf"
    sub_wf_dir = op.join(wf_dir, sub_dir)
    if op.isdir(sub_wf_dir):
        rmtree(sub_wf_dir)
    else:
        print(f"Not removing {sub_name}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13673900518532744, 'fraction_numerical': 0.013827539850201652, 'mean_word_length': 4.051406401551891, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '445607', 'license': 'Apache-2.0', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1976, 'n_words': 222, 'path': 'processing/02a_check_fmriprep_completion.ipynb', 'repo_name': 'NBCLab/power-replication', 'size': '5207'}