<commit_before>#!/bin/bash

#==============================================================================
# Combine stats measures of surface parcellations and segmentations for
# NSPN MPM and DTI data for all subjects
# Created by <NAME>
# Contact <EMAIL>
#==============================================================================

#==============================================================================
# USAGE: NSPN_Report_ROIstats_Allsubs.sh <data_dir>
#==============================================================================
function usage {

    echo "USAGE: NSPN_Report_ROIstats_Allsubs.sh <data_dir>"
    echo "Note that data dir expects to find SUB_DATA within it"
    echo "and then the standard NSPN directory structure"
    echo ""
    echo "DESCRIPTION: This code looks for the output of NSPN_ExtractRois.sh"
    echo "in each subject's directory and then combines that information together"
    echo "in the FS_ROIS folder within DATA_DIR"
    exit
}

#=============================================================================
# READ IN COMMAND LINE ARGUMENTS
#=============================================================================

data_dir=$1

if [[ ! -d ${data_dir} ]]; then
    usage
fi

#=============================================================================
# GET STARTED
#=============================================================================

mkdir -p ${data_dir}/FS_ROIS/

#=============================================================================
# SEGMENTATIONS
#=============================================================================
# Loop through the various segmentations
for seg in aseg wmparc lobesStrict; do

    for measure in R1 MT R2s A FA MD MO L1 L23; do

        # Find all the individual stats files for that segmentation
        inputs=(`ls -d ${data_dir}/SUB_DATA/*/SURFER/*/stats/${measure}_${seg}.stats 2> /dev/null `)

        if [[ ${#inputs[@]} -gt 0 ]]; then

            #===== NSPN_ID AND OCC VALUES ====================================
            # We need to edit the first two columns so they're nice and easily
            # readable with the nspn_ids etc
            echo "nspn_id,occ" > ${data_dir}/FS_ROIS/nspn_id_col
            for fname in ${inputs[@]}; do
                fname_parts=(`echo "${fname/${data_dir}/}" | tr "/" " "`)
                sub=${fname_parts[1]}
                occ=${fname_parts[3]}

                echo ${sub},${occ} >> ${data_dir}/FS_ROIS/nspn_id_col
            done

            # Write out each statistic
            # This is silly because it loops over volume many times
            # but to be honest, I think the code was looking super messy when
            # I had it being faster. So just be patient and don't worry about
            # speed ;)

            for stat in mean std volume; do
                # Now write out the mean values for the measure
                asegstats2table --inputs ${inputs[@]} \
                                -t ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}_temp.csv \
                                -d comma \
                                --all-segs \
                                --meas ${stat}

                # Now paste the data together
                paste -d , ${data_dir}/FS_ROIS/nspn_id_col \
                            ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}_temp.csv \
                                > ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}.csv

                # And replace all '-' with '_' because statsmodels in python
                # likes that more :P but only for the first line
                sed -i "1 s/-/_/g" ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}.csv
                # And replace the : marker
                sed -i "s/://g" ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}.csv

                # Remove the temporary files
                rm ${data_dir}/FS_ROIS/*temp.csv
            done

        # Get rid of the nspn_id_col file ready for the next loop
        rm ${data_dir}/FS_ROIS/nspn_id_col

        else
            echo "    No input files for ${measure}_${seg}!"
        fi
    done
done

#=============================================================================
# PARCELLATIONS
#=============================================================================
# Loop through the various parcellations

subjects=(`ls -d ${data_dir}/SUB_DATA/*/SURFER/*/ 2> /dev/null`)

for parc in aparc 500.aparc lobesStrict HCP Yeo2011_7Networks_N1000; do

    # Start by pulling out the standard measures
    for measure in area volume thickness thicknessstd meancurv gauscurv foldind curvind; do

        for hemi in lh rh; do

            # Combine stats for all subjects for each measure and for each
            # hemisphere separately
            aparcstats2table --hemi ${hemi} \
                                --subjects ${subjects[@]} \
                                --parc ${parc} \
                                --meas ${measure} \
                                -d comma \
                                --skip \
                                -t ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temptemp.csv

            # Drop the first column because it isn't necessary
            cut -d, -f2- ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temptemp.csv \
                    > ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temp.csv

            # But save it for later!
            cut -d, -f1 ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temptemp.csv \
                    > ${data_dir}/FS_ROIS/nspn_id_col
        done

        sed -i "s|${data_dir}/SUB_DATA/||g" ${data_dir}/FS_ROIS/nspn_id_col
        sed -i "s|/SURFER/|,|g" ${data_dir}/FS_ROIS/nspn_id_col
        sed -i "s|/||g" ${data_dir}/FS_ROIS/nspn_id_col<commit_msg>Added economo parcellation to report ROIstats code<commit_after>#!/bin/bash

#==============================================================================
# Combine stats measures of surface parcellations and segmentations for
# NSPN MPM and DTI data for all subjects
# Created by <NAME>
# Contact <EMAIL>
#==============================================================================

#==============================================================================
# USAGE: NSPN_Report_ROIstats_Allsubs.sh <data_dir>
#==============================================================================
function usage {

    echo "USAGE: NSPN_Report_ROIstats_Allsubs.sh <data_dir>"
    echo "Note that data dir expects to find SUB_DATA within it"
    echo "and then the standard NSPN directory structure"
    echo ""
    echo "DESCRIPTION: This code looks for the output of NSPN_ExtractRois.sh"
    echo "in each subject's directory and then combines that information together"
    echo "in the FS_ROIS folder within DATA_DIR"
    exit
}

#=============================================================================
# READ IN COMMAND LINE ARGUMENTS
#=============================================================================

data_dir=$1

if [[ ! -d ${data_dir} ]]; then
    usage
fi

#=============================================================================
# GET STARTED
#=============================================================================

mkdir -p ${data_dir}/FS_ROIS/

#=============================================================================
# SEGMENTATIONS
#=============================================================================
# Loop through the various segmentations
for seg in aseg wmparc lobesStrict; do

    for measure in R1 MT R2s A FA MD MO L1 L23; do

        # Find all the individual stats files for that segmentation
        inputs=(`ls -d ${data_dir}/SUB_DATA/*/SURFER/*/stats/${measure}_${seg}.stats 2> /dev/null `)

        if [[ ${#inputs[@]} -gt 0 ]]; then

            #===== NSPN_ID AND OCC VALUES ====================================
            # We need to edit the first two columns so they're nice and easily
            # readable with the nspn_ids etc
            echo "nspn_id,occ" > ${data_dir}/FS_ROIS/nspn_id_col
            for fname in ${inputs[@]}; do
                fname_parts=(`echo "${fname/${data_dir}/}" | tr "/" " "`)
                sub=${fname_parts[1]}
                occ=${fname_parts[3]}

                echo ${sub},${occ} >> ${data_dir}/FS_ROIS/nspn_id_col
            done

            # Write out each statistic
            # This is silly because it loops over volume many times
            # but to be honest, I think the code was looking super messy when
            # I had it being faster. So just be patient and don't worry about
            # speed ;)

            for stat in mean std volume; do
                # Now write out the mean values for the measure
                asegstats2table --inputs ${inputs[@]} \
                                -t ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}_temp.csv \
                                -d comma \
                                --all-segs \
                                --meas ${stat}

                # Now paste the data together
                paste -d , ${data_dir}/FS_ROIS/nspn_id_col \
                            ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}_temp.csv \
                                > ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}.csv

                # And replace all '-' with '_' because statsmodels in python
                # likes that more :P but only for the first line
                sed -i "1 s/-/_/g" ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}.csv
                # And replace the : marker
                sed -i "s/://g" ${data_dir}/FS_ROIS/SEG_${measure}_${seg}_${stat}.csv

                # Remove the temporary files
                rm ${data_dir}/FS_ROIS/*temp.csv
            done

        # Get rid of the nspn_id_col file ready for the next loop
        rm ${data_dir}/FS_ROIS/nspn_id_col

        else
            echo "    No input files for ${measure}_${seg}!"
        fi
    done
done

#=============================================================================
# PARCELLATIONS
#=============================================================================
# Loop through the various parcellations

subjects=(`ls -d ${data_dir}/SUB_DATA/*/SURFER/*/ 2> /dev/null`)

for parc in aparc 500.aparc lobesStrict HCP Yeo2011_7Networks_N1000 economo; do

    # Start by pulling out the standard measures
    for measure in area volume thickness thicknessstd meancurv gauscurv foldind curvind; do

        for hemi in lh rh; do

            # Combine stats for all subjects for each measure and for each
            # hemisphere separately
            aparcstats2table --hemi ${hemi} \
                                --subjects ${subjects[@]} \
                                --parc ${parc} \
                                --meas ${measure} \
                                -d comma \
                                --skip \
                                -t ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temptemp.csv

            # Drop the first column because it isn't necessary
            cut -d, -f2- ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temptemp.csv \
                    > ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temp.csv

            # But save it for later!
            cut -d, -f1 ${data_dir}/FS_ROIS/PARC_${parc}_${measure}_${hemi}_temptemp.csv \
                    > ${data_dir}/FS_ROIS/nspn_id_col
        done

        sed -i "s|${data_dir}/SUB_DATA/||g" ${data_dir}/FS_ROIS/nspn_id_col
        sed -i "s|/SURFER/|,|g" ${data_dir}/FS_ROIS/nspn_id_col
        sed -i "s|/||g" ${data_dir}/FS_ROIS/nspn_id_col
{'fraction_non_alphanumeric': 0.2570185169383053, 'fraction_numerical': 0.004778564723952556, 'mean_word_length': 2.358166189111748, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4529934', 'n_tokens_mistral': 3350, 'n_tokens_neox': 3068, 'n_words': 862}