<commit_before>#!/usr/bin/env bash

# set up environment variables for other libraries
export FREESURFER_HOME=/home/sfp_user/freesurfer
export PATH=$FREESURFER_HOME/bin:$PATH

export PATH=/home/sfp_user/matlab/bin:$PATH

export FSLOUTPUTTYPE=NIFTI_GZ
export FSLDIR=/home/sfp_user/fsl
export PATH=$FSLDIR/bin:$PATH

if [ -f /home/sfp_user/spatial-frequency-preferences/config.json ]; then
    cp /home/sfp_user/spatial-frequency-preferences/config.json /home/sfp_user/sfp_config.json
    sed -i 's|"DATA_DIR":.*|"DATA_DIR": "/home/sfp_user/sfp_data"|g' /home/sfp_user/sfp_config.json
    sed -i 's|"MRI_TOOLS":.*|"MRI_TOOLS": "/home/sfp_user/MRI_tools"|g' /home/sfp_user/sfp_config.json
    sed -i 's|"GLMDENOISE_PATH":.*|"GLMDENOISE_PATH": "/home/sfp_user/GLMdenoise"|g' /home/sfp_user/sfp_config.json
    sed -i 's|"VISTASOFT_PATH":.*|"VISTASOFT_PATH": "/home/sfp_user/vistasoft"|g' /home/sfp_user/sfp_config.json
    sed -i 's|"WORKING_DIR":.*|"WORKING_DIR": "/home/sfp_user/sfp_data/preproc_tmp_dir"|g' /home/sfp_user/sfp_config.json
    mkdir -p /home/sfp_user/sfp_data/preproc_tmp_dir
    echo "this is a temporary directory created for preprocessing, and can be deleted" > /home/sfp_user/sfp_data/preproc_tmp_dir/delete_me.txt
fi<commit_msg>corrects singularity_env.sh for json<commit_after>#!/usr/bin/env bash

# set up environment variables for other libraries
export FREESURFER_HOME=/home/sfp_user/freesurfer
export PATH=$FREESURFER_HOME/bin:$PATH

export PATH=/home/sfp_user/matlab/bin:$PATH

export FSLOUTPUTTYPE=NIFTI_GZ
export FSLDIR=/home/sfp_user/fsl
export PATH=$FSLDIR/bin:$PATH

if [ -f /home/sfp_user/spatial-frequency-preferences/config.json ]; then
    cp /home/sfp_user/spatial-frequency-preferences/config.json /home/sfp_user/sfp_config.json
    sed -i 's|"DATA_DIR":.*|"DATA_DIR": "/home/sfp_user/sfp_data",|g' /home/sfp_user/sfp_config.json
    sed -i 's|"MRI_TOOLS":.*|"MRI_TOOLS": "/home/sfp_user/MRI_tools",|g' /home/sfp_user/sfp_config.json
    sed -i 's|"GLMDENOISE_PATH":.*|"GLMDENOISE_PATH": "/home/sfp_user/GLMdenoise",|g' /home/sfp_user/sfp_config.json
    sed -i 's|"VISTASOFT_PATH":.*|"VISTASOFT_PATH": "/home/sfp_user/vistasoft",|g' /home/sfp_user/sfp_config.json
    sed -i 's|"WORKING_DIR":.*|"WORKING_DIR": "/home/sfp_user/sfp_data/preproc_tmp_dir",|g' /home/sfp_user/sfp_config.json
    mkdir -p /home/sfp_user/sfp_data/preproc_tmp_dir
    echo "this is a temporary directory created for preprocessing, and can be deleted" > /home/sfp_user/sfp_data/preproc_tmp_dir/delete_me.txt
fi
{'fraction_non_alphanumeric': 0.1514670896114195, 'fraction_numerical': 0.0, 'mean_word_length': 5.199017199017199, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '722505', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1057, 'n_words': 144}