<commit_before>"""Preprocessing workflow definition.

See the docstring of create_preprocessing_workflow() for information
about the workflow itself. This module contains the main setup
function and assorted supporting functions for preprocessing.

"""
import os
import numpy as np
import scipy as sp
import pandas as pd
import nibabel as nib
import matplotlib as mpl
import matplotlib.pyplot as plt

import moss
import seaborn

from nipype import fsl
from nipype import freesurfer as fs
from nipype import (Node, MapNode, Workflow,
                    IdentityInterface, Function)
from nipype.workflows.fmri.fsl import create_susan_smooth

# For nipype Function interfaces
imports = ["import os",
           "import numpy as np",
           "import scipy as sp",
           "import pandas as pd",
           "import nibabel as nib",
           "import matplotlib as mpl",
           "import matplotlib.pyplot as plt",
           "import moss",
           "import seaborn"]


def create_preprocessing_workflow(name="preproc",
                                  temporal_interp=True,
                                  frames_to_toss=6,
                                  interleaved=True,
                                  slice_order="up",
                                  TR=2,
                                  intensity_threshold=3,
                                  motion_threshold=1,
                                  smooth_fwhm=6,
                                  highpass_sigma=32,
                                  partial_brain=False):
    """Return a Nipype workflow for fMRI preprocessing.

    This mostly follows the preprocessing in FSL, although some
    of the processing has been moved into pure Python.

    Parameters
    ----------
    name : string
        workflow object name
    temporal_interp : bool
        whether to perform slice-time correction
    frames_to_toss : int
        number of initial frames to remove
    interleaved : bool
        whether slice acquisition is interleaved/alternating
    slice_order : "up" | "down"
        direction of slice acquisition
    TR : float
        repetition time of the sequence
    intensity_threshold : float
        z-score threshold for whole-brain intensity artifacts
    motion_thresold : float
        mm threshold for scan-to-scan motion artifacts
    smooth_fwhm : float
        mm smoothing kernel for spatial smoothing
    highpass_sigma : float
        sigma (in s) of high-pass smoothing kernal
    partial_brain : bool
        protocol is partial brain/hi-res

    """
    preproc = Workflow(name)

    # Define the inputs for the preprocessing workflow
    in_fields = ["timeseries", "subject_id"]

    if partial_brain:
        in_fields.append("whole_brain_epi")

    inputnode = Node(IdentityInterface(in_fields), "inputs")

    # Remove equilibrium frames and convert to float
    prepare = MapNode(Function(["in_file", "frames_to_toss"],
                               ["out_file"],
                               prep_timeseries,
                               imports),
                      "in_file",
                      "prep_timeseries")
    prepare.inputs.frames_to_toss = frames_to_toss

    # Motion and slice time correct
    realign = create_realignment_workflow()

    # Run a conservative skull strip and get a brain mask
    skullstrip = create_skullstrip_workflow()

    # Automatically detect motion and intensity outliers
    artifacts = MapNode(Function(["timeseries",
                                  "mask_file",
                                  "motion_file",
                                  "intensity_thresh",
                                  "motion_thresh"],
                                 ["artifact_report"],
                                 detect_artifacts,
                                 imports),
                        ["timeseries", "mask_file", "motion_file"],
                        "artifacts")
    artifacts.inputs.intensity_thresh = intensity_threshold
    artifacts.inputs.motion_thresh = motion_threshold

    # Estimate a registration from funtional to anatomical space
    coregister = create_bbregister_workflow()

    # Smooth intelligently in the volume
    susan = create_susan_smooth()
    susan.inputs.inputnode.fwhm = smooth_fwhm

    # Scale and filter the timeseries
    filter_smooth = create_filtering_workflow("filter_smooth",
                                              highpass_sigma,
                                              "smoothed_timeseries")

    filter_rough = create_filtering_workflow("filter_rough",
                                              highpass_sigma,
                                              "unsmoothed_timeseries")

    preproc.connect([
        (inputnode, prepare,
            [("timeseries", "in_file")]),
        (prepare, realign,
            [("out_file", "inputs.timeseries")]),
        (realign, skullstrip,
            [("outputs.timeseries", "inputs.timeseries")]),
        (realign, artifacts,
            [("outputs.motion_file", "motion_file")]),
        (skullstrip, artifacts,
            [("outputs.timeseries", "timeseries"),
             ("outputs.mask_file", "mask_file")]),
        (skullstrip, coregister,
            [("outputs.mean_file", "inputs.source_file")]),
        (inputnode, coregister,
            [("subject_id", "inputs.subject_id")]),
        (skullstrip, susan,
            [("outputs.mask_file", "inputnode.mask_file"),
             ("outputs.timeseries", "inputnode.in_files")]),
        (susan, filter_smooth,
            [("outputnode.smoothed_files", "inputs.timeseries")]),
        (skullstrip, filter_smooth,
            [("outputs.mask_file", "inputs.mask_file")]),
        (skullstrip, filter_rough,
            [("outputs.timeseries", "inputs.timeseries")]),
        (skullstrip, filter_rough,
            [("outputs.mask_file", "inputs.mask_file")]),
        ])

    # Define the outputs of the top-level workflow
    output_fields = ["smoothed_timeseries",
                     "unsmoothed_timeseries",
                     "example_func",
                     "mean_func",
                     "functional_mask",
                     "realign_report",
                     "mask_report",
                     "artifact_report",
                     "flirt_affine",
                     "tkreg_affine",
                     "coreg_report"]

    outputnode = Node(IdentityInterface(output_fields), "outputs")

    preproc.connect([
        (realign, outputnode,
            [("outputs.example_func", "example_func"),
             ("outputs.report", "realign_report")]),
        (skullstrip, outputnode,
            [("outputs.mean_file", "mean_func"),
             ("outputs.mask_file", "functional_mask"),
             ("outputs.report", "mask_report")]),
        (artifacts, outputnode,
            [("artifact_report", "artifact_report")]),<commit_msg>Moving artifact detection to after filtering<commit_after>"""Preprocessing workflow definition.

See the docstring of create_preprocessing_workflow() for information
about the workflow itself. This module contains the main setup
function and assorted supporting functions for preprocessing.

"""
import os
import numpy as np
import scipy as sp
import pandas as pd
import nibabel as nib
import matplotlib as mpl
import matplotlib.pyplot as plt

import moss
import seaborn

from nipype import fsl
from nipype import freesurfer as fs
from nipype import (Node, MapNode, Workflow,
                    IdentityInterface, Function)
from nipype.workflows.fmri.fsl import create_susan_smooth

# For nipype Function interfaces
imports = ["import os",
           "import numpy as np",
           "import scipy as sp",
           "import pandas as pd",
           "import nibabel as nib",
           "import matplotlib as mpl",
           "import matplotlib.pyplot as plt",
           "import moss",
           "import seaborn"]


def create_preprocessing_workflow(name="preproc",
                                  temporal_interp=True,
                                  frames_to_toss=6,
                                  interleaved=True,
                                  slice_order="up",
                                  TR=2,
                                  intensity_threshold=3,
                                  motion_threshold=1,
                                  smooth_fwhm=6,
                                  highpass_sigma=32,
                                  partial_brain=False):
    """Return a Nipype workflow for fMRI preprocessing.

    This mostly follows the preprocessing in FSL, although some
    of the processing has been moved into pure Python.

    Parameters
    ----------
    name : string
        workflow object name
    temporal_interp : bool
        whether to perform slice-time correction
    frames_to_toss : int
        number of initial frames to remove
    interleaved : bool
        whether slice acquisition is interleaved/alternating
    slice_order : "up" | "down"
        direction of slice acquisition
    TR : float
        repetition time of the sequence
    intensity_threshold : float
        z-score threshold for whole-brain intensity artifacts
    motion_thresold : float
        mm threshold for scan-to-scan motion artifacts
    smooth_fwhm : float
        mm smoothing kernel for spatial smoothing
    highpass_sigma : float
        sigma (in s) of high-pass smoothing kernal
    partial_brain : bool
        protocol is partial brain/hi-res

    """
    preproc = Workflow(name)

    # Define the inputs for the preprocessing workflow
    in_fields = ["timeseries", "subject_id"]

    if partial_brain:
        in_fields.append("whole_brain_epi")

    inputnode = Node(IdentityInterface(in_fields), "inputs")

    # Remove equilibrium frames and convert to float
    prepare = MapNode(Function(["in_file", "frames_to_toss"],
                               ["out_file"],
                               prep_timeseries,
                               imports),
                      "in_file",
                      "prep_timeseries")
    prepare.inputs.frames_to_toss = frames_to_toss

    # Motion and slice time correct
    realign = create_realignment_workflow()

    # Run a conservative skull strip and get a brain mask
    skullstrip = create_skullstrip_workflow()

    # Estimate a registration from funtional to anatomical space
    coregister = create_bbregister_workflow()

    # Smooth intelligently in the volume
    susan = create_susan_smooth()
    susan.inputs.inputnode.fwhm = smooth_fwhm

    # Scale and filter the timeseries
    filter_smooth = create_filtering_workflow("filter_smooth",
                                              highpass_sigma,
                                              "smoothed_timeseries")

    filter_rough = create_filtering_workflow("filter_rough",
                                              highpass_sigma,
                                              "unsmoothed_timeseries")

    # Automatically detect motion and intensity outliers
    artifacts = MapNode(Function(["timeseries",
                                  "mask_file",
                                  "motion_file",
                                  "intensity_thresh",
                                  "motion_thresh"],
                                 ["artifact_report"],
                                 detect_artifacts,
                                 imports),
                        ["timeseries", "mask_file", "motion_file"],
                        "artifacts")
    artifacts.inputs.intensity_thresh = intensity_threshold
    artifacts.inputs.motion_thresh = motion_threshold

    preproc.connect([
        (inputnode, prepare,
            [("timeseries", "in_file")]),
        (prepare, realign,
            [("out_file", "inputs.timeseries")]),
        (realign, skullstrip,
            [("outputs.timeseries", "inputs.timeseries")]),
        (realign, artifacts,
            [("outputs.motion_file", "motion_file")]),
        (skullstrip, artifacts,
            [("outputs.mask_file", "mask_file")]),
        (skullstrip, coregister,
            [("outputs.mean_file", "inputs.source_file")]),
        (inputnode, coregister,
            [("subject_id", "inputs.subject_id")]),
        (skullstrip, susan,
            [("outputs.mask_file", "inputnode.mask_file"),
             ("outputs.timeseries", "inputnode.in_files")]),
        (susan, filter_smooth,
            [("outputnode.smoothed_files", "inputs.timeseries")]),
        (skullstrip, filter_smooth,
            [("outputs.mask_file", "inputs.mask_file")]),
        (skullstrip, filter_rough,
            [("outputs.timeseries", "inputs.timeseries")]),
        (skullstrip, filter_rough,
            [("outputs.mask_file", "inputs.mask_file")]),
        (filter_rough, artifacts,
            [("outputs.timeseries", "timeseries")]),
        ])

    # Define the outputs of the top-level workflow
    output_fields = ["smoothed_timeseries",
                     "unsmoothed_timeseries",
                     "example_func",
                     "mean_func",
                     "functional_mask",
                     "realign_report",
                     "mask_report",
                     "artifact_report",
                     "flirt_affine",
                     "tkreg_affine",
                     "coreg_report"]

    outputnode = Node(IdentityInterface(output_fields), "outputs")

    preproc.connect([
        (realign, outputnode,
            [("outputs.example_func", "example_func"),
             ("outputs.report", "realign_report")]),
        (skullstrip, outputnode,
            [("outputs.mean_file", "mean_func"),
             ("outputs.mask_file", "functional_mask"),
             ("outputs.report", "mask_report")]),
        (artifacts, outputnode,
            [("artifact_report", "artifact_report")]),
{'fraction_non_alphanumeric': 0.08706306306306306, 'fraction_numerical': 0.001009009009009009, 'mean_word_length': 1.7824343292560658, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595260', 'n_tokens_mistral': 3653, 'n_tokens_neox': 3426, 'n_words': 876}