<commit_before>"""

Pre-process DICOM files in a study folder

"""

import logging, os
import StringIO

from datetime import datetime, timedelta
from functools import partial
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.operators import SpmOperator
from airflow import configuration

from util import dicom_import
from util import nifti_import


# constants

DAG_NAME = 'pre_process_dicom'

pipelines_path = str(configuration.get('mri', 'PIPELINES_PATH'))
protocols_file = str(configuration.get('mri', 'PROTOCOLS_FILE'))
dicom_to_nifti_local_output_folder = str(configuration.get('mri', 'NIFTI_LOCAL_OUTPUT_FOLDER'))
dicom_to_nifti_server_output_folder = str(configuration.get('mri', 'NIFTI_SERVER_OUTPUT_FOLDER'))
dicom_to_nifti_pipeline_path = pipelines_path + '/Nifti_Conversion_Pipeline'
neuro_morphometric_atlas_local_output_folder = str(configuration.get('mri', 'NEURO_MORPHOMETRIC_ATLAS_LOCAL_OUTPUT_FOLDER'))
neuro_morphometric_atlas_server_output_folder = str(configuration.get('mri', 'NEURO_MORPHOMETRIC_ATLAS_SERVER_OUTPUT_FOLDER'))
neuro_morphometric_atlas_pipeline_path = pipelines_path + '/NeuroMorphometric_Pipeline/NeuroMorphometric_tbx/label'
mpm_maps_local_output_folder = str(configuration.get('mri', 'MPM_MAPS_LOCAL_OUTPUT_FOLDER'))
mpm_maps_server_output_folder = str(configuration.get('mri', 'MPM_MAPS_SERVER_OUTPUT_FOLDER'))
mpm_maps_pipeline_path = pipelines_path + '/MPMs_Pipeline'
misc_library_path = pipelines_path + '/../Miscellaneous&Others'


# functions

# Extract the information from DICOM files located inside a folder.
# The folder information should be given in the configuration parameter 'folder' of the DAG run
def extract_dicom_info_fn(**kwargs):
    ti = kwargs['task_instance']
    dr = kwargs['dag_run']
    input_data_folder = dr.conf['folder']
    session_id = dr.conf['session_id']

    logging.info('folder %s, session_id %s' % (input_data_folder, session_id))

    (participant_id, scan_date) = dicom_import.visit_info(input_data_folder)
    dicom_import.dicom2db(input_data_folder)

    ti.xcom_push(key='folder', value=input_data_folder)
    ti.xcom_push(key='session_id', value=session_id)
    ti.xcom_push(key='participant_id', value=participant_id)
    ti.xcom_push(key='scan_date', value=scan_date)

    return "ok"

# Conversion pipeline from DICOM to Nifti format.
# It converts all files located in the sub folder 'session_id' of 'folder'
# parent_task should contain XCOM keys 'folder' and 'session_id'
def dicom_to_nifti_pipeline_fn(parent_task, **kwargs):
    engine = kwargs['engine']
    ti = kwargs['task_instance']
    input_data_folder = ti.xcom_pull(key='folder', task_ids=parent_task)
    session_id = ti.xcom_pull(key='session_id', task_ids=parent_task)
    participant_id = ti.xcom_pull(key='participant_id', task_ids=parent_task)
    scan_date = ti.xcom_pull(key='scan_date', task_ids=parent_task)
    logging.info("DICOM to Nifti pipeline: session_id=%s, input_folder=%s" % (session_id, input_data_folder))

    out = StringIO.StringIO()
    err = StringIO.StringIO()
    parent_data_folder = os.path.abspath(input_data_folder + '/..')
    logging.info("Calling DCM2NII_LREN(%s,%s,%s,%s,%s,%s)" % (parent_data_folder, session_id, dicom_to_nifti_local_output_folder, dicom_to_nifti_server_output_folder, protocols_file))
    success = engine.DCM2NII_LREN(
        parent_data_folder,<commit_msg>Fix log<commit_after>"""

Pre-process DICOM files in a study folder

"""

import logging, os
import StringIO

from datetime import datetime, timedelta
from functools import partial
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.operators import SpmOperator
from airflow import configuration

from util import dicom_import
from util import nifti_import


# constants

DAG_NAME = 'pre_process_dicom'

pipelines_path = str(configuration.get('mri', 'PIPELINES_PATH'))
protocols_file = str(configuration.get('mri', 'PROTOCOLS_FILE'))
dicom_to_nifti_local_output_folder = str(configuration.get('mri', 'NIFTI_LOCAL_OUTPUT_FOLDER'))
dicom_to_nifti_server_output_folder = str(configuration.get('mri', 'NIFTI_SERVER_OUTPUT_FOLDER'))
dicom_to_nifti_pipeline_path = pipelines_path + '/Nifti_Conversion_Pipeline'
neuro_morphometric_atlas_local_output_folder = str(configuration.get('mri', 'NEURO_MORPHOMETRIC_ATLAS_LOCAL_OUTPUT_FOLDER'))
neuro_morphometric_atlas_server_output_folder = str(configuration.get('mri', 'NEURO_MORPHOMETRIC_ATLAS_SERVER_OUTPUT_FOLDER'))
neuro_morphometric_atlas_pipeline_path = pipelines_path + '/NeuroMorphometric_Pipeline/NeuroMorphometric_tbx/label'
mpm_maps_local_output_folder = str(configuration.get('mri', 'MPM_MAPS_LOCAL_OUTPUT_FOLDER'))
mpm_maps_server_output_folder = str(configuration.get('mri', 'MPM_MAPS_SERVER_OUTPUT_FOLDER'))
mpm_maps_pipeline_path = pipelines_path + '/MPMs_Pipeline'
misc_library_path = pipelines_path + '/../Miscellaneous&Others'


# functions

# Extract the information from DICOM files located inside a folder.
# The folder information should be given in the configuration parameter 'folder' of the DAG run
def extract_dicom_info_fn(**kwargs):
    ti = kwargs['task_instance']
    dr = kwargs['dag_run']
    input_data_folder = dr.conf['folder']
    session_id = dr.conf['session_id']

    logging.info('folder %s, session_id %s' % (input_data_folder, session_id))

    (participant_id, scan_date) = dicom_import.visit_info(input_data_folder)
    dicom_import.dicom2db(input_data_folder)

    ti.xcom_push(key='folder', value=input_data_folder)
    ti.xcom_push(key='session_id', value=session_id)
    ti.xcom_push(key='participant_id', value=participant_id)
    ti.xcom_push(key='scan_date', value=scan_date)

    return "ok"

# Conversion pipeline from DICOM to Nifti format.
# It converts all files located in the sub folder 'session_id' of 'folder'
# parent_task should contain XCOM keys 'folder' and 'session_id'
def dicom_to_nifti_pipeline_fn(parent_task, **kwargs):
    engine = kwargs['engine']
    ti = kwargs['task_instance']
    input_data_folder = ti.xcom_pull(key='folder', task_ids=parent_task)
    session_id = ti.xcom_pull(key='session_id', task_ids=parent_task)
    participant_id = ti.xcom_pull(key='participant_id', task_ids=parent_task)
    scan_date = ti.xcom_pull(key='scan_date', task_ids=parent_task)
    logging.info("DICOM to Nifti pipeline: session_id=%s, input_folder=%s" % (session_id, input_data_folder))

    out = StringIO.StringIO()
    err = StringIO.StringIO()
    parent_data_folder = os.path.abspath(input_data_folder + '/..')
    logging.info("Calling DCM2NII_LREN(%s,%s,%s,%s,%s)" % (parent_data_folder, session_id, dicom_to_nifti_local_output_folder, dicom_to_nifti_server_output_folder, protocols_file))
    success = engine.DCM2NII_LREN(
        parent_data_folder,
{'fraction_non_alphanumeric': 0.10194814706313168, 'fraction_numerical': 0.0008788633367511352, 'mean_word_length': 7.05188679245283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2184393', 'n_tokens_mistral': 2427, 'n_tokens_neox': 2393, 'n_words': 422}