<jupyter_start><jupyter_text>ECG Processing & Feature Computation<jupyter_code>import json
import re
from pathlib import Path

import pandas as pd
import numpy as np
import pingouin as pg

import matplotlib.pyplot as plt
import seaborn as sns

from tqdm.auto import tqdm

import biopsykit as bp

from cft_analysis.datasets import CftDatasetRaw
from cft_analysis.feature_extraction.hrv import hrv_continuous_dict
from cft_analysis.datasets.helper import load_subject_data_dicts, load_subject_continuous_hrv_data


%load_ext autoreload
%autoreload 2
%matplotlib widget
plt.close("all")

palette = bp.colors.fau_palette
sns.set_theme(context="notebook", style="ticks", palette=palette)

plt.rcParams["figure.figsize"] = (10, 5)
plt.rcParams["pdf.fonttype"] = 42
plt.rcParams["mathtext.default"] = "regular"

palette<jupyter_output><empty_output><jupyter_text>Data Import<jupyter_code># build path to data folder
config_dict = json.load(Path("../../config.json").open(encoding="utf-8"))
base_path = Path("..").joinpath(config_dict["base_path"])
export_path = Path("../../data")
export_path_ecg = export_path.joinpath("ecg")

bp.utils.file_handling.mkdirs([export_path, export_path_ecg])
# flag indicating whether to ignore already existing processing results and re-process data or not
overwrite_results = False
dataset = CftDatasetRaw(base_path)
dataset<jupyter_output><empty_output><jupyter_text>Process ECG Signal<jupyter_code>for subset in tqdm(dataset.groupby("subject")):
    subject_id = subset.index["subject"][0]

    export_paths = subset.setup_export_paths()
    if not overwrite_results or export_paths["hr_result"].exists():
        continue

    ecg_data = subset.ecg
    ep = bp.signals.ecg.EcgProcessor(data=ecg_data, sampling_rate=subset.sampling_rate)
    ep.ecg_process(title=subject_id)

    dict_hrv_continuous = hrv_continuous_dict(ep)

    paths = subset.setup_export_paths()

    # save HR data and R-Peak data to file
    bp.io.ecg.write_hr_phase_dict(ep.heart_rate, paths["hr_result"])
    bp.io.ecg.write_pandas_dict_excel(ep.rpeaks, paths["rpeaks_result"])
    bp.io.ecg.write_pandas_dict_excel(dict_hrv_continuous, paths["hrv_cont"])
hr_subject_data_dict, rpeaks_subject_data_dict = load_subject_data_dicts(dataset)
hrv_subject_data_dict = load_subject_continuous_hrv_data(dataset)<jupyter_output><empty_output><jupyter_text>Compute HR(V) Parameters over MIST<jupyter_code># get condition list from dataset
condition_list = dataset.condition_list

# HRV parameters of interest
hrv_columns = ["HRV_SDNN", "HRV_RMSSD", "HRV_pNN50", "HRV_pNN20"]<jupyter_output><empty_output><jupyter_text>Create `MIST` Object<jupyter_code>subphases = {"BL": 60, "RP_CFI": 120, "AT": 240, "FB": 0}

structure = {
    "Pre": None,
    "MIST": {
        "MIST1": subphases,
        "MIST2": subphases,
        "MIST3": subphases,
    },
    "Post": None,
}

mist = bp.protocols.MIST(name="CFT", structure=structure)
mist<jupyter_output><empty_output><jupyter_text>Add Heart Rate Data<jupyter_code>mist.add_hr_data(hr_data=hr_subject_data_dict, rpeak_data=rpeaks_subject_data_dict)<jupyter_output><empty_output><jupyter_text>Compute *Ensemble Time-Series*<jupyter_code>mist.compute_hr_ensemble(
    "hr_ensemble", select_phases=True, params={"normalize_to": "Pre", "select_phases": ["MIST1", "MIST2", "MIST3"]}
)<jupyter_output><empty_output><jupyter_text>Compute *Aggregated Heart Rate Results* The following parameters are computed:* *Aggregated Heart Rate Results* are computed for both absolute and relative heart rate (relative heart rate: normalized to `Pre` phase on a subject-level basis)* Relative amount of heart rate above baseline (baseline: `Pre` phase)<jupyter_code>mist.compute_hr_results(
    "hr_mean",
    resample_sec=False,
    normalize_to=False,
    select_phases=True,
    split_into_subphases=True,
    add_conditions=True,
    params={
        "select_phases": ["MIST1", "MIST2", "MIST3"],
        "split_into_subphases": subphases,
        "add_conditions": condition_list,
    },
)

mist.compute_hr_results(
    "hr_mean_normalized",
    resample_sec=False,
    normalize_to=True,
    select_phases=True,
    split_into_subphases=True,
    add_conditions=True,
    params={
        "normalize_to": "Pre",
        "select_phases": ["MIST1", "MIST2", "MIST3"],
        "split_into_subphases": subphases,
        "add_conditions": condition_list,
    },
)

mist.compute_hr_above_baseline(
    "hr_above_bl_glo",
    "Pre",
    select_phases=True,
    split_into_subphases=True,
    add_conditions=True,
    params={
        "select_phases": ["MIST1", "MIST2", "MIST3"],
        "split_into_subphases": subphases,
        "add_conditions": condition_list,
    },
)<jupyter_output><empty_output><jupyter_text>Compute *HRV* Parameter The following HRV parameters are computed:* Time-domain and non-linear HRV parameters are computed over phases and MIST subphases, respectively* Relative amount of HRV parameters above baseline are computed per subphase from continuous HRV data (baseline: `Pre` phase)<jupyter_code># ignore neurokit warnings
from neurokit2 import NeuroKitWarning
import warnings

warnings.filterwarnings("ignore", category=NeuroKitWarning)

mist.compute_hrv_results(
    "hrv_phases",
    add_conditions=True,
    params={"add_conditions": condition_list},
    hrv_params={"hrv_types": ["hrv_time", "hrv_nonlinear"]},
)

mist.compute_hrv_results(
    "hrv_subphases",
    split_into_subphases=True,
    select_phases=True,
    add_conditions=True,
    params={
        "split_into_subphases": subphases,
        "select_phases": ["MIST1", "MIST2", "MIST3"],
        "add_conditions": condition_list,
    },
    hrv_params={"hrv_types": ["hrv_time", "hrv_nonlinear"]},
)

mist.compute_hrv_above_baseline(
    "hrv_above_bl_glo",
    "Pre",
    hrv_subject_data_dict,
    select_phases=True,
    split_into_subphases=True,
    add_conditions=True,
    hrv_columns=hrv_columns,
    params={
        "select_phases": ["MIST1", "MIST2", "MIST3"],
        "split_into_subphases": subphases,
        "add_conditions": condition_list,
    },
)<jupyter_output><empty_output><jupyter_text>Compute CFT Parameter<jupyter_code>from cft_analysis.feature_extraction.cft import cft_parameter_per_phase
cft_params = cft_parameter_per_phase(hr_subject_data_dict, dataset.get_subset(condition="CFT").condition_list)
cft_params.head()
from cft_analysis.utils.data_reshaping import (
    reshape_hr_data,
    reshape_hrv_data,
    reshape_time_above_bl_glo,
    reshape_cft_params,
)
hr_data = reshape_hr_data(mist)
hrv_data = reshape_hrv_data(mist, hrv_columns)
above_bl_data = reshape_time_above_bl_glo(mist)
cft_params_data = reshape_cft_params(cft_params, condition_list)
concat_dict = {"HR": hr_data, "Time_BL_Glo": above_bl_data, "HRV": hrv_data, "CFT": cft_params_data}
data_concat = pd.concat(concat_dict, names=["category"])
data_concat = data_concat.reorder_levels(["condition", "subject", "phase", "subphase", "category", "type"]).sort_index()
data_concat<jupyter_output><empty_output><jupyter_text>Export Data<jupyter_code>mist.to_file(export_path.joinpath("mist_cft.json"))
mist.export_hr_ensemble(export_path_ecg)
cft_params.to_csv(export_path_ecg.joinpath("cft_parameter.csv"))
data_concat.to_csv(export_path_ecg.joinpath("cft_hr_features_merged.csv"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11521115211152111, 'fraction_numerical': 0.005876725433921006, 'mean_word_length': 6.05009633911368, 'pattern_counts': {'":': 35, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '607023', 'license': 'MIT', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2619, 'n_words': 447, 'path': 'experiments/2022_scientific_reports/notebooks/data_processing/ECG_Processing_Feature_Computation.ipynb', 'repo_name': 'mad-lab-fau/cft-analysis', 'size': '7317'}