<jupyter_start><jupyter_text>Saliva Data Processing  Setup and Helper Functions<jupyter_code>import json
import re
from pathlib import Path

import pandas as pd
import numpy as np
import pingouin as pg

import matplotlib.pyplot as plt
import seaborn as sns

import biopsykit as bp

from cft_analysis.datasets import CftDatasetRaw

%load_ext autoreload
%autoreload 2
%matplotlib widget
plt.close("all")

palette = bp.colors.fau_palette
sns.set_theme(context="notebook", style="ticks", palette=palette)

plt.rcParams["figure.figsize"] = (10, 5)
plt.rcParams["pdf.fonttype"] = 42
plt.rcParams["mathtext.default"] = "regular"

palette<jupyter_output><empty_output><jupyter_text>Data Import<jupyter_code># build path to data folder
config_dict = json.load(Path("../../config.json").open(encoding="utf-8"))
base_path = Path("..").joinpath(config_dict["base_path"])
dataset = CftDatasetRaw(base_path)
dataset
data = dataset.cortisol
data.head()
# path to export processed saliva data in the Data repository
saliva_path_export = base_path.joinpath("saliva/processed")
# path to export processed saliva data in the Analysis repository
saliva_path_export_analysis = Path("../../data/saliva")

bp.utils.file_handling.mkdirs([saliva_path_export, saliva_path_export_analysis])<jupyter_output><empty_output><jupyter_text>Data Processing<jupyter_code>sample_times = [-30, -1, 30, 40, 50, 60, 70]
auc = bp.saliva.auc(data, remove_s0=True, sample_times=sample_times, compute_auc_post=True)
max_inc = bp.saliva.max_increase(data, remove_s0=True)
max_inc = max_inc.join(bp.saliva.max_increase(data, remove_s0=True, percent=True))
slope = bp.saliva.slope(data, sample_labels=["S1", "S4"], sample_times=sample_times)
cort_features = pd.concat([auc, max_inc, slope], axis=1)
cort_features = bp.saliva.utils.saliva_feature_wide_to_long(cort_features, saliva_type="cortisol")
cort_features.head()<jupyter_output><empty_output><jupyter_text>Export<jupyter_code>data.to_csv(saliva_path_export.joinpath("cortisol_samples.csv"))
data.to_csv(saliva_path_export_analysis.joinpath("cortisol_samples.csv"))
cort_features.to_csv(saliva_path_export.joinpath("cortisol_features.csv"))
cort_features.to_csv(saliva_path_export_analysis.joinpath("cortisol_features.csv"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12152010605391074, 'fraction_numerical': 0.011489173663278833, 'mean_word_length': 8.552742616033756, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '367330', 'license': 'MIT', 'n_tokens_mistral': 857, 'n_tokens_neox': 807, 'n_words': 141, 'path': 'experiments/2022_scientific_reports/notebooks/data_processing/Saliva_Processing.ipynb', 'repo_name': 'mad-lab-fau/cft-analysis', 'size': '2263'}