<jupyter_start><jupyter_text>__README:__Refer to paper:- <NAME>, <NAME>, <NAME>, et al. Detecting Alzheimer's Disease from Speech Using Neural Networks with Bottleneck Features and Data Augmentation. ICASSP 2021.This notebook repeats 10-fold cross-validation for 10 times and saves the split result.The split result should be generated before train any model.<jupyter_code># from IPython.core.display import display, HTML
# display(HTML("<style>.container { width:80% !important; }</style>"))

import os
import json
import numpy as np
import pandas as pd

%matplotlib inline
%config InlineBackend.figure_format = 'svg'

%load_ext autoreload
%autoreload 2

import matplotlib.pyplot as plt

import sys
sys.path.append('..')
from ad_detection.mlcode.util.helper_functions import *<jupyter_output><empty_output><jupyter_text>Without Augmentation  Prepare Features<jupyter_code>from ad_detection.mlcode.merge_feature import merge_all, preprocess_steps, remove_sample_in_blacklist

proj_root_path = '../ws_en/'

# ['acoustic_CPE16.csv', 'acoustic_CPE16_lsa.csv',
# 'acoustic_IS09.csv',  'acoustic_IS09_lsa.csv',
# 'acoustic_IS10.csv',  'acoustic_IS10_lsa.csv',
# 'acoustic_IS11.csv',  'acoustic_IS11_lsa.csv', 
# 'acoustic_IS12.csv', , 'acoustic_IS12_lsa.csv',
# 'acoustic_egemaps.csv',  'acoustic_egemaps_lsa.csv',
# 'duration.csv', 'egemaps.csv', 'merged.csv', 'temp_data_after_FE.csv']


# 剔除掉不要的数据
csv_label_path = proj_root_path + 'label/summary.csv'
csv_label_selected_path = proj_root_path + 'label/summary_selected.csv'
blacklist_path = proj_root_path + 'label/blacklist.csv'  # None
remove_sample_in_blacklist(csv_label_path, blacklist_path, csv_label_selected_path)

# 合并不同的特征集
egemaps_fp = proj_root_path + 'fusion/acoustic_CPE16.csv'
allfeature_out_fp = proj_root_path + 'fusion/merged_CPE16.csv'

csv_label_path = csv_label_selected_path

merge_all([csv_label_path, egemaps_fp], 
          allfeature_out_fp,
          [None, 'CPE16_'])

# 简单数据处理
preprocess_steps(allfeature_out_fp, allfeature_out_fp)
from ad_detection.mlcode.data_manager import MLDataSet

CLASS_CHOOSE=['CTRL','AD']
SEEDS = list(range(2008, 2018))

file_path = proj_root_path + 'fusion/merged_CPE16.csv'
fe_file_path = proj_root_path + 'fusion/data_after_FE_CPE16.csv'


ad_dataset = MLDataSet(file_path)
ad_dataset.feature_engineering(class_col_name='label', class_namelist=CLASS_CHOOSE)
# ad_datasets.df = ad_datasets.drop_nan_row(ad_datasets.df, 'mmse')
ad_dataset.fill_nan_value_mean(ad_dataset.df, ['age'])

ad_dataset.write_FE_df(fe_file_path)


print()
par_id_uni = ad_dataset.df.participant_id.unique()
print('Unique %d\n' % len(par_id_uni))  # , par_id_uni
ad_dataset.df.iloc[:, 0:16].describe()
# duration egemaps linguistic score demographics    doctor all propose select test
feature_group='^CPE16_*'
# PPL_USAGE = 'origin' # None  # ['both', 'origin', 'perp']  None means auto.

ad_dataset.feature_filter(feature_regex=feature_group)<jupyter_output><empty_output><jupyter_text>Dataset Split<jupyter_code>from ad_detection.mlcode.data_splitter import KFoldSplitter

# 重新划分数据
# 从这里开始 df里的数据顺序不能改变，否则会对不上号
k_splitter = KFoldSplitter(sampling_method=None,
                           split_file_dir='../ws_en/list/split/',
                           result_file_dir='../ws_en/list/result/')  #
k_splitter.clean()
k_splitter.split(ad_dataset.df, seeds=SEEDS)
# 3.4GHz 4 process 提一次 约 6min
# !./extract_perplexity.py
# !../code/extract_perplexity_parallel.py --process_num 20<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11471178663607685, 'fraction_numerical': 0.01720676799541153, 'mean_word_length': 6.176954732510288, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '604217', 'license': 'MIT', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1323, 'n_words': 221, 'path': 'notebook/ML7-DataPrepareAndSplit.ipynb', 'repo_name': 'hellolzc/AD_detection_open', 'size': '3485'}