<jupyter_start><jupyter_text>Explore ECG datasetStefan/Yuzhao HengSince Sun. Oct. 31st, 2021Check out data format of publicly available datasets & determine which to use.e.g. If they contain 12-lead recordings, is each beat labeled Setup<jupyter_code>import os
import glob

import h5py
import pandas as pd
import wfdb
from wfdb import processing
from scipy.io import loadmat
from icecream import ic


from util import *


!ls ../../datasets
dsets = list(config('datasets').keys())
ic(dsets)

BASE = os.path.join(PATH_BASE, DIR_DSET)<jupyter_output>[1m[36mCODE-test[m[m            [1m[36mGeorgia-12-Lead[m[m      [1m[36mPTB-Diagnostic[m[m
[1m[36mCSPC-2018[m[m            [1m[36mKIT-20-PVC[m[m           [1m[36mPTB-XL[m[m
[1m[36mCSPC-2018-CinC[m[m       [1m[36mMIT-BIH-MVED[m[m         [1m[36mSt-Petersburg-INCART[m[m
[1m[36mCSPC-2018-Extra-CinC[m[m [1m[36mNingbo-CinC[m[m          [1m[36mStef-Combined[m[m
[1m[36mChapman-Shaoxing[m[m     [1m[36mPSTOV[m[m<jupyter_text>INCARTSt Petersburg INCART 12-lead Arrhythmia Database Is the R-peak accurate, i.e. local maxima/minima?<jupyter_code># dset = config(f'datasets.{dsets[1]}')
# ic(dset['nm'])
# rnm = 'I01'
# fnm = f'{dset["dir_nm"]}/{rnm}'
dnm = 'INCART'
fnm = get_rec_paths(dnm)[0]
fnm = fnm[:fnm.index('.')]
# ic(fnm)
rec = wfdb.rdrecord(fnm)
# rec = get_record_eg(dnm)
ic(rec.p_signal.shape)
ic(rec.fs)

sampto = 2 ** 8
rec = wfdb.rdrecord(fnm, sampto=sampto)
# rec = get_record_eg(dnm, n=sampto)
ann = wfdb.rdann(fnm, extension='atr', sampto=sampto)

sig = rec.p_signal[:, 0]
idxs_rpeak = ann.sample
plot_rpeak(sig, idxs_rpeak, title='Record eg')
idxs_rpeak = refine_rpeak(sig, idxs_rpeak, fqs=rec.fs)
plot_rpeak(sig, idxs_rpeak, title='Record eg, refined peak')<jupyter_output>ic| rec.p_signal.shape: (462600, 12)
ic| rec.fs: 257<jupyter_text>KIT-20-PVCHuman PVC and Paced beats (KIT-20-PVC_Clinical_1906-10-30)BSPM data, not ECG<jupyter_code># fnm = 'KIT-20-PVC/Interventions/InterventionPace1/Subject20_run01.mat'
# mat = loadmat(fnm)
# ic(mat)

fnm = f'{BASE}/KIT-20-PVC/Interventions/PVC/Subject20_run01.mat'
mat = loadmat(fnm)
data = mat['bspm']
ic(type(data), data.shape)

data = data[0][0]
ic(type(data), len(data))

data, unit, beat_type, file_nm = data
ic(data, unit, beat_type, file_nm)
ic(type(data), data.shape)<jupyter_output>ic| type(data): <class 'numpy.ndarray'>, data.shape: (1, 1)
ic| type(data): <class 'numpy.void'>, len(data): 4
ic| data: array([[ 0.        , -0.00313843, -0.0030269 , ..., -0.03043172,
                  -0.03204859, -0.02841264],
                 [ 0.        , -0.00255374, -0.00326898, ...,  0.03951429,
                   0.03700217,  0.03904396],
                 [ 0.        ,  0.00011707,  0.00174486, ...,  0.03171732,
                   0.03228344,  0.03745946],
                 ...,
                 [ 0.        ,  0.0028107 ,  0.0032005 , ...,  0.01739727,
                   0.02797695,  0.03283168],
                 [ 0.        ,  0.00250143,  0.00237055, ...,  0.04354762,
                   0.04959223,  0.05541237],
                 [ 0.        ,  0.0039265 ,  0.00644877, ...,  0.05749454,
                   0.06818049,  0.07494887]])
    unit: array(['mV'], dtype='<U2')
    beat_type: array(['PVC'], dtype='<U3')
    file_nm: array(['Subject20_run01.mat'], dtype='<U19')
ic| type[...]<jupyter_text>PSTOV[Multiple Ventricle Pacing Sites Pat3 (Charles_PSTOV-12-07-29)](https://edgar.sci.utah.edu/human-pacing-site/11-charles_pstov-12-07-29.html)Seems to only contain the QRS complex<jupyter_code>fnm = f'{BASE}/PSTOV/Interventions/interventionLeftVentPace01/2012-07-29Subject_run1'
mat = loadmat(fnm)
ic(mat)
data = mat['bspm']
ic(type(data), data.shape)

data = data[0][0]
ic(type(data), len(data))

data, unit, description = data
ic(data, unit, description)
ic(type(data), data.shape)

plot_single(data[0])<jupyter_output>ic| mat: {'__globals__': [],
          '__header__': b'MATLAB 5.0 MAT-file, Platform: MACI64, Created on: Fri Oct 1'
                        b'6 14:36:16 2015',
          '__version__': '1.0',
          'bspm': array([[(array([[ 0.      ,  0.022696,  0.032321, ...,  0.042838,  0.018738,
                  0.010643],
                [ 0.      ,  0.020598,  0.030319, ...,  0.009359, -0.012449,
                 -0.022405],
                [ 0.      ,  0.017847,  0.025191, ..., -0.009756, -0.033478,
                 -0.042839],
                ...,
                [ 0.      ,  0.003845,  0.008847, ...,  0.009791, -0.005612,
                 -0.009291],
                [ 0.      ,  0.006458,  0.012307, ...,  0.003302, -0.014493,
                 -0.018685],
                [ 0.      ,  0.007566,  0.012633, ..., -0.032618, -0.047524,
                 -0.053156]]), array(['mV'], dtype='<U2'), array(['BSPM from pacing localization on subject3 from the pstov dataset. Ventricle LV, location ID: 1[...]<jupyter_text>PTB-XLLooks like I need to detect R peaks myself<jupyter_code>def explore_singe_lead(dnm_, rnm_, sampto_=2**8):
    dset_ = config(f'{DIR_DSET}.{dnm_}')
    ic(dset_['nm'])

    fnm_ = f'{BASE}/{dset_["dir_nm"]}/{rnm_}'

    rec_ = wfdb.rdrecord(fnm_, sampto=sampto_ * 2**3)
    ic(rec_.p_signal.shape)
    ic(rec_.comments)

    sig_ = rec_.p_signal[:, 0]
    plot_single(sig_)

    rec_ = wfdb.rdrecord(fnm_, sampto=sampto_)
    sig_ = rec_.p_signal[:, 0]

    hard_peaks, soft_peaks = wfdb.processing.find_peaks(sig_)
    ic(hard_peaks.shape, hard_peaks[:20])
    ic(soft_peaks.shape, soft_peaks[:20])
    plot_rpeak(sig_, hard_peaks, title=rnm_)
    plot_rpeak(sig_, hard_peaks, title=rnm_)
    plot_rpeak(sig_, soft_peaks, title=rnm_)

dnm = dsets[2]
ic(dnm)
# R-peak from wfdb doesn't look good
explore_singe_lead(dnm, rnm_='records500/00000/00001_hr')

dir_nm = config(f'{DIR_DSET}.{dnm}.dir_nm')
path = f'{DIR_DSET}/{dir_nm}'
df = pd.read_csv(f'{PATH_BASE}/{path}/ptbxl_database.csv')
df.head(2)<jupyter_output>ic| dnm: 'PTB_XL'
ic| dset_['nm']: 'PTB-XL, a large publicly available electrocardiography dataset'
ic| rec_.p_signal.shape: (2048, 12)
ic| rec_.comments: []<jupyter_text>PTB-Diagnostic<jupyter_code>dnm = dsets[3]
explore_singe_lead(dnm, rnm_='patient001/s0010_re', sampto_=2**10)<jupyter_output>ic| dset_['nm']: 'PTB Diagnostic ECG Database'
ic| rec_.p_signal.shape: (8192, 15)
ic| rec_.comments: ['age: 81',
                    'sex: female',
                    'ECG date: 01/10/1990',
                    'Diagnose:',
                    'Reason for admission: Myocardial infarction',
                    'Acute infarction (localization): infero-latera',
                    'Former infarction (localization): no',
                    'Additional diagnoses: Diabetes mellitus',
                    'Smoker: no',
                    'Number of coronary vessels involved: 1',
                    'Infarction date (acute): 29-Sep-90',
                    'Previous infarction (1) date: n/a',
                    'Previous infarction (2) date: n/a',
                    'Hemodynamics:',
                    'Catheterization date: 16-Oct-90',
                    'Ventriculography: Akinesia inferior wall',
                    'Chest X-ray: Heart size upper limit of norm',
                    'Pe[...]<jupyter_text>CSPC-2018China Physiological Signal Challenge 2018<jupyter_code>fnm = f'{BASE}/CSPC-2018/A0001.mat'
mat = loadmat(fnm)
ic(mat.keys())

data = mat['ECG']
ic(type(data), data.shape, data.dtype)
data = data[0][0]
ic(type(data), data.shape)
sex, age, sig = data
ic(sex, age, sig)
ic(sig.shape)
ic(sig[0, :20])<jupyter_output>ic| mat.keys(): dict_keys(['__header__', '__version__', '__globals__', 'ECG'])
ic| type(data): <class 'numpy.ndarray'>
    data.shape: (1, 1)
    data.dtype: dtype([('sex', 'O'), ('age', 'O'), ('data', 'O')])
ic| type(data): <class 'numpy.void'>, data.shape: ()
ic| sex: array(['Male'], dtype='<U4')
    age: array([[74]], dtype=uint8)
    sig: array([[ 0.0282288 ,  0.0392288 ,  0.0452288 , ...,  0.2582288 ,
                  0.2592288 ,  0.2592288 ],
                [ 0.00672947,  0.01072947,  0.01472947, ...,  0.24772947,
                  0.24872947,  0.24972947],
                [-0.02149933, -0.02849933, -0.03049933, ..., -0.01049933,
                 -0.01049933, -0.00949933],
                ...,
                [-0.11200653, -0.11000653, -0.10800653, ...,  0.19399347,
                  0.19399347,  0.19499347],
                [-0.5959572 , -0.5899572 , -0.5819572 , ...,  0.3070428 ,
                  0.3070428 ,  0.3070428 ],
                [-0.01558507, -0.00658507,  0.0024149[...]<jupyter_text>CSPC-2018But that from CinC, uses the standard `wfdb` format<jupyter_code>dnm = 'CSPC_CinC'
d_dset = config(f'{DIR_DSET}.{dnm}')
rnm = 'A0001'
ic(d_dset['nm'])

fnm = f'{BASE}/{d_dset["dir_nm"]}/{rnm}'
ic(fnm)

sampto = 2**8
rec = wfdb.rdrecord(fnm, sampto=sampto)
sig = rec.p_signal

ic(sig.shape)
ic(rec.d_signal)
ic(rec.record_name)
ic(rec.fs)
ic(rec.sig_name)
ic(rec.comments)

ic(sig[:20, 0])  # Differ by 0.0002288?

# ic(vars(rec))<jupyter_output>ic| d_dset['nm']: 'China Physiological Signal Challenge 2018 - from CinC'
ic| fnm: '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CSPC-2018-CinC/A0001'
ic| sig.shape: (256, 12)
ic| rec.d_signal: None
ic| rec.record_name: 'A0001'
ic| rec.fs: 500
ic| rec.sig_name: ['I', 'II', 'III', 'aVR', 'aVL', 'aVF', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6']
ic| rec.comments: ['Age: 74',
                   'Sex: Male',
                   'Dx: 59118001',
                   'Rx: Unknown',
                   'Hx: Unknown',
                   'Sx: Unknown']
ic| sig[:20, 0]: array([0.028, 0.039, 0.045, 0.049, 0.054, 0.056, 0.058, 0.06 , 0.06 ,
                        0.061, 0.065, 0.069, 0.073, 0.082, 0.087, 0.095, 0.104, 0.113,
                        0.121, 0.13 ])<jupyter_text>What's `Dx` in the comments?Class label for diagnosis - Confirmed by paper<jupyter_code>dir_nm = d_dset['dir_nm']
path = f'{PATH_BASE}/{DIR_DSET}/{dir_nm}'
fmt = f'{path}/{d_dset["rec_fmt"]}'
recs = sorted(glob.iglob(fmt, recursive=True))
ic(recs[:5], len(recs))

n = 256
dxs = [wfdb.rdrecord(fnm[:fnm.index('.')], sampto=1).comments[2] for fnm in recs[:n]]

ic(dxs[:5], len(dxs))
df = pd.DataFrame(dxs)
uniqs = pd.unique(df[0])
ic(list(uniqs), uniqs.size)
ic(df[0].value_counts())<jupyter_output>ic| recs[:5]: ['/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CSPC-2018-CinC/A0001.mat',
               '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CSPC-2018-CinC/A0002.mat',
               '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CSPC-2018-CinC/A0003.mat',
               '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CSPC-2018-CinC/A0004.mat',
               '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CSPC-2018-CinC/A0005.mat']
    len(recs): 6877
ic| dxs[:5]: ['Dx: 59118001',
              'Dx: 426783006',
              'Dx: 164889003',
              'Dx: 164889003',
              'Dx: 164884008']
    len(dxs): 256
ic| list(uniqs): ['Dx: 59118001',
                  'Dx: 426783006',
                  'Dx: 164889003',
                  'Dx: 164884008',
                  'Dx: 429622005',
                  'Dx: 164909002',
                  'Dx: 164931005',
                  'Dx: 270492004',
    [...]<jupyter_text>CSPC-2018 extraAn unused dataset that's not publicly available from [original website](http://2018.icbeb.org/Challenge.html),but available from CinC<jupyter_code>dnm = 'CSPC_Extra_CinC'
dset_ = config(f'{DIR_DSET}.{dnm}')
rnm = 'Q0001'
ic(dset_['nm'])

fnm = f'{BASE}/{dset_["dir_nm"]}/{rnm}'
ic(fnm)

sampto = 2**8
rec = wfdb.rdrecord(fnm, sampto=sampto)
sig = rec.p_signal

ic(sig.shape)
ic(rec.d_signal)
ic(rec.record_name)
ic(rec.fs)
ic(rec.sig_name)
ic(rec.comments)

# ic(vars(rec))<jupyter_output>ic| dset_['nm']: 'China Physiological Signal Challenge 2018, unused/extra - from CinC'
ic| fnm: '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CSPC-2018-Extra-CinC/Q0001'
ic| sig.shape: (256, 12)
ic| rec.d_signal: None
ic| rec.record_name: 'Q0001'
ic| rec.fs: 500
ic| rec.sig_name: ['I', 'II', 'III', 'aVR', 'aVL', 'aVF', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6']
ic| rec.comments: ['Age: 53',
                   'Sex: Male',
                   'Dx: 164867002,427084000',
                   'Rx: Unknown',
                   'Hx: Unknown',
                   'Sx: Unknown']<jupyter_text>Georgia 12-LeadGeorgia 12-lead ECG Challenge (G12EC) Database<jupyter_code>dnm = 'G12EC'
d_dset = config(f'{DIR_DSET}.{dnm}')
rnm = 'E00001'  # Looks like the header file is incorrect - `.mat` need to be processed
ic(d_dset['nm'])

fnm = f'{BASE}/{d_dset["dir_nm"]}/{rnm}'
sampto = 2**8
rec = wfdb.rdrecord(fnm, sampto=sampto)
sig = rec.p_signal
ic(rec)
ic(sig.shape)
# ic(vars(rec))
ic(rec.comments)<jupyter_output>ic| d_dset['nm']: 'Georgia 12-lead ECG Challenge (G12EC) Database'
ic| rec: <wfdb.io.record.Record object at 0x16a107310>
ic| sig.shape: (256, 12)
ic| rec.comments: ['Age: NaN', 'Sex: Female', 'Dx: 426783006', 'Rx: Unknown', 'Hx: Unknown']<jupyter_text>Chapman-Shaoxing<jupyter_code>dnm = 'CHAP_SHAO'
fnm = get_rec_paths(dnm)[0]
df = pd.read_csv(fnm)
ic(fnm)
ic(df.head(5), len(df))

plot_single(df.iloc[:512]['I'], label='Lead I')<jupyter_output>ic| fnm: '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/Chapman-Shaoxing/ECGData/MUSE_20180111_155115_19000.csv'
ic| df.head(5):         I      II    III     aVR     aVL     aVF      V1      V2      V3  \
                0 -214.72 -229.36 -14.64  224.48 -102.48 -122.00  614.88  814.96  912.56   
                1 -200.08 -209.84  -9.76  204.96  -97.60 -112.24  605.12  819.84  927.20   
                2 -190.32 -195.20  -4.88  195.20  -92.72 -102.48  600.24  829.60  956.48   
                3 -165.92 -165.92   0.00  165.92  -82.96  -82.96  590.48  844.24  971.12   
                4 -161.04 -156.16   4.88  161.04  -82.96  -78.08  585.60  844.24  985.76   
                
                       V4      V5      V6  
                0  126.88 -239.12 -507.52  
                1  165.92 -195.20 -463.60  
                2  204.96 -151.28 -429.44  
                3  239.12 -117.12 -400.16  
                4  273.28  -78.08 -370.88  
    len(df): 5000<jupyter_text>CODE-testTesting data, released from paper *Automatic diagnosis of the 12-lead ECG using a deep neural network*<jupyter_code>dnm = 'CODE_TEST'
fnm = get_rec_paths(dnm)[0]
ic(fnm)

rec = h5py.File(fnm, 'r')
ic(rec.keys())
data = rec['tracings']
ic(data, type(data))
data = data[:]
ic(data.max(), data.min(), data.shape)

plot_single(data[0, :2048, 0], label='Lead I')<jupyter_output>ic| fnm: '/Users/stefanh/Documents/UMich/Research/ECG-Classify/datasets/CODE-test/ecg_tracings.hdf5'
ic| rec.keys(): <KeysViewHDF5 ['tracings']>
ic| data: <HDF5 dataset "tracings": shape (827, 4096, 12), type "<f8">
    type(data): <class 'h5py._hl.dataset.Dataset'>
ic| data.max(): 66.97279486521437
    data.min(): -51.1795425976763
    data.shape: (827, 4096, 12)
{'fraction_non_alphanumeric': 0.17600531738118977, 'fraction_numerical': 0.12336324360252576, 'mean_word_length': 2.636056065732238, 'pattern_counts': {'":': 1, '<': 54, '<?xml version=': 0, '>': 48, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '423264', 'license': 'MIT', 'n_tokens_mistral': 7494, 'n_tokens_neox': 6145, 'n_words': 1117, 'path': 'notebook/pre-processing/explore_dataset.ipynb', 'repo_name': 'StefanHeng/ECG-Representation-Learning', 'size': '15045'}