<jupyter_start><jupyter_text>Correlation between entropy/FD metrics<jupyter_code>import numpy as np
import pandas as pd
import entropy as ent
import seaborn as sns
import matplotlib.pyplot as plt
sns.set(font_scale=1.25)
# Generate 1000 1Hz sines with increasing noise
np.random.seed(123)
N = 1000
sf = 100
n_sines = 100
noises_factor = np.linspace(0, 1, n_sines)
noises = np.random.rand(n_sines, N)
sines = np.zeros(shape=(n_sines, N))
for i in range(n_sines):
    sines[i] = np.sin(2 * np.pi * np.arange(N) / sf) + noises_factor[i] * noises[i, :]
    
# Plot the first and last sines
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 6), sharex=True, sharey=False)
ax1.plot(sines[0])
ax2.plot(sines[-1]);
# Compute the entropy metrics
df = pd.DataFrame()

for i in range(n_sines):
    df = df.append({
        'PermEnt': ent.perm_entropy(sines[i], order=3, normalize=True),
        'SVDEnt' : ent.svd_entropy(sines[i], order=3, normalize=True),
        'SpecEnt' : ent.spectral_entropy(sines[i], sf, normalize=True),
        'AppEnt': ent.app_entropy(sines[i], order=2),
        'SampleEnt': ent.sample_entropy(sines[i], order=2),
        'PetrosianFD': ent.petrosian_fd(sines[i]),
        'KatzFD': ent.katz_fd(sines[i]),
        'HiguchiFD': ent.higuchi_fd(sines[i]),
        'DFA': ent.detrended_fluctuation(sines[i])}, ignore_index=True)

df.head().round(3)
# Describe
df.agg(['min', 'median', 'max']).round(2).T
# Distribution
df.hist(figsize=(6, 7), layout=(3, 3), sharey=True)
plt.tight_layout()
plt.figure(figsize=(8, 8))
sns.heatmap(df.corr().round(2), annot=True, vmin=-1, vmax=1, cbar=False, 
            cmap='Spectral_r', square=True);
sns.clustermap(df, row_cluster=False, standard_scale=1);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16868163500287853, 'fraction_numerical': 0.02936096718480138, 'mean_word_length': 4.717105263157895, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '170272', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 742, 'n_tokens_neox': 699, 'n_words': 139, 'path': 'notebooks/complexity_corr_sine.ipynb', 'repo_name': 'pranaysy/entropy', 'size': '1737'}