<jupyter_start><jupyter_text>Correlation analysis formula: $ \boxed{ \rho_{i, k} = \frac{N^{11}N^{00} - N^{01}N^{10}} {\sqrt{(N^{11}+N^{10})(N^{01}+N^{00})(N^{11}+N^{01})(N^{10}+N^{00})}}} $ where $ N^{11} $ and $ N^{00} $ represent the percentage of cases when both descriptors predicted correctly or incorrectly, respectively; while $ N^{10} $ and $ N^{01} $ represent cases of divergence, the first descriptor correctly predicts the class but not the second, and vice-versa, respectively; i and k refers to descriptor i and descriptor k.<jupyter_code>import numpy as np
data_true = np.array([1, 2, 3, 4, 5])

data_cf1 = np.array([1, 2, 3, 4, 5])

data_cf2 = np.array([1, 3, 3, 4, 5])
d1 = data_true == data_cf1
data_true == data_cf1
d2 = data_true == data_cf2
data_true == data_cf2
d1 == True
d2 == False
def compute_n10(gt, cf1, cf2):
    assert gt.shape == cf1.shape ==cf2.shape
    N = gt.shape[0]
    a1 = gt == cf1
    a2 = gt == cf2
    
    b1 = a1 == True
    b2 = a2 ==False
    
    c = sum(b1 == b2)
    
    return c/N
def compute_n11(gt, cf1, cf2):
    assert gt.shape == cf1.shape ==cf2.shape
    N = gt.shape[0]
    a1 = gt == cf1
    a2 = gt == cf2
    
    b1 = a1 == True
    b2 = a2 ==True
    
    c = sum(b1 == b2)
    
    return c/N
def compute_n00(gt, cf1, cf2):
    assert gt.shape == cf1.shape ==cf2.shape
    N = gt.shape[0]
    a1 = gt == cf1
    a2 = gt == cf2
    
    b1 = a1 == False
    b2 = a2 ==False
    
    c = sum(b1 == b2)
    
    return c/N
def compute_n01(gt, cf1, cf2):
    assert gt.shape == cf1.shape ==cf2.shape
    N = gt.shape[0]
    a1 = gt == cf1
    a2 = gt == cf2
    
    b1 = a1 == False
    b2 = a2 ==True
    
    c = sum(b1 == b2)
    
    return c/N
def correlation(gt, cf1, cf2):
    n11 = compute_n11(gt, cf1, cf2)
    n10 = compute_n10(gt, cf1, cf2)
    n00 = compute_n00(gt, cf1, cf2)
    n01 = compute_n01(gt, cf1, cf2)
    
    rho = (n11*n00-n01*n10)/(np.sqrt((n11+n10)*(n01+n00)*(n11+n01)*(n10+n00)))
    return round(rho, 3)
data_true = np.array([1, 2, 3, 4, 5])

data_cf1 = np.array([1, 2, 3, 4, 5])

data_cf2 = np.array([1, 4, 8, 2, 1])
correlation(data_true, data_cf1, data_cf2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17186067827681026, 'fraction_numerical': 0.08936755270394134, 'mean_word_length': 2.7702936096718482, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '153551', 'license': 'MIT', 'n_tokens_mistral': 1022, 'n_tokens_neox': 942, 'n_words': 247, 'path': '.ipynb_checkpoints/correlation-analysis-checkpoint.ipynb', 'repo_name': 'Wanke15/Feature_extraction-SVM-classification-Remote-sensing', 'size': '2182'}