<jupyter_start><jupyter_text>This notebook implements the mutlinomial estimation of joint entropy using SInfoME building block<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.models import Model
from keras.layers import Dense, Input
from keras import optimizers
from tqdm import tqdm_notebook as tqdm
from scipy.stats import random_correlation
import scipy.stats as stats
from entropy import *
from scipy.stats import multinomial, geom
from scipy.stats import poisson
import os
import sys

# import npeet package for KNN-based estimators
sys.path.insert(0, '<path to NPEET package> \\NPEET-master\\npeet')
import entropy_estimators as ee
# convert class to one hot 
def convert_to_one_hot(y, dict_size=None):
    if dict_size is None:
        dict_size = np.unique(y).shape[0]
    y_hot = np.eye(dict_size)[y.astype('int32')]
    return y_hot

def make_one_hot(y, dims, dict_size=None):
    y_hot = []
    for i in range(dims):
        y_hot.append(convert_to_one_hot(y[:, i], dict_size))
    return y_hot
# Basic fully connected model with number of losses equales to dims - the dimensions of Y
def model_basic_classification(input_shape, class_size):
    l0 = Input(shape=input_shape, dtype = 'float32', name = 'input_l')
    X = Dense(units=500, kernel_initializer='random_uniform', name = 'l1')(l0)
    X = Dense(units=500, kernel_initializer='random_uniform', name = 'l2')(X)


    output = [Dense(class_size, activation='softmax')(X) for i in range(1) ]
    model = Model(input = [l0], outputs =  output )
    return model<jupyter_output><empty_output><jupyter_text>Estimating H(X)<jupyter_code>#Training paramteters
epochs = 10
batch_size = 512

# lists that will obtain the entropies estimations
sub_loss_lst = []
H_y_lst = [ [] for _ in range(dims)]
H_yx_lst = [ [] for _ in range(dims)]
H_y_total = []
results = []

# KNN based lists
H_sota_1 = []
H_sota_3 = []
H_sota_5 = []
H_sota_10 = []
H_sota_20 = []
hist_lst = []

# Will be used to convert KNN estimates from bits to nats
scale = np.log2(np.exp(1))

# parameters for simulated data and entropy true value of multinomial entropy given the known values of p
length = 1000 #dataset size
p = np.random.uniform(size=dims) #multinomial
p /= p.sum()
H_true = multinomial.entropy(n=100, p=p)


n = np.repeat(length, 100) #number of experiments
for k, size in enumerate(n):    
    
    # Generating multinomial data
    data = multinomial.rvs(n=100, p=p,size=size)
    data = np.array(data)
    
    # The number of classes - the maximum value obtained from data + 1 (including zero)
    class_size = data.max() + 1
    
    # Estimating entropy with the KNN estimator
    H_y_lst = [ [] for _ in range(dims)]
    H_sota_1.append(ee.entropy(data, k=1)/scale)
    H_sota_3.append(ee.entropy(data, k=3)/scale)
    H_sota_5.append(ee.entropy(data, k=5)/scale)
    H_sota_10.append(ee.entropy(data, k=10)/scale)
    H_sota_20.append(ee.entropy(data, k=20)/scale)
    
    
    
   # Preparing m - 1 NN models
    model_lst = []
    opt_lst = []
    for m in range(0, dims):
        if m == 0:
            model_lst.append(None)
        else:
            model_lst.append(model_basic_classification([m], class_size))

    for m in range(0, dims):
        if m == 0:
            opt_lst.append(None)
        else:
            opt_lst.append(optimizers.Adam())

    # Apply SInfoME block m - 1 times
    for m in range (1, dims):
        model_lst[m].compile(loss='categorical_crossentropy', optimizer=opt_lst[m], metrics=['accuracy'])
    
    for j in range(0, dims):
        print('dim number', j)
        if j != 0:
            x = data[:, range(j)]
            y = data[:, j]
            y = np.reshape(y, [-1, 1])
            y_hot = make_one_hot(y, 1, class_size)
            hist = model_lst[j].fit(x, y_hot, epochs=epochs, batch_size=batch_size, validation_split=0.3, verbose=0)
            H_y_lst[j].append(hist.history['loss'])
        else:
            y = data[:, j]
            _, p_1 = np.unique(y, return_counts=True)
            p_1 = p_1/(p_1.sum() + 10**-5) 
        
            H_y_lst[j].append([np.sum(np.array(p_1)*np.log(p_1))]*epochs)
    plt.plot(hist.history['loss'])
    plt.plot(hist.history['val_loss'])
    plt.show()

    H_y_total.append(np.reshape(np.sum(H_y_lst, axis=0, ), [-1]))
    results.append(np.reshape(H_y_total, [-1])[-1])
    
print('Results from SInfoME:', results) 
print('Results from KNN(1):', H_sota_1)
print('Results from KNN(3):', H_sota_3)
print('Results from KNN(5):', H_sota_5)
print('Results from KNN(10):', H_sota_10)
print('Results from KNN(20):', H_sota_20)
print(' Ground truth:', H_true)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12756410256410255, 'fraction_numerical': 0.02200854700854701, 'mean_word_length': 3.411875589066918, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '28697', 'license': 'MIT', 'n_tokens_mistral': 1772, 'n_tokens_neox': 1702, 'n_words': 418, 'path': 'multinomial entropy estimation-SInfoME.ipynb', 'repo_name': 'YuvalShalev/NJEE', 'size': '4680'}