<jupyter_start><jupyter_text>Univariate Resemblance Analysis (URA) Dataset D<jupyter_code>#import libraries
import warnings
warnings.filterwarnings("ignore")
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
import os
print('Libraries imported!!')
#define directory of functions and actual directory
HOME_PATH = '' #home path of the project
FUNCTIONS_DIR = 'EVALUATION FUNCTIONS/RESEMBLANCE'
ACTUAL_DIR = os.getcwd()

#change directory to functions directory
os.chdir(HOME_PATH + FUNCTIONS_DIR)

#import functions for univariate resemblance analisys
from univariate_resemblance import basic_stats
from univariate_resemblance import student_t_tests
from univariate_resemblance import mann_whitney_tests
from univariate_resemblance import ks_tests
from univariate_resemblance import chi_squared_tests
from univariate_resemblance import scale_data
from univariate_resemblance import euclidean_distances
from univariate_resemblance import cosine_distances
from univariate_resemblance import js_distances
from univariate_resemblance import wass_distances

#change directory to actual directory
os.chdir(ACTUAL_DIR)
print('Functions imported!!')<jupyter_output>Functions imported!!<jupyter_text>1. Read real and synthetic datasetsIn this part real and synthetic datasets are read.<jupyter_code>#Define global variables
DATA_TYPES = ['Real','GM','SDV','CTGAN','WGANGP']
SYNTHESIZERS = ['GM','SDV','CTGAN','WGANGP']
FILEPATHS = {'Real' : HOME_PATH + 'REAL DATASETS/TRAIN DATASETS/D_ContraceptiveMethod_Real_Train.csv',
            'GM' : HOME_PATH + 'SYNTHETIC DATASETS/GM/D_ContraceptiveMethod_Synthetic_GM.csv',
            'SDV' : HOME_PATH + 'SYNTHETIC DATASETS/SDV/D_ContraceptiveMethod_Synthetic_SDV.csv',
            'CTGAN' : HOME_PATH + 'SYNTHETIC DATASETS/CTGAN/D_ContraceptiveMethod_Synthetic_CTGAN.csv',
            'WGANGP' : HOME_PATH + 'SYNTHETIC DATASETS/WGANGP/D_ContraceptiveMethod_Synthetic_WGANGP.csv'}
categorical_columns = ['wife_education','husband_education','wife_religion','wife_working','husband_occupation',
                        'standard_of_living_index','media_exposure','contraceptive_method_used']
data = dict()
#iterate over all datasets filepaths and read each dataset
for name, path in FILEPATHS.items() :
    data[name] = pd.read_csv(path)
    for col in categorical_columns :
        data[name][col] = data[name][col].astype('category')
data<jupyter_output><empty_output><jupyter_text>2. Basic statistics comparissonIn this part the basic statistics of each attribute for each dataset are computed.- means and stds of all variables are compared.<jupyter_code>df_means_std = pd.DataFrame(index=data['Real'].select_dtypes(include=['float64','int64']).columns.tolist())
for name in DATA_TYPES :
    means_stds = basic_stats(data[name])
    means_stds.columns = [name]
    df_means_std = pd.concat([df_means_std,means_stds], axis=1)
df_means_std<jupyter_output><empty_output><jupyter_text>3. Statistical TestsIn this part some statistical tests are performed to compare variables from real and synthetic data.  3.1. Sudent t-test- null hypothesis (p > 0.05): Means of real and synthetic feature are the same.- alternative hypothesis (p < 0.05): Mean of real and synthetic feature are significantly different.<jupyter_code>p_values_student = dict()
for name in SYNTHESIZERS :
    p_values_student[name] = student_t_tests(data['Real'], data[name])
df_student_test = pd.DataFrame(data=p_values_student, index=(data['Real'].select_dtypes(include=['int64','float64'])).columns)
df_student_test.to_csv('UNIVARIATE RESEMBLANCE RESULTS/student_t_tests.csv')
df_student_test<jupyter_output><empty_output><jupyter_text>3.2. Mann-Whitney U-Test- null hypothesis (p > 0.05): Real feature and synthetic feature come from the same population.- alternative hypothesis (p < 0.05): Real feature and synthetic feature does not come from the same population.<jupyter_code>p_values_mann_whitnney = dict()
for name in SYNTHESIZERS :
    p_values_mann_whitnney[name] = mann_whitney_tests(data['Real'], data[name])
df_mann_whitney = pd.DataFrame(data=p_values_mann_whitnney, index=(data['Real'].select_dtypes(include=['int64','float64'])).columns)
df_mann_whitney.to_csv('UNIVARIATE RESEMBLANCE RESULTS/mann_whitnney_tests.csv')
df_mann_whitney<jupyter_output><empty_output><jupyter_text>3.3. Kolmogorov-Smirnov Test- null hypothesis (p > 0.05): Real feature and synthetic feature distributions are equal.- alternative hypothesis (p < 0.05): Real feature and synthetic feature distributions are not equal.<jupyter_code>p_values_ks = dict()
for name in SYNTHESIZERS :
    p_values_ks[name] = ks_tests(data['Real'], data[name])
df_ks = pd.DataFrame(data=p_values_ks, index=(data['Real'].select_dtypes(include=['int64','float64'])).columns)
df_ks.to_csv('UNIVARIATE RESEMBLANCE RESULTS/ks_tests.csv')
df_ks<jupyter_output><empty_output><jupyter_text>3.4. Join all p-values of all tests in one unique dataset<jupyter_code>tests_all_data = np.round(np.concatenate([df_student_test.values, df_mann_whitney.values, df_ks.values] , axis=1),4)
#join results of all tests
df_tests_all = pd.DataFrame(data=tests_all_data, index=(data['Real'].select_dtypes(include=['int64','float64'])).columns,
                columns=['GM_S','SDV_S','CTGAN_S','WGANGP_S','GM_M','SDV_M','CTGAN_M','WGANGP_M','GM_K','SDV_K','CTGAN_K','WGANGP_K'])
df_tests_all.to_csv('UNIVARIATE RESEMBLANCE RESULTS/STATISTICAL_TESTS_ALL.csv')
df_tests_all<jupyter_output><empty_output><jupyter_text>3.4. Chi-Squared Test (categorical variables)- null hypothesis (p > 0.05): There is not statistical relationship between real and synthetic feature.- alternative hypothesis (p < 0.05): There is a statistical relationship between real and synthetic feature.<jupyter_code>p_values_chi = dict()
for name in SYNTHESIZERS :
    p_values_chi[name] = chi_squared_tests(data['Real'], data[name])
df_chi = pd.DataFrame(data=p_values_chi, index=(data['Real'].select_dtypes(include=['category'])).columns)
df_chi.to_csv('UNIVARIATE RESEMBLANCE RESULTS/chi_squared_tests.csv')
df_chi<jupyter_output><empty_output><jupyter_text>4. Distances calculation for each numerical variableIn this part some distance metrics are calculated for each numerical variable. First the data is scaled between 0 and 1.<jupyter_code>#Scale the data
num_cols = (data['Real'].select_dtypes(include=['int64','float64'])).columns
scaled_data = dict()
for name in DATA_TYPES :
    scaled_data[name] = scale_data(data[name][num_cols])
scaled_data<jupyter_output><empty_output><jupyter_text>4.1. Euclidean distancesSquare root of the sum of squares of differences between real feature and synthetic feature<jupyter_code>euclidean_dists = dict()
#calculate distances for each synthesizer
for name in SYNTHESIZERS :
    euclidean_dists[name] = euclidean_distances(scaled_data['Real'], scaled_data[name])
#create dataframe with distances
df_euclidean_dists = pd.DataFrame(data=euclidean_dists, index=num_cols)
df_euclidean_dists.to_csv('UNIVARIATE RESEMBLANCE RESULTS/euclidean_distances.csv')
df_euclidean_dists<jupyter_output><empty_output><jupyter_text>4.2. Cosine distancesRelated with cosine similarity; the dot product of the two vectors divided by the product of the two vectors' lengths.<jupyter_code>cosine_dists = dict()
#calculate distances for each synthesizer
for name in SYNTHESIZERS :
    cosine_dists[name] = cosine_distances(scaled_data['Real'], scaled_data[name])
#create dataframe with distances
df_cosine_dists = pd.DataFrame(data=cosine_dists, index=num_cols)
df_cosine_dists.to_csv('UNIVARIATE RESEMBLANCE RESULTS/cosine_distances.csv')
df_cosine_dists<jupyter_output><empty_output><jupyter_text>4.3. Jensen-Shannon distancesMeasures the similarity between two probability distributions.<jupyter_code>js_dists = dict()
#calculate distances for each synthesizer
for name in SYNTHESIZERS :
    js_dists[name] = js_distances(scaled_data['Real'], scaled_data[name])
#create dataframe with distances
df_js_dists = pd.DataFrame(data=js_dists, index=num_cols)
df_js_dists.to_csv('UNIVARIATE RESEMBLANCE RESULTS/js_distances.csv')
df_js_dists<jupyter_output><empty_output><jupyter_text>4.4. Wasserstein distancesMinimum amount of work required to transform a vector into another vector.<jupyter_code>was_dists = dict()
#calculate distances for each synthesizer
for name in SYNTHESIZERS :
    was_dists[name] = wass_distances(scaled_data['Real'], scaled_data[name])
#create dataframe with distances
df_was_dists = pd.DataFrame(data=was_dists, index=num_cols)
df_was_dists.to_csv('UNIVARIATE RESEMBLANCE RESULTS/wass_distances.csv')
df_was_dists<jupyter_output><empty_output><jupyter_text>4.5. Join all distances in an unique dataframe<jupyter_code>distances_all_data = np.round(np.concatenate([df_cosine_dists.values, df_js_dists.values, df_was_dists.values] , axis=1),4)
#join results of all tests
df_dists_all = pd.DataFrame(data=distances_all_data, index=(data['Real'].select_dtypes(include=['int64','float64'])).columns,
                columns=['GM_C','SDV_C','CTGAN_C','WGANGP_C','GM_J','SDV_J','CTGAN_J','WGANGP_J','GM_W','SDV_W','CTGAN_W','WGANGP_W'])
df_dists_all.to_csv('UNIVARIATE RESEMBLANCE RESULTS/DISTANCES_ALL.csv')
df_dists_all<jupyter_output><empty_output><jupyter_text>5. Boxplots comparissonIn this part boxplots of each variable are plotted to analyse the statistical properties of each variable.<jupyter_code>columns = (data['Real'].select_dtypes(include=['float64','int64'])).columns
len(columns)
boxplots_data = dict()
for c in columns :
    boxplots_data[c] = data['Real'][c]
    for name in SYNTHESIZERS :
        boxplots_data[c] = np.column_stack((boxplots_data[c], data[name][c]))
fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(5,2.5))

for i in range(0,len(columns)) :
    ax = axs[i]
    ax.boxplot(boxplots_data[columns[i]])
    ax.set_title(columns[i])
    ax.set_xticklabels(DATA_TYPES)

for ax in axs.ravel():
    ax.set_xticklabels(ax.get_xticklabels(), rotation = 30, ha="right")
fig.tight_layout()
fig.savefig('UNIVARIATE RESEMBLANCE RESULTS/BOXPLOTS.svg', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>6. Frequency distributions comparissonIn this part histograms of each variable are plotted to compare the frequency distributions.<jupyter_code>columns = data['Real'].columns
columns
hists_data = dict()
for c in columns :
    hists_data[c] = data['Real'][c]
    for name in SYNTHESIZERS :
        hists_data[c] = np.column_stack((hists_data[c], data[name][c]))
fig, axs = plt.subplots(nrows=2, ncols=5, figsize=(14,5))
idx = {0:[0,0], 1:[0,1], 2:[0,2], 3:[0,3], 4:[0,4], 5:[1,0], 6:[1,1], 7:[1,2], 8:[1,3], 9:[1,4]}

for i in range(0,len(columns)) :
    ax = axs[idx[i][0], idx[i][1]]
    ax.set_title(columns[i])
        
    if i in [0,3] :
        arr = hists_data[columns[i]]
        for col in range(arr.shape[1]) :
            arr_col = np.sort(arr[:,col])
            sns.distplot(arr_col[int(0.05*len(arr_col)) : int(0.95 * len(arr_col))], hist=False, norm_hist=False, ax=ax)
    else :
        ax.hist(hists_data[columns[i]], density=False, histtype='bar', label=DATA_TYPES, align='mid')
        
        if columns[i] in ['wife_education','husband_education','husband_occupation','standard_of_living_index'] :
            ax.set_xticks([1.15, 2.05, 2.95, 3.85])
            ax.set_xticklabels([1,2,3,4])
            
        elif columns[i] in ['wife_religion','wife_working','media_exposure'] :
            ax.set_xticks([0.05, 0.95])
            ax.set_xticklabels([0,1])
        
        else :
            ax.set_xticks([1.1, 2.1, 2.9])
            ax.set_xticklabels([1,2,3])
            
    ax.set_yticks([])
    
fig.tight_layout()
axs[idx[i][0], idx[i][1]].legend(ncol=5, bbox_to_anchor=(-0.8,-0.2))
fig.savefig('UNIVARIATE RESEMBLANCE RESULTS/HISTOGRAMS.svg', bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11969339622641509, 'fraction_numerical': 0.01549865229110512, 'mean_word_length': 6.4113607990012484, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '98252', 'license': 'MIT', 'n_tokens_mistral': 4379, 'n_tokens_neox': 4116, 'n_words': 832, 'path': 'notebooks/Dataset D - Contraceptive Method Choice/Synthetic data evaluation/Resemblance/1_Univariate_Resemblance_DatasetD.ipynb', 'repo_name': 'Vicomtech/STDG-evaluation-metrics', 'size': '11872'}