<jupyter_start><jupyter_text>The code below try to train a model to distinguish 2 datasets<jupyter_code>### Use Neruron Network to classify those two dataset
### And the accuracy as really hign (88% on test set)
### So this model of NN can de-couple those dimensions and uses them to classify 2 datasets pretty well

#mlp = MLPClassifier([60, 40, 20, 5], activation="relu", max_iter=500, verbose=True)
mlp = MLPClassifier([60, 40, 20, 10, 5], activation="relu", max_iter=500, verbose=True)
rf = RandomForestClassifier(max_depth = 20)
svc = SVC(kernel = "rbf", verbose=True)

# data and model
model = mlp


data_ = all_data
labels_ = labels


kf = KFold(2, shuffle = True)
acc_test_mlp = []
acc_train_mlp = []
res = np.zeros(1)
for train, test in kf.split(data_):
    model = mlp
    model.fit(data_[train], labels_[train])
    out_train = model.predict(data_[train])
    out_test = model.predict(data_[test])
    
    acc_test_mlp.append(np.sum(out_test == labels[test]) / labels[test].shape[0])
    acc_train_mlp.append(np.sum(out_train == labels[train]) / labels[train].shape[0])
acc_train_mlp
acc_test_mlp<jupyter_output><empty_output><jupyter_text>The code below try to embed and visualise the output of each layer<jupyter_code># calculate the output of each layer of out neuron network

act1 = ACTIVATIONS['relu'](np.matmul(data_[test], model.coefs_[0]) + model.intercepts_[0])
act2 = ACTIVATIONS['relu'](np.matmul(act1, model.coefs_[1]) + model.intercepts_[1])
act3 = ACTIVATIONS['relu'](np.matmul(act2, model.coefs_[2]) + model.intercepts_[2])
act4 = ACTIVATIONS['relu'](np.matmul(act3, model.coefs_[3]) + model.intercepts_[3])
act5 = ACTIVATIONS['relu'](np.matmul(act4, model.coefs_[4]) + model.intercepts_[4])
act6 = ACTIVATIONS['relu'](np.matmul(act5, model.coefs_[5]) + model.intercepts_[5])
# conduct tsne on each output

tsne = TSNE(2, perplexity = 50)
act0_tsne = tsne.fit_transform(data_[:20000])
print("finish 0")
act1_tsne = tsne.fit_transform(act1[:20000])
print("finish 1")
act2_tsne = tsne.fit_transform(act2[:20000])
print("finish 2")
act3_tsne = tsne.fit_transform(act3[:20000])
print("finish 3")
act4_tsne = tsne.fit_transform(act4[:20000])
print("finish 4")
act5_tsne = tsne.fit_transform(act5[:20000])
print("finish 5")
# plot

plt.rcParams['figure.figsize'] = (15.0, 10.0)
plt.figure()


plt.subplot(2,3,1)
plt.scatter(act0_tsne[labels_[test][:20000] == 0, 0], act0_tsne[labels_[test][:20000] == 0, 1], s = 1, label = "real data", alpha=0.7) # real data
plt.scatter(act0_tsne[labels_[test][:20000] == 1, 0], act0_tsne[labels_[test][:20000] == 1, 1], s = 1, label = "syn data", alpha=0.7)
plt.legend(loc="upper right")
plt.title("Data Input")


plt.subplot(2,3,2)
plt.scatter(act1_tsne[labels_[test][:20000] == 0, 0], act1_tsne[labels_[test][:20000] == 0, 1], s = 1, label = "real data", alpha=0.7) # real data
plt.scatter(act1_tsne[labels_[test][:20000] == 1, 0], act1_tsne[labels_[test][:20000] == 1, 1], s = 1, label = "syn data", alpha=0.7)
plt.legend(loc="upper right")
plt.title("1st Layer Output")


plt.subplot(2,3,3)
plt.scatter(act2_tsne[labels_[test][:20000] == 0, 0], act2_tsne[labels_[test][:20000] == 0, 1], s = 1, label = "real data", alpha=0.7) # real data
plt.scatter(act2_tsne[labels_[test][:20000] == 1, 0], act2_tsne[labels_[test][:20000] == 1, 1], s = 1, label = "syn data", alpha=0.7)
plt.legend(loc="upper right")
plt.title("2nd Layer Output")

plt.subplot(2,3,4)
plt.scatter(act3_tsne[labels_[test][:20000] == 0, 0], act3_tsne[labels_[test][:20000] == 0, 1], s = 1, label = "real data", alpha=0.7) # real data
plt.scatter(act3_tsne[labels_[test][:20000] == 1, 0], act3_tsne[labels_[test][:20000] == 1, 1], s = 1, label = "syn data", alpha=0.7)
plt.legend(loc="upper right")
plt.title("3rd Layer Output")

plt.subplot(2,3,5)
plt.scatter(act4_tsne[labels_[test][:20000] == 0, 0], act4_tsne[labels_[test][:20000] == 0, 1], s = 1, label = "real data", alpha=0.7) # real data
plt.scatter(act4_tsne[labels_[test][:20000] == 1, 0], act4_tsne[labels_[test][:20000] == 1, 1], s = 1, label = "syn data", alpha=0.7)
plt.legend(loc="upper right")
plt.title("4th Layer Output")

plt.subplot(2,3,6)
plt.scatter(act5_tsne[labels_[test][:20000] == 0, 0], act5_tsne[labels_[test][:20000] == 0, 1], s = 1, label = "real data", alpha=0.7) # real data
plt.scatter(act5_tsne[labels_[test][:20000] == 1, 0], act5_tsne[labels_[test][:20000] == 1, 1], s = 1, label = "syn data", alpha=0.7)
plt.legend(loc="upper right")
plt.title("5th Layer Output")


plt.show()<jupyter_output><empty_output><jupyter_text>The code below tries to find which features really matter in trainingBy checking the plots above, we find that after the 1st layer, the data are greatly decoupled, so we think the 1st layer makes a real difference.To check which feature the 1st layer is detecting, we get inspiration from the paper "<NAME>., <NAME>. (2014) Visualizing and Understanding Convolutional Networks". We find the samples mostly active the 1st layer, then recover input by taking pseudo-inverse.$A^{[1]} = Relu(WX + b)$,So we can get $ X = W^{-1}(A^{[1]} - b)$, where $W^{-1}$means pseudo-inverse.<jupyter_code>act1_argsort = np.argsort(act1, axis=0) # find the index of samples that mostly active the neurons in 1st layer
len(list(act1_argsort[-5:].flatten())) # for each neurons, find the 5th most active sample
act_samples = set(list(act1_argsort[-5:].flatten())) 
len(act_samples)
act_data = data_[test].copy()
act_data = act_data[list(act_samples), :]
act_data_labels = labels_[test].copy()[list(act_samples)]
act_out = act1[list(act_samples), :]
#act_out = np.matmul((act_out - model.intercepts_[4]), np.linalg.pinv(model.coefs_[4]))
#act_out = np.matmul((act_out - model.intercepts_[3]), np.linalg.pinv(model.coefs_[3]))
#act_out = np.matmul((act_out - model.intercepts_[2]), np.linalg.pinv(model.coefs_[2]))
#act_out = np.matmul((act_out - model.intercepts_[1]), np.linalg.pinv(model.coefs_[1]))
act_out = np.matmul((act_out - model.intercepts_[0]), np.linalg.pinv(model.coefs_[0]))
act_data_recover = act_out
diff = -act_data.T + act_data_recover.T

# shows the distribution of diff
plt.hist(diff.flatten(), bins = 1000, density = True)
plt.show()
mask = diff < 0

# mask out those diff is < 0 (means these dimension are not so useful in the 1st layer beacuse they locate in x<0 in relu function)
diff[diff < 0] = 0
print(mask)
print(diff)

plt.show()
plt.imshow(diff, cmap='gray') 
plt.show()

# here we find important dimensions, 22 dimensions are chosen, shows below

print(np.sum(mask > 0, axis=1))


important_dims = np.where(np.sum(mask > 0, axis=1) >120)[0]
#important_dims = np.where(mask)[0]
print(important_dims.shape)
important_dims

important_dims2 = important_dims.copy()<jupyter_output><empty_output><jupyter_text>The code below find the most important combination of dimensions.<jupyter_code># each time find 2 dimensions and train the model with only 2 dimension, print the combination with an accuracy > 0.6
# find that[25,29],[29,30],[25,30] is the most important dimensions.


for i in important_dims2:
    for j in important_dims2:
        if i == j:
            continue
        important_dims = [i, j]
        #mlp = MLPClassifier([60, 40, 20, 5], activation="relu", max_iter=500, verbose=True)
        mlp = MLPClassifier([60, 40, 20, 10, 5], activation="relu", max_iter=500, verbose=True)
        rf = RandomForestClassifier(max_depth = 10)
        svc = SVC(kernel = "rbf", verbose=True)

        # data and model


        data_2 = all_data[:, important_dims]
        labels_2 = labels

        #print(all_data.shape)
        #print(data_2.shape)
    

        kf = KFold(2, shuffle = True)
        acc_test_mlp2 = []
        acc_train_mlp2 = []

        for train2, test2 in kf.split(data_2):
            model2 = rf
            model2.fit(data_2[train2], labels_2[train2])
            out_train2 = model2.predict(data_2[train2])
            out_test2 = model2.predict(data_2[test2])
        
            acc_test_mlp2.append(np.sum(out_test2 == labels_2[test2]) / labels_2[test2].shape[0])
            acc_train_mlp2.append(np.sum(out_train2 == labels_2[train2]) / labels_2[train2].shape[0])
        
        if np.mean(acc_test_mlp2) > 0.6:
            print(important_dims)
        print(":::::" + str(j)) # just to show which step I am in
    print(i)
# train the model with only [25,29,30] and all other dimensions separately.
# to see if these 3 dimensions really matter
# the result is using [25,29,30] the accuracy is 72%, while using other is 74%
# so we can get a conclusion that these 3 dimensions really matter

important_dims = [ 0,  2,  4,  6,  8,  9, 10, 11, 13, 15, 16, 18, 21, 22, 28, 31, 32, 33, 34]
important_dims = [25, 29, 30]
#mlp = MLPClassifier([60, 40, 20, 5], activation="relu", max_iter=500, verbose=True)
mlp = MLPClassifier([60, 40, 20, 10, 5], activation="relu", max_iter=500, verbose=True)
rf = RandomForestClassifier()
svc = SVC(kernel = "rbf", verbose=True)

data_2 = all_data[:, important_dims]
labels_2 = labels

kf = KFold(2, shuffle = True)
acc_test_mlp2 = []
acc_train_mlp2 = []

for train2, test2 in kf.split(data_2):
    model2 = rf
    model2.fit(data_2[train2], labels_2[train2])
    out_train2 = model2.predict(data_2[train2])
    out_test2 = model2.predict(data_2[test2])
        
    acc_test_mlp2.append(np.sum(out_test2 == labels_2[test2]) / labels_2[test2].shape[0])
    acc_train_mlp2.append(np.sum(out_train2 == labels_2[train2]) / labels_2[train2].shape[0])
acc_test_mlp2
acc_train_mlp2
# get names of all dimensions 

col_names = []
for i in pd.concat([real_data, syn_data]).columns:
    if i in not_in_pc: 
        continue
    col_names.append(i)
np.array(col_names)
np.array(col_names)[important_dims] # print the name of these 3 important dimensions<jupyter_output><empty_output><jupyter_text>After finding these 3 dimensions, we check in which way these 3 dimensions of real data are different from the synthetic data.<jupyter_code>real_data_3dim = data_2[labels == 0]
syn_data_3dim = data_2[labels == 1]
df_real = pd.DataFrame(real_data_3dim)
df_real.columns = ['insulin', 'change', 'diabetesMed']
df_real.corr() 
df_syn = pd.DataFrame(syn_data_3dim)
df_syn.columns = ['insulin', 'change', 'diabetesMed']
df_syn.corr()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15560975609756098, 'fraction_numerical': 0.06097560975609756, 'mean_word_length': 4.959883720930232, 'pattern_counts': {'":': 1, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '250768', 'license': 'MIT', 'n_tokens_mistral': 4271, 'n_tokens_neox': 3859, 'n_words': 921, 'path': '2_Find_the_difference_between_synthetic_and_real_data.ipynb', 'repo_name': 'hejj16/Hachathon-2020', 'size': '10256'}