<jupyter_start><jupyter_text>Data preprocess<jupyter_code>import emd.search_emd as emd
# Reading training and test sets
datasetFile = "/content/Latent_vs_Original_Space_Image_Classification/data/train-images-idx3-ubyte"
dlabelsFile = "/content/Latent_vs_Original_Space_Image_Classification/data/train-labels-idx1-ubyte"
testsetFile = "/content/Latent_vs_Original_Space_Image_Classification/data/t10k-images-idx3-ubyte"
tlabelsFile = "/content/Latent_vs_Original_Space_Image_Classification/data/t10k-labels-idx1-ubyte"


def load_mnist(dataset, digits=np.arange(10), type='data', numOfElements=-1):
    intType = np.dtype( 'int32' ).newbyteorder( '>' )
    if not os.path.isfile(dataset):
        return None
    fname = os.path.join(".", dataset)
    if (type == 'data'):
        nMetaDataBytes = 4 * intType.itemsize
        images = np.fromfile(fname, dtype = 'ubyte')
        magicBytes, size, rows, cols = np.frombuffer(images[:nMetaDataBytes].tobytes(), intType)
        if numOfElements == -1:
            numOfElements = size #int(len(ind) * size/100.)
        images = images[nMetaDataBytes:].astype(dtype = 'float32').reshape([numOfElements, rows, cols, 1])
        return images
    elif (type == 'labels'):
        nMetaDataBytes = 2 * intType.itemsize
        labels = np.fromfile(fname, dtype = 'ubyte')[nMetaDataBytes:]
        return labels
    else:
        return None


train_X = emd.load_mnist(datasetFile, type='data')[0:]
train_Y = emd.load_mnist(dlabelsFile, type='labels')[0:]
test_X  = emd.load_mnist(testsetFile, type='data')[0:]
test_Y  = emd.load_mnist(tlabelsFile, type='labels')[0:]

# reshape labels
train_Y = train_Y.reshape((-1,1))
test_Y  = test_Y.reshape((-1,1))

input_shape = train_X.shape[1:]
num_of_classes = train_Y.shape

print(train_X.shape, train_Y.shape)<jupyter_output>(60000, 28, 28, 1) (60000, 1)<jupyter_text>Manhattan Distance Knn<jupyter_code>t = 100
q = 5

# preprocess for knn manhattan
x_train = train_X.reshape((60000, -1))[0:t]
x_test = test_X.reshape((10000, -1))[0:q]
y_train = train_Y[0:t]
y_test  = test_Y[0:q]

knn = emd.KNN(10, emd.manhattan_distances)
knn.fit(x_train, y_train)
prediction = knn.predict(x_test)
print("Time: ",knn.prediction_time)
print("Accuracy: ", emd.get_Accuracy(prediction, y_test))<jupyter_output>Time:  0.0017328262329101562
Accuracy:  0.56<jupyter_text>Earths Movers Distance Knn  Cluster size 14x14<jupyter_code># preprocess for EMD
dim = 14
clusters = emd.get_Clusters(train_X[0:t], [dim, dim])
test_clusters = emd.get_Clusters(test_X[0:q], [dim, dim])

distances, distances_array = emd.get_Clusters_distances(dim, [28,28])
# 2 times the weights for the linprog
num_of_weights = 2*len(clusters[0])
# variables are the of distances
num_of_variables = len(distances)
print(num_of_weights, num_of_variables)
# get A for the EMD coefficients
A = emd.get_A(num_of_weights, num_of_variables)

emd_knn = emd.KNN(10, emd.earths_movers_distances, distances, A)
emd_knn.fit(clusters, y_train)
prediction = emd_knn.predict(test_clusters)
print("Time: ",knn.prediction_time)
print("Accuracy: ", emd.get_Accuracy(prediction, y_test))<jupyter_output>8 16
Time:  0.0017328262329101562
Accuracy:  0.12000000000000002<jupyter_text>Cluster size 7x7<jupyter_code># preprocess for EMD
dim = 7
clusters = emd.get_Clusters(train_X[0:t], [dim, dim])
test_clusters = emd.get_Clusters(test_X[0:q], [dim, dim])

distances, distances_array = emd.get_Clusters_distances(dim, [28,28])
# 2 times the weights for the linprog
num_of_weights = 2*len(clusters[0])
# variables are the of distances
num_of_variables = len(distances)
print(num_of_weights, num_of_variables)
# get A for the EMD coefficients
A = emd.get_A(num_of_weights, num_of_variables)

emd_knn = emd.KNN(10, emd.earths_movers_distances, distances, A)
emd_knn.fit(clusters, y_train)
prediction = emd_knn.predict(test_clusters)
print("Time: ",knn.prediction_time)
print("Accuracy: ", emd.get_Accuracy(prediction, y_test))<jupyter_output>32 256
Time:  0.0017328262329101562
Accuracy:  0.1<jupyter_text>Cluster size 4x4<jupyter_code># preprocess for EMD
dim = 4
clusters = emd.get_Clusters(train_X[0:t], [dim, dim])
test_clusters = emd.get_Clusters(test_X[0:q], [dim, dim])

distances, distances_array = emd.get_Clusters_distances(dim, [28,28])
# 2 times the weights for the linprog
num_of_weights = 2*len(clusters[0])
# variables are the of distances
num_of_variables = len(distances)
print(num_of_weights, num_of_variables)
# get A for the EMD coefficients
A = emd.get_A(num_of_weights, num_of_variables)

emd_knn = emd.KNN(10, emd.earths_movers_distances, distances, A)
emd_knn.fit(clusters, y_train)
prediction = emd_knn.predict(test_clusters)
print("Time: ",knn.prediction_time)
print("Accuracy: ", emd.get_Accuracy(prediction, y_test))<jupyter_output>98 2401
Time:  0.0017328262329101562
Accuracy:  0.24
{'fraction_non_alphanumeric': 0.1282635723166183, 'fraction_numerical': 0.048487360132615, 'mean_word_length': 6.026200873362446, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '346787', 'license': 'MIT', 'n_tokens_mistral': 2065, 'n_tokens_neox': 1909, 'n_words': 343, 'path': 'experiments/emd_experiments.ipynb', 'repo_name': 'AGiannoutsos/Latent_vs_Original_Space_Image_Classification', 'size': '4826'}