<jupyter_start><jupyter_text>Exercise Sheet 7 Ex. 1 Clustering  The aim of this exercise is to familiarise yourself with the analysis of clustering hyperparameters. For this purpose, let us consider the AgglomerativeClustering algorithm in sklearn. Using make circles, make moons, make blobs visualise the different performance of different linkages (ward, complete, average, single). Perform a little bit of hyperparameter tuning and show for the different linkage models the best model you have found. In the second part of this exercise, compare these results with the performance of t-SNE and k-means.<jupyter_code># Imports useful libraries

import numpy as np
import matplotlib.pyplot as plt
import tensorflow

from sklearn.cluster import AgglomerativeClustering
from sklearn.datasets import make_moons, make_circles, make_blobs
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
from pandas import DataFrame


from tensorflow.keras.models import Model, Sequential # imports network type
from tensorflow.keras.layers import Dense, Input # imports our layers want to use

from tensorflow.python.keras.losses import binary_crossentropy, categorical_crossentropy # loss function
from tensorflow.python.keras.optimizers import Adam, SGD # optimizers<jupyter_output><empty_output><jupyter_text>In the following we generate labelled data using the suggested libraries and we then implement the requested clustering algorithms. We start with blobs and then switch to circles and moons.<jupyter_code># Generate sample data (BLOBS)
X, y = make_blobs(n_samples=100, centers=3, n_features=2, random_state=4)

# Plot sample data with true classification
df = DataFrame(dict(x=X[:, 0], y=X[:, 1], label=y))
colors = {0: 'red', 1: 'blue', 2: 'green'}
fig, ax = plt.subplots()
grouped = df.groupby('label')
for key, group in grouped:
    group.plot(ax=ax, kind='scatter', x='x', y='y', color=colors[key])
plt.title('True Classification')
plt.show()
for linkage in ('ward', 'complete', 'average', 'single'):
    model = AgglomerativeClustering(affinity='euclidean', compute_full_tree='auto', connectivity=None, linkage=linkage, n_clusters=3, pooling_func='deprecated')
    model.fit(X)
    y_predicted = model.fit_predict(X)

    # Plot data with predicted classification
    df_predicted = DataFrame(dict(x=X[:, 0], y=X[:, 1], label=y_predicted))
    grouped_predicted = df_predicted.groupby('label')
    colors_predicted = {0: 'red', 1: 'green', 2: 'blue'}
    fig_predicted, ax_predicted = plt.subplots()
    for key, group in grouped_predicted:
        group.plot(ax=ax_predicted, kind='scatter', x='x', y='y', color=colors_predicted[key])
    performance = str(np.floor(100 - np.linalg.norm(y-y_predicted)))
    plt.title('Classification ' + '"' + linkage + '"')
    plt.show()
    print('Accuracy ' + performance + '%')

# Comparison with k-Means
kmeans = KMeans(n_clusters=3, random_state=0).fit(X)
y_kmeans = kmeans.labels_
performance_kmeans = str(np.floor(100 - np.linalg.norm(y-y_kmeans)))
print('KMeans Accuracy ' + performance_kmeans + '%')
# Generate sample data (CIRCLES)
X, y = make_circles(n_samples=100, noise=0.05, random_state=4)

# Plot sample data with true classification
df = DataFrame(dict(x=X[:, 0], y=X[:, 1], label=y))
fig, ax = plt.subplots()
grouped = df.groupby('label')
for key, group in grouped:
    group.plot(ax=ax, kind='scatter', x='x', y='y', color=colors[key])
plt.title('True Classification')
plt.show()

for linkage in ('ward', 'complete', 'average', 'single'):
    model = AgglomerativeClustering(affinity='euclidean', compute_full_tree='auto',
                                    connectivity=None, linkage=linkage, n_clusters=2,
                                    pooling_func='deprecated')

    model.fit(X)
    y_predicted = model.fit_predict(X)

    # Plot data with predicted classification
    df_predicted = DataFrame(dict(x=X[:, 0], y=X[:, 1], label=y_predicted))
    grouped_predicted = df_predicted.groupby('label')
    colors_predicted = {0: 'red', 1: 'blue'}
    fig_predicted, ax_predicted = plt.subplots()
    for key, group in grouped_predicted:
        group.plot(ax=ax_predicted, kind='scatter', x='x', y='y', color=colors_predicted[key])
    performance = str(np.floor(100 - np.linalg.norm(y-y_predicted)))
    plt.title('Classification ' + '"' + linkage + '"')
    plt.show()
    print('Accuracy ' + performance + '%')

# Comparison with k-Means
kmeans = KMeans(n_clusters=3, random_state=0).fit(X)
y_kmeans = kmeans.labels_
performance_kmeans = str(np.floor(100 - np.linalg.norm(y-y_kmeans)))
print('KMeans Accuracy ' + performance_kmeans + '%')

# Generate sample data (MOONS)
X, y = make_moons(n_samples=100, noise=0.05, random_state=4)

# Plot sample data with true classification
df = DataFrame(dict(x=X[:, 0], y=X[:, 1], label=y))
fig, ax = plt.subplots()
grouped = df.groupby('label')
for key, group in grouped:
    group.plot(ax=ax, kind='scatter', x='x', y='y', color=colors[key])
plt.title('True Moon Classification')
plt.show()

for linkage in ('ward', 'complete', 'average', 'single'):
    model = AgglomerativeClustering(affinity='euclidean', compute_full_tree='auto', connectivity=None, linkage=linkage, n_clusters=2, pooling_func='deprecated')

    model.fit(X)
    y_predicted = model.fit_predict(X)

    # Plot data with predicted classification
    df_predicted = DataFrame(dict(x=X[:, 0], y=X[:, 1], label=y_predicted))
    grouped_predicted = df_predicted.groupby('label')
    colors_predicted = {0: 'red', 1: 'blue'}
    fig_predicted, ax_predicted = plt.subplots()
    for key, group in grouped_predicted:
        group.plot(ax=ax_predicted, kind='scatter', x='x', y='y', color=colors_predicted[key])
    performance = str(np.floor(100 - np.linalg.norm(y-y_predicted)))
    plt.title('Classification ' + '"' + linkage + '"')
    plt.show()
    print('Accuracy ' + performance + '%')


# Comparison with k-Means
kmeans = KMeans(n_clusters=3, random_state=0).fit(X)
y_kmeans = kmeans.labels_
performance_kmeans = str(np.floor(100 - np.linalg.norm(y-y_kmeans)))
print('KMeans Accuracy ' + performance_kmeans + '%')<jupyter_output><empty_output><jupyter_text>Ex. 2 AutoEncoder  2.1 Generate 2-d images (1-channel) showing polynomials up to a maximum degree (e.g. 40x40) in two variables. Here we create 40x40 images of two variable, given maximum degree polynomials. For semplicity we exclude cross terms in the two variables $x$ and $y$ as for example $xy, x^2y$, etc.<jupyter_code># CREATES 40X40 IMAGES OF POLYNOMIALS OF MAXIMUM DEGREE
x=np.linspace(-1,1,40)
xx,yy=np.meshgrid(x,x)
       

def random_poly(deg, xx=xx, yy=yy): #function that creates polinomial of 2 variables of given degree with no cross terms
    x_params=np.random.uniform(-5.,5., deg+1)
    y_params=np.random.uniform(-5.,5., deg+1)
    
    p=np.zeros((40,40))
    
    for i in range(deg+1):
        p+=x_params[i]*xx**i + y_params[i]*yy**i 
        
    return p


# CREATES DATASET
size = 6000
data=[]

for i in range(size):
    degree=int(np.random.uniform(1,9))
    data.append(random_poly(degree))
data=np.array(data)

plt.imshow(data[1298]) # shows element 1298 of the dataset
plt.show()<jupyter_output><empty_output><jupyter_text>2.2 Build two autoencoder architectures (e.g. similar to the ones presented in the lecture) which involve a single hidden dense layer and several hidden dense layers.<jupyter_code># Autoencoder with a single dense hidden layer

model1=Sequential()
model1.add(Dense(800, input_dim=1600, activation='relu')) # encoder
model1.add(Dense(1600, activation='relu')) # decoder

model1.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy'])

# Arranges dataset for training

data=data.reshape((len(data), np.prod(data.shape[1:])))
x_train=data[:5900]
x_test=data[5900:]

history1 = model1.fit(x_train, x_train, epochs=100, batch_size=256, shuffle=True, verbose=1, validation_data=(x_test, x_test))
# Autoencoder with multiple hidden layers

model2=Sequential()
model2.add(Dense(800, input_dim=1600, activation='relu')) # encoder
model2.add(Dense(400, input_dim=1600, activation='relu')) # encoder
model2.add(Dense(200, input_dim=1600, activation='relu')) # encoder

model2.add(Dense(400, activation='relu')) # decoder
model2.add(Dense(800, activation='relu')) # decoder
model2.add(Dense(1600, activation='relu')) # decoder


model2.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy'])
history2 = model2.fit(x_train, x_train, epochs=100, batch_size=256, shuffle=True, verbose=1, validation_data=(x_test, x_test))<jupyter_output>Train on 5900 samples, validate on 100 samples
Epoch 1/100
5900/5900 [==============================] - 2s 329us/sample - loss: -15.8014 - acc: 0.0000e+00 - val_loss: -24.3783 - val_acc: 0.0000e+00
Epoch 2/100
5900/5900 [==============================] - 1s 243us/sample - loss: -22.5769 - acc: 0.0000e+00 - val_loss: -27.2968 - val_acc: 0.0000e+00
Epoch 3/100
5900/5900 [==============================] - 1s 243us/sample - loss: -25.9306 - acc: 0.0000e+00 - val_loss: -29.6808 - val_acc: 0.0000e+00
Epoch 4/100
5900/5900 [==============================] - 1s 245us/sample - loss: -30.5874 - acc: 0.0000e+00 - val_loss: -36.0546 - val_acc: 0.0000e+00
Epoch 5/100
5900/5900 [==============================] - 2s 266us/sample - loss: -36.1050 - acc: 0.0000e+00 - val_loss: -38.3320 - val_acc: 0.0000e+00
Epoch 6/100
5900/5900 [==============================] - 1s 245us/sample - loss: -39.0942 - acc: 0.0000e+00 - val_loss: -42.6779 - val_acc: 0.0000e+00
Epoch 7/100
5900/5900 [========================[...]<jupyter_text>2.3 For quadratic polynomials and two latent dimensions, visualise the results of your latent dimensions. Is an interpretation of your latent parameters easily visible?<jupyter_code># CREATES 40X40 IMAGES OF POLYNOMIALS OF MAXIMUM DEGREE
x=np.linspace(-1,1,40)
xx,yy=np.meshgrid(x,x)
       

def random_poly(deg, xx=xx, yy=yy): #function that creates polinomial of 2 variables of given degree with no cross terms
    x_params=np.random.uniform(-5.,5., deg+1)
    y_params=np.random.uniform(-5.,5., deg+1)
    
    p=np.zeros((40,40))
    
    for i in range(deg+1):
        p+=x_params[i]*xx**i + y_params[i]*yy**i 
        
    return p


# CREATES DATASET
size = 6000
data=[]

for i in range(size):
    degree=2
    data.append(random_poly(degree))
data=np.array(data)

plt.imshow(data[1298]) # shows element 1298 of the dataset
plt.show()


# Arranges dataset for training

data=data.reshape((len(data), np.prod(data.shape[1:])))
x_train=data[:5900]
x_test=data[5900:]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15453601434909847, 'fraction_numerical': 0.04880581516095535, 'mean_word_length': 4.975183305132544, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '652175', 'license': 'MIT', 'n_tokens_mistral': 4060, 'n_tokens_neox': 3614, 'n_words': 913, 'path': '07/.ipynb_checkpoints/MatteoGuardiani_07-checkpoint.ipynb', 'repo_name': 'mu-gaimann/machine-learning-in-fundamental-physics-2019', 'size': '10593'}