<jupyter_start><jupyter_text>Easily export jupyter cells to python modulehttps://github.com/fastai/course-v3/blob/master/nbs/dl2/notebook2script.py<jupyter_code>! python /tf/src/scripts/notebook2script.py visualization.ipynb
%matplotlib inline
! pip install -U scikit-learn seaborn
#export
from exp.nb_clustering import *
import matplotlib.pyplot as plt
import matplotlib
import seaborn as sns
from data import D2VEmbedder
cd /tf/src/data/features<jupyter_output>/tf/src/data/features<jupyter_text>Generate all the feature vectors(Skip if already done)<jupyter_code>embdr = D2VEmbedder("/tf/src/data/doc2vec/model")
# Generate and Save Human Features
hman_dict = embdr("/tf/src/data/methods/DATA00M_[god-r]/test")

with open('hman_features.pickle', 'wb') as f:
    pickle.dump(hman_dict, f, protocol=pickle.HIGHEST_PROTOCOL)
# Generate and Save GPT-2 Pretrained Features
m1_dict = embdr("/tf/src/data/samples/unconditional/m1_example")

with open('m1_features.pickle', 'wb') as f:
    pickle.dump(m1_dict, f, protocol=pickle.HIGHEST_PROTOCOL)<jupyter_output><empty_output><jupyter_text>Read in Feature Vectors<jupyter_code>models_path = "/tf/src/data/features/output_space"
models_features = load_features(models_path)
# models_features[0] = models_features[1]<jupyter_output><empty_output><jupyter_text>Visualize Features<jupyter_code>models_clusters = cluster(models_features, k_range = [2, 3, 4, 5])
models_clusters[0][0].shape
len(list(models_features[0].values()))
colours = np.array(list(range(1, 10)))
colours
for i, feature_dict in enumerate(models_features):
    feature_vectors = reduce_dims(np.array(list(feature_dict.values())[:100]), 2)
    labels, centroids, kmeans = models_clusters[i]
#     kmeans = KMeans(init='k-means++', n_clusters=5, n_init=10)
#     kmeans.fit(feature_vectors)
    # Step size of the mesh. Decrease to increase the quality of the VQ.
    h = .02     # point in the mesh [x_min, x_max]x[y_min, y_max].

    # Plot the decision boundary. For that, we will assign a color to each
    x_min, x_max = feature_vectors[:, 0].min() - 1, feature_vectors[:, 0].max() + 1
    y_min, y_max = feature_vectors[:, 1].min() - 1, feature_vectors[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    
    Z = kmeans.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    plt.figure(1)
    plt.clf()
    plt.imshow(Z, interpolation='nearest',
               extent=(xx.min(), xx.max(), yy.min(), yy.max()),
               cmap=plt.cm.Paired,
               aspect='auto', origin='lower')

    plt.plot(feature_vectors[:, 0], feature_vectors[:, 1], 'k.', markersize=2)
    # Plot the centroids as a white X
    centroids = kmeans.cluster_centers_
    plt.scatter(centroids[:, 0], centroids[:, 1],
                marker='x', s=169, linewidths=3,
                color='w', zorder=10)
    plt.title('K-means clustering on the digits dataset (PCA-reduced data)\n'
              'Centroids are marked with white cross')
    plt.xlim(x_min, x_max)
    plt.ylim(y_min, y_max)
    plt.xticks(())
    plt.yticks(())
    plt.show()
def setup_data(clusters):
    feature_vectors, _, centroids, kmeans = clusters
    # Step size of the mesh. Decrease to increase the quality of the VQ.
    h = .02     # point in the mesh [x_min, x_max]x[y_min, y_max].

    # Plot the decision boundary. For that, we will assign a color to each
    x_min, x_max = feature_vectors[:, 0].min() - 1, feature_vectors[:, 0].max() + 1
    y_min, y_max = feature_vectors[:, 1].min() - 1, feature_vectors[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    
    Z = kmeans.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    
    return feature_vectors, centroids, xx, yy, Z
def plot_features(models_features):
    plt.figure(figsize=(9, 3))
    plt.clf()
    for i, clusters in enumerate(models_clusters):
        # Setup data to be plotted
        feature_vectors, centroids, xx, yy, Z = setup_data(clusters)
        
        # Plot data
        plt.subplot(1, 2, 1 + i)
        plt.imshow(Z, interpolation='nearest',
               extent=(xx.min(), xx.max(), yy.min(), yy.max()),
               cmap=plt.cm.Paired,
               aspect='auto', origin='lower')

        plt.plot(feature_vectors[:, 0], feature_vectors[:, 1], 'k.', markersize=2)
        # Plot the centroids as a white X
        plt.scatter(centroids[:, 0], centroids[:, 1],
                    marker='x', s=169, linewidths=3,
                    color='w', zorder=10)
        plt.title('K-means clustering on the digits dataset\n(PCA-reduced data)\n'
                  'Centroids are marked with white cross')
        plt.xlim(xx.min(), xx.max())
        plt.ylim(yy.min(), yy.max())
        plt.xticks(())
        plt.yticks(())
    plt.show()
plot_features(models_features)
colmap = {0: 'r',
          1: 'g',
          2: 'b',
          3: 'k',
          4: 'y',
          5: 'm',
          6: 'c'
         }
plt.figure(figsize=(9, 3))
for i, feature_dict in enumerate(models_features):
#     if i == 1: break
    
    feature_vectors = np.array(list(feature_dict.values())[:100])
    
    # Step size of the mesh. Decrease to increase the quality of the VQ.
    h = .02     # point in the mesh [x_min, x_max]x[y_min, y_max].

    # Plot the decision boundary. For that, we will assign a color to each
    x_min, x_max = feature_vectors[:, 0].min() - 1, feature_vectors[:, 0].max() + 1
    y_min, y_max = feature_vectors[:, 1].min() - 1, feature_vectors[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    
    Z = kmeans.predict(np.c_[xx.ravel(), yy.ravel()])

#     colors = list(map(lambda x: colmap[x], models_clusters[i][0]))
    
    plt.subplot(1, 3, 1 + i)
    plt.scatter(feature_vectors[:, 0], feature_vectors[:, 1], color = colors, alpha=0.5, edgecolor='k')
#     for idx, centroid in enumerate(models_clusters[i][1]):
#         plt.scatter(*centroid, color=colmap[idx])
#     plt.xlim(-60, 50)
#     plt.ylim(-60, 50)
plt.show()
plt.figure(1)
plt.subplot(211)
plt.plot(t1, f(t1), 'bo', t2, f(t2), 'k')

plt.subplot(212)
plt.plot(t2, np.cos(2*np.pi*t2), 'r--')
plt.show()
kmeans = KMeans(n_clusters=3)

kmeans.fit(pca_features)

colmap = {1: 'r', 2: 'g', 3: 'b'}

labels = kmeans.predict(pca_features)
centroids = kmeans.cluster_centers_

print(labels)
print(centroids)
fig = plt.figure(figsize=(5, 5))

colors = list(map(lambda x: colmap[x+1], labels))

plt.scatter(pca_features[:, 0], pca_features[:, 1], color=colors, alpha=0.05, edgecolor='k')
for idx, centroid in enumerate(centroids):
    plt.scatter(*centroid, color=colmap[idx+1])
plt.xlim(-10, 10)
plt.ylim(-10, 10)
plt.show()<jupyter_output><empty_output><jupyter_text>T-SNE Only<jupyter_code>fig = plt.figure(figsize=(5, 5))

colors = list(map(lambda x: colmap[x+1], labels))
# print(colors)

plt.scatter(pca_features[:, 0], pca_features[:, 1], color=colors, alpha=0.1, edgecolor='k')
for idx, centroid in enumerate(centroids):
    plt.scatter(*centroid, color=colmap[idx+1])
plt.xlim(-10, 10)
plt.ylim(-10, 10)
plt.show()
tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
tsne_results = tsne.fit_transform(feature_vectors)
print(tsne_results.shape)
kmeans = KMeans(n_clusters=3)

kmeans.fit(tsne_results)

colmap = {1: 'r', 2: 'g', 3: 'b'}

tsne_labels = kmeans.predict(tsne_results)
tsne_centroids = kmeans.cluster_centers_

print(tsne_labels)
print(tsne_centroids)
fig = plt.figure(figsize=(5, 5))

colors = list(map(lambda x: colmap[x+1], labels))
# print(colors)

plt.scatter(tsne_results[:, 0], tsne_results[:, 1], color=colors, alpha=0.05, edgecolor='k')
for idx, centroid in enumerate(tsne_centroids):
    
    plt.scatter(*centroid, color=colmap[idx+1])
plt.xlim(-10, 10)
plt.ylim(-10, 10)
plt.show()
pca = PCA(n_components=50)
pca_features = pca.fit_transform(feature_vectors)

print(pca_features.shape)
tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
tsne_results = tsne.fit_transform(pca_features)

print(tsne_results.shape)
kmeans = KMeans(n_clusters=3)

kmeans.fit(tsne_results)

colmap = {1: 'r', 2: 'g', 3: 'b'}

tsne_labels = kmeans.predict(tsne_results)
tsne_centroids = kmeans.cluster_centers_

print(tsne_labels)
print(tsne_centroids)
fig = plt.figure(figsize=(5, 5))

colors = list(map(lambda x: colmap[x+1], tsne_labels))
# print(colors)

plt.scatter(tsne_results[:, 0], tsne_results[:, 1], color=colors, alpha=0.05, edgecolor='k')
for idx, centroid in enumerate(tsne_centroids):
    plt.scatter(*centroid, color=colmap[idx+1])
plt.xlim(-10, 10)
plt.ylim(-10, 10)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1593450939697913, 'fraction_numerical': 0.02651908220915485, 'mean_word_length': 3.917233560090703, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '646139', 'license': 'MIT', 'n_tokens_mistral': 3521, 'n_tokens_neox': 3321, 'n_words': 701, 'path': 'main/nbs/poc/visualization.ipynb', 'repo_name': 'jason424217/Artificial-Code-Gen', 'size': '8673'}