<jupyter_start><jupyter_text>**VAE for Breast Cancer Dataset**<jupyter_code>from sklearn.datasets import load_breast_cancer

# Import dataset
breast_cancer_dataset = load_breast_cancer()
data = breast_cancer_dataset['data']
labels = breast_cancer_dataset['target']
target_names = breast_cancer_dataset['target_names']
feature_names = breast_cancer_dataset['feature_names']

# Split test/train
data_train = data[:500,]
labels_train = labels[:500,]
data_test = data[500:,]
labels_test = labels[500:,]

# Print out key stats
print(f'Number of data samples: {data.shape[0]}')
print(f'Number of features: {len(feature_names)}')
# Preliminaries
import tensorflow as tf
from tensorflow.layers import dense
import numpy as np

seed = 11

def accuracy(guesses, labels):
    return np.mean([g == l for g, l in zip(guesses, labels)])<jupyter_output><empty_output><jupyter_text>**Define Supervised Graph**<jupyter_code>g_super = tf.Graph()
with g_super.as_default():
    
    # Set tf seed
    tf.set_random_seed(seed)
    
    # Inputs
    x_super = tf.placeholder(tf.float32, shape=[None, 30], name='x')
    y_super = tf.placeholder(tf.int32, shape=[None,], name='y')

    # Model
    logits = dense(dense(inputs=x_super, activation='relu', units=30), activation=None, units=2)
    y_hat_super = tf.argmax(logits, 1)

    # Loss
    cost = tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=y_super)
    optimizer = tf.train.AdamOptimizer(learning_rate=1e-3).minimize(cost)

    # Summaries
    tf.summary.scalar("Total_Loss", cost)
    merged_super = tf.summary.merge_all()
    
    # Saver
    supervised_saver = tf.train.Saver()<jupyter_output><empty_output><jupyter_text>**Run Supervised Training on Supervised Graph**<jupyter_code>np.random.seed(seed)
with tf.Session(graph=g_super) as sess:
    # Initialize variables and saver and Tensorboard writer
    writer = tf.summary.FileWriter('./supervised', g_super)
    tf.global_variables_initializer().run()

    for step in range(2501):
        # Generate training batches
        indexes = np.random.randint(low=0, high=data_train.shape[0]-1, size=250)
        feed_dict = {x_super: data_train[indexes], y_super: labels_train[indexes]}

        # Training iteration
        summary, y_hatt, _ = sess.run([merged_super, y_hat_super, optimizer], feed_dict=feed_dict)
        if step % 500 == 0:
            print(f'Accuracy of supervised model on train set at {step} iterations: {accuracy(y_hatt, labels[indexes])}')
        writer.add_summary(summary=summary, global_step=step)
    
    # Save model weights
    save_path = supervised_saver.save(sess, "./supervised/model.ckpt")
    print("Model saved in path: %s" % save_path)<jupyter_output>Accuracy of supervised model on train set at 0 iterations: 0.592
Accuracy of supervised model on train set at 500 iterations: 0.888
Accuracy of supervised model on train set at 1000 iterations: 0.932
Accuracy of supervised model on train set at 1500 iterations: 0.944
Accuracy of supervised model on train set at 2000 iterations: 0.98
Accuracy of supervised model on train set at 2500 iterations: 0.96
Model saved in path: ./supervised/model.ckpt<jupyter_text>**Evaluate Trained Supervised Model on Test Set**<jupyter_code>with tf.Session(graph=g_super) as sess:
    # Initialize variables and Tensorboard writer
    tf.global_variables_initializer().run()
    supervised_saver.restore(sess, "./supervised/model.ckpt")
    guesses = sess.run([y_hat_super], feed_dict={x_super: data_test})[0]
print(f'Accuracy of supervised model on test set: {accuracy(guesses, labels_test)}')<jupyter_output>INFO:tensorflow:Restoring parameters from ./supervised/model.ckpt
Accuracy of supervised model on test set: 0.9855072463768116<jupyter_text>**Define Semi-Supervised Graph**<jupyter_code>g_semi = tf.Graph()
with g_semi.as_default():
    
    # Set tf seed
    tf.set_random_seed(seed)
    
    # Inputs
    x_semi = tf.placeholder(tf.float32, shape=[None, 30], name='x')
    y_semi = tf.placeholder(tf.int32, shape=[None,], name='y')
    eps = tf.placeholder(tf.float32, shape=[None, 10], name='eps')

    # Unsupervised Model
    with tf.variable_scope("unsupervised"):
        mu = dense(inputs=x_semi, activation='relu', units=10)
        sigma = dense(inputs=x_semi, activation='relu', units=10)
        z = mu + sigma * eps
        x_hat = dense(dense(inputs=z, units=10), units=30)

    # Supervised Model ON TOP of Unsupervised Latent Variables
    with tf.variable_scope("supervised"):
        logits = dense(inputs=z, activation=None, units=2)
        y_hat_semi = tf.argmax(logits, 1)
        
    # Unsupervised Loss
    unsupervised_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "unsupervised")
    recon = tf.reduce_sum(tf.squared_difference(x_semi, x_hat))
    vae = -0.5 * tf.reduce_sum(1.0 - tf.square(mu) - tf.square(sigma) + 2.0 * tf.log(sigma + 1e-8))
    vae_cost = tf.reduce_sum(recon + 0.01 * vae)
    vae_optimizer = tf.train.AdamOptimizer(learning_rate=1e-3).minimize(vae_cost, var_list=unsupervised_vars)
    
    # Supervised Loss
    supervised_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "supervised")
    super_cost = tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=y_semi)
    super_optimizer = tf.train.AdamOptimizer(learning_rate=8e-3).minimize(super_cost, var_list=supervised_vars)

    # Summaries
    tf.summary.scalar("Unsuper_Vae_loss", vae)
    tf.summary.scalar("Unsuper_Recon_loss", recon)
    tf.summary.scalar("Unsuper_Total_loss", vae_cost)
    tf.summary.scalar("Super_Total_loss", super_cost)
    merged_semi = tf.summary.merge_all()
        
    # Saver
    semi_saver = tf.train.Saver()<jupyter_output><empty_output><jupyter_text>**Run Unsupervised Training on Semi-Supervised Graph**<jupyter_code>np.random.seed(seed)
with tf.Session(graph=g_semi) as sess:
    # Initialize variables and Tensorboard writer
    writer = tf.summary.FileWriter('./vae', g_semi)
    tf.global_variables_initializer().run()

    for step in range(10001):
        # Generate training batches
        epsilon = np.random.normal(size=(250, 10))
        indexes = np.random.randint(low=0, high=data_train.shape[0]-1, size=250)
        feed_dict = {x_semi: data_train[indexes], y_semi: labels_train[indexes], eps: epsilon}

        # Training iteration
        summary, y_hatt, _ = sess.run([merged_semi, y_hat_semi, vae_optimizer], feed_dict=feed_dict)
        if step % 1000 == 0:
            print(f'Accuracy of unsupervised model on train set at {step} iterations: {accuracy(y_hatt, labels[indexes])}')
        writer.add_summary(summary=summary, global_step=step)
        
    save_path = semi_saver.save(sess, "./vae/model.ckpt")
    print("Model saved in path: %s" % save_path)<jupyter_output>Accuracy of unsupervised model on train set at 0 iterations: 0.572
Accuracy of unsupervised model on train set at 1000 iterations: 0.632
Accuracy of unsupervised model on train set at 2000 iterations: 0.768
Accuracy of unsupervised model on train set at 3000 iterations: 0.336
Accuracy of unsupervised model on train set at 4000 iterations: 0.344
Accuracy of unsupervised model on train set at 5000 iterations: 0.416
Accuracy of unsupervised model on train set at 6000 iterations: 0.368
Accuracy of unsupervised model on train set at 7000 iterations: 0.42
Accuracy of unsupervised model on train set at 8000 iterations: 0.356
Accuracy of unsupervised model on train set at 9000 iterations: 0.472
Accuracy of unsupervised model on train set at 10000 iterations: 0.36
Model saved in path: ./vae/model.ckpt<jupyter_text>**Run Supervised Training on Semi-Supervised Graph**<jupyter_code>np.random.seed(seed)
with tf.Session(graph=g_semi) as sess:
    # Initialize variables and Tensorboard writer
    writer = tf.summary.FileWriter('./semisupervised', g_semi)
    tf.global_variables_initializer().run()
    semi_saver.restore(sess, "./vae/model.ckpt")

    for step in range(2501):
        # Generate training batches
        epsilon = np.random.normal(size=(250, 10))
        indexes = np.random.randint(low=0, high=data_train.shape[0]-1, size=250)
        feed_dict = {x_semi: data_train[indexes], y_semi: labels_train[indexes], eps: epsilon}

        # Training iteration
        summary, y_hatt, _ = sess.run([merged_semi, y_hat_semi, super_optimizer], feed_dict=feed_dict)
        if step % 500 == 0:
            print(f'Accuracy of supervised model on train set at {step} iterations: {accuracy(y_hatt, labels[indexes])}')
        writer.add_summary(summary=summary, global_step=step)
    
    save_path = semi_saver.save(sess, "./semisupervised/model.ckpt")
    print("Model saved in path: %s" % save_path)<jupyter_output>INFO:tensorflow:Restoring parameters from ./vae/model.ckpt
Accuracy of supervised model on train set at 0 iterations: 0.416
Accuracy of supervised model on train set at 500 iterations: 0.944
Accuracy of supervised model on train set at 1000 iterations: 0.94
Accuracy of supervised model on train set at 1500 iterations: 0.908
Accuracy of supervised model on train set at 2000 iterations: 0.96
Accuracy of supervised model on train set at 2500 iterations: 0.92
Model saved in path: ./semisupervised/model.ckpt<jupyter_text>**Evaluate Trained Semi-Supervised Model on Test Set**<jupyter_code>with tf.Session(graph=g_semi) as sess:
    # Initialize variables and Tensorboard writer
    tf.global_variables_initializer().run()
    semi_saver.restore(sess, "./semisupervised/model2.ckpt")
    epsilon = np.random.normal(size=(data_test.shape[0], 10))
    guesses = sess.run([y_hat_semi], feed_dict={x_semi: data_test, eps: epsilon})[0]
print(f'Accuracy of semi-supervised model on test set: {accuracy(guesses, labels_test)}')<jupyter_output>INFO:tensorflow:Restoring parameters from ./semisupervised/model2.ckpt
Accuracy of semi-supervised model on test set: 0.9855072463768116
{'fraction_non_alphanumeric': 0.1089561332250203, 'fraction_numerical': 0.03290008123476848, 'mean_word_length': 4.51147174034695, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '355851', 'license': 'MIT', 'n_tokens_mistral': 3624, 'n_tokens_neox': 3213, 'n_words': 802, 'path': 'example.ipynb', 'repo_name': 'dancsalo/Tensorflow-VAE', 'size': '9848'}