<jupyter_start><jupyter_text>Prepare MNIST Data<jupyter_code>mnist_path = os.path.join(DS_PATH, "mnist")
inputs, targets = mnist_loader.load(mnist_path)
selector = (targets==0) | (targets==1)
new_inputs = inputs[selector].astype("float32")/255.0
new_targets = targets[selector]
in_shape = new_inputs.shape
new_inputs = new_inputs.reshape(in_shape[0], 28, 28, 1)
plt.imshow(new_inputs[20].reshape(28, 28))
x_train, x_test, y_train, y_test = train_test_split(new_inputs, new_targets)
x_test, x_val, y_test, y_val = train_test_split(x_test, y_test)
train_of_class_0 = (y_train == 0).sum()
test_of_class_0 = (y_test == 0).sum()
val_of_class_0 = (y_val == 0).sum()
print("Dist-Train")
num_samples_train = y_train.shape[0]
perc_class_0 = train_of_class_0/num_samples_train
print("class 0: ", str(perc_class_0))
print("Class 1: ", str(1-perc_class_0))

print("\nDist-Test")
num_samples_test = y_test.shape[0]
perc_class_0 = test_of_class_0/num_samples_test
print("Class 0: ", str(perc_class_0))
print("Class 1: ", str(1-perc_class_0))

print("\nDist-Val")
num_samples_val = y_val.shape[0]
perc_class_0 = val_of_class_0/num_samples_val
print("Class 0: ", str(perc_class_0))
print("Class 1: ", str(1-perc_class_0))
print("Train-shape: ", x_train.shape, x_train.dtype)
print("Test-shape: ", x_test.shape, x_test.dtype)<jupyter_output>Train-shape:  (11085, 28, 28, 1) float32
Test-shape:  (2771, 28, 28, 1) float32<jupyter_text>Perform moment propagation<jupyter_code>import tensorflow as tf
from tensorflow import keras
import tensorflow.keras
from tensorflow.keras import Sequential
from tensorflow.keras.layers import  Conv2D, MaxPool2D, Dropout, Flatten, Dense, Softmax
gpus = tf.config.experimental.list_physical_devices("GPU")
cpus = tf.config.experimental.list_physical_devices("CPU")

if gpus:
    try:
        for gpu in gpus:
            tf.config.experimental.set_memory_growth(gpu, True)
            
        logical_gpus = tf.config.experimental.list_logical_devices("GPU")
        print(len(gpus), "Physical GPU's, ", len(logical_gpus), "Logical GPU's")
        
    except RuntimeError as e:
        print(e)
        
elif cpus:
    try:
        logical_cpus = tf.config.experimental.list_logical_devices("CPU")
        print(len(cpus), "Physical CPU,", len(logical_cpus), "Logical CPU")
        
    except RuntimeError as e:
        print(e)
        
        
tfk = tf.keras
tfk.backend.clear_session()
input_shape = (28, 28, 1)
input_shape = (28, 28, 1)
model = Sequential([
    Conv2D(128, 4,activation="relu", input_shape=input_shape),
    MaxPool2D(),
    Dropout(.2),
    Conv2D(64, 3, activation="relu"),
    MaxPool2D(),
    Dropout(.2),
    Flatten(),
    Dense(512, activation="relu"),
    Dense(256, activation="relu"),
    Dense(128, activation="relu"),
    Dense(1, activation="sigmoid")
])

model.build(input_shape)
model.summary()
#optimizer = keras.optimizers.Adam(learning_rate=0.0001)
optimizer = keras.optimizers.Adadelta(learning_rate=0.001)
model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=[keras.metrics.binary_accuracy])
history = model.fit(
    x=x_train,
    y=y_train,
    validation_data=(x_val, y_val),
    batch_size=40,
    epochs=100
)
def plot_define(x_label, y_label="", legend=False):
    plt.xlabel(x_label)
    plt.ylabel(y_label)
# T-sne Plot
plt.figure(figsize=(15, 8))

plt.subplot(2, 2, 1)
plt.plot(history.history["binary_accuracy"], label="accuracy")
plt.plot(history.history["val_binary_accuracy"], label="val_accuracy")
plot_define("Epochs", "Accuracy")
plt.legend()

plt.subplot(2, 2, 2)
plt.plot(history.history["loss"], label="loss")
plt.plot(history.history["val_loss"], label="val_loss")
plot_define("Epochs", "Loss")
plt.legend()
import bayesian.utils as butils
importlib.reload(butils)
n_predictions = butils.batch_predict_n(model, x_test, 50)
n_predictions[0][:20]
measure_results = butils.measures(n_predictions, batch_dim=True)
avg, var = measure_results
print(avg.shape)
print(var.shape)
alt_class_prob = np.repeat(1, 2)
alt_class_prob
measure_data_stacked = np.hstack([avg, 1-avg, var, np.expand_dims(y_test, axis=1)])
measure_data_stacked.shape
print(measure_data_stacked.shape)
avg_df = pd.DataFrame(measure_data_stacked, columns=["class_0", "class_1", "var", "target"])
avg_df.head()
avg_df.shape
import seaborn as sns
plt.figure(figsize=(15, 10))
sns.scatterplot(data=avg_df, y="class_0", x="class_1", hue="var", style="target")
import importlib
importlib.reload(mp)

mp_model = mp.MP()
mp_model.create_MP_Model(model=model, use_mp=True, verbose=True)
pred_mp,var_mp = mp_model.model.predict(x_test)
var = np.sum(var_mp, axis=1)/2
columns = ["class_0", "class_1", "var", "target"]
y_len = len(y_test)
mp_df = pd.DataFrame(np.hstack([pred_mp, 1-pred_mp, var.reshape(y_len, 1), y_test.reshape(y_len, 1)]), columns=columns)
mp_df.head()
sns.scatterplot(data=mp_df, y="class_0", x="class_1", hue="var", style="target")
import active_learning
importlib.reload(active_learning)
from active_learning import Checkpoint
mc_checkpoint = Checkpoint("mc_dropout")
mc_checkpoint.new(model)
mp_checkpoint = Checkpoint("mp_checkpoint")
mp_checkpoint.new(mp_model.create_MP_Model(model))<jupyter_output>/home/exleonem/Desktop/workspace/thesis/wp/models/checkpoints/mp_checkpoint/mp_checkpoint_14_05_21_10_33_36.h5
{'fraction_non_alphanumeric': 0.14742619227857684, 'fraction_numerical': 0.03217259651778955, 'mean_word_length': 6.027925531914893, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '311873', 'license': 'MIT', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2029, 'n_words': 336, 'path': 'wp/notebooks/model/tf_moment_propagation.ipynb', 'repo_name': 'ExLeonem/master-thesis-code', 'size': '5284'}