<jupyter_start><jupyter_text>Using checkpointsThis can help us save time in case of unexpected crashes and system errors.<jupyter_code>import os

cur_dir = os.getcwd()
checkpoint_dir = os.path.join(cur_dir, 'checkpoints', 'mnist_cnn_seq')
os.makedirs(checkpoint_dir, exist_ok=True)
checkpoint_prefix = os.path.join(checkpoint_dir, 'mnist_cnn_seq')<jupyter_output><empty_output><jupyter_text>Data Pipelining with one-hot encoding<jupyter_code># One-hot encoding
from tensorflow.keras.utils import to_categorical

# MNIST dataset
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()

# Normalization
X_train = X_train.astype(np.float32) / 255.
X_test = X_test.astype(np.float32) / 255.

# Convert it to 4D array (or we can use np.expand_dims for dimension expansion)
X_train = X_train[..., tf.newaxis]
X_test = X_test[..., tf.newaxis]

# one-hot encoding
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)

# Build dataset pipeline
train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(buffer_size=100000).batch(batch_size)
test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(batch_size)<jupyter_output><empty_output><jupyter_text>Building the model<jupyter_code>def create_model():
    model = tf.keras.Sequential(name='CNN_Sequential')
    model.add(tf.keras.layers.Flatten(input_shape=(28, 28)))
    model.add(tf.keras.layers.Dense(64, activation=tf.keras.activations.relu))
    model.add(tf.keras.layers.Dense(64, activation=tf.keras.activations.relu))
#     model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=(3, 3), activation=tf.keras.activations.relu,
#                                      padding='SAME', input_shape=(28, 28, 1)))
#     model.add(tf.keras.layers.MaxPool2D(padding='SAME'))
#     model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=(3, 3), activation=tf.keras.activations.relu,
#                                      padding='SAME'))
#     model.add(tf.keras.layers.MaxPool2D(padding='SAME'))
#     model.add(tf.keras.layers.Conv2D(filters=128, kernel_size=(3, 3), activation=tf.keras.activations.relu,
#                                      padding='SAME'))
#     model.add(tf.keras.layers.MaxPool2D(padding='SAME'))
#     model.add(tf.keras.layers.Flatten())
#     model.add(tf.keras.layers.Dense(256, activation=tf.keras.activations.relu))
#     model.add(tf.keras.layers.Dropout(0.4))
    model.add(tf.keras.layers.Dense(118, activation=tf.keras.activations.relu))
    model.add(tf.keras.layers.Dense(10))
    return model

# Create model
model = create_model()
# model.summary()<jupyter_output><empty_output><jupyter_text>Loss, gradient and optimizer functions<jupyter_code># Loss function
def loss_fn(model, images, labels):
#     logits = model(images, training=True)
#     loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels))
    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    return loss

# Gradient Function
def grad(model, images, labels):
    with tf.GradientTape() as tape:
        loss = loss_fn(model, images, labels)
    return tape.gradient(loss, model.trainable_variables)
# Optimizer
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

# Evaluation function
def evaluate(model, images, labels):
    logits = model(images, training=False)
    correct_predict = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_predict, tf.float32))
    return accuracy

# Checkpoint
checkpoint = tf.train.Checkpoint(cnn=model)<jupyter_output><empty_output><jupyter_text>Data Augmentation<jupyter_code>from scipy import ndimage
import random

def data_augmentation(images, labels):
    aug_images = []
    aug_labels = []
    
    for image, label in zip(images, labels):
        aug_images.append(image)
        aug_labels.append(label)
        
        # Background image for filling empty pixel
        bg_value = np.median(image)
        for _ in range(4):
            # Rotation
            rot_image = ndimage.rotate(image, angle=random.randint(-15, 15), 
                                       reshape=False, cval=bg_value)
            # Shift
            shift_image = ndimage.shift(rot_image, shift=np.random.randint(-2, 2, 2), 
                                        cval=bg_value)
            
            aug_images.append(shift_image)
            aug_labels.append(label)
    aug_images = np.array(aug_images)
    aug_labels = np.array(aug_labels)
    return aug_images, aug_labels
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
X_train, y_train = data_augmentation(X_train, y_train)

# Convert numpy float type and normalize it
X_train = X_train.astype(np.float32) / 255.
X_test = X_test.astype(np.float32) / 255.

# Convert it to 4D array (or we can use np.expand_dims for dimension expansion)
X_train = X_train[..., tf.newaxis]
X_test = X_test[..., tf.newaxis]

# one-hot encoding
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)

# Build dataset pipeline
train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(buffer_size=500000).batch(batch_size)
test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(batch_size)<jupyter_output><empty_output><jupyter_text>Training and Validation<jupyter_code>for e in range(training_epochs):
    avg_loss = 0.
    avg_train_acc = 0.
    avg_test_acc = 0.
    train_step = 0
    test_step = 0
    
    for images, labels in train_ds:
        grads = grad(model, images, labels)
        optimizer.apply_gradients(zip(grads, model.trainable_variables))
        loss = loss_fn(model, images, labels)
        acc = evaluate(model, images, labels)
        avg_loss = avg_loss + loss
        avg_train_acc = avg_train_acc + acc
        train_step += 1
    avg_loss = avg_loss / train_step
    avg_train_acc = avg_train_acc / train_step
    
    for images, labels in test_ds:
        acc = evaluate(model, images, labels)
        avg_test_acc = avg_test_acc + acc
        test_step += 1
    avg_test_acc = avg_test_acc / test_step
    
    print("Epoch: {}".format(e + 1),
          "loss: {:.8f}".format(avg_loss),
          "train acc: {:.4f}".format(avg_train_acc),
          "test acc: {:.4f}".format(avg_test_acc))
    
    checkpoint.save(file_prefix=checkpoint_prefix)<jupyter_output><empty_output><jupyter_text>sandbox below<jupyter_code>for images, labels in test_ds:
    acc = evaluate(model, images, labels)
    avg_test_acc = avg_test_acc + acc
    test_step += 1
avg_test_acc = avg_test_acc / test_step
print("test acc: {:.4f}".format(avg_test_acc))
a_weights = model.get_weights()
i = 1
for _layer in a_weights:
    b_weights = []
    for j in _layer[0][0]:
        b_weights.append(j[0])
    np.savetxt("layer"+ str(i) +".csv", b_weights, delimiter=",")
    i = i + 1
np.shape(model.get_weights()[6])
model.get_weights()[0]
model.compile(
    optimizer=tf.keras.optimizers.Adam(0.001),
    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
)
model.fit(
    train_ds,
    epochs=6,
    validation_data=test_ds,
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1242898711375918, 'fraction_numerical': 0.016765969239296106, 'mean_word_length': 3.8248663101604277, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '173720', 'license': 'MIT', 'n_tokens_mistral': 2689, 'n_tokens_neox': 2575, 'n_words': 435, 'path': 'python/CNN.ipynb', 'repo_name': 'catFurr/majorproj', 'size': '7217'}