<jupyter_start><jupyter_text>INITIAL IMPORTS<jupyter_code>import os
# Generally TF gives tons of warnings along with the error to avoid that..
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import tensorflow as tf<jupyter_output><empty_output><jupyter_text>TENSOR BASICS<jupyter_code>x = tf.constant(4, shape=(1,1), dtype=float)
x
x = tf.constant([[1,2,3], [4,5,6]])
x
x = tf.ones((4,4))
x
x = tf.zeros((3,3))
x
x = tf.eye((3))
x
x = tf.random.normal((3,3), mean=0, stddev=1)
x
x = tf.random.uniform((1,3), minval=0, maxval=1)
x
x = tf.range(9)
x
x = tf.range(start=1, limit=20, delta=3)
x
x = tf.constant([1,2,3])
y = tf.constant([7,8,9])
z = tf.add(x,y)
z
x = tf.constant([1,2,3])
y = tf.constant([7,8,9])
z = tf.subtract(x,y)
z
x = tf.constant([1,2,3])
y = tf.constant([7,8,9])
z = tf.multiply(x,y)
z
x = tf.constant([1,2,3])
y = tf.constant([7,8,9])
z = tf.tensordot(x,y, axes=1)
z
x = tf.constant([1,2,3])
y = tf.constant([7,8,9])
z = tf.reduce_sum(x*y, axis=0)
z
x = tf.random.normal((2,3))
y = tf.random.normal((3,4))
z = tf.matmul(x,y)
z
x = tf.constant([[1,2],
                 [3,4],
                 [5,6]])
x[0,1]
x = tf.range(9)
x = tf.reshape(x, (3,3))
x = tf.transpose(x, perm=[1,0])
x<jupyter_output><empty_output><jupyter_text>DESIGN BASIC NN USING SEQUENTIAL API<jupyter_code>from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.datasets import mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
print(x_train.shape)

x_train = x_train.reshape(-1, 28*28).astype('float32') 
x_test = x_test.reshape(-1, 28*28).astype('float32') 
model = keras.Sequential(
    [
     keras.Input(shape=(28*28)),
     layers.Dense(512, activation='relu'),
     layers.Dense(256, activation='relu'),
     layers.Dense(10),
    ]

)
print(model.summary())
model.compile(
    
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=0.001),
    metrics = ['accuracy'],
)

model.fit(x_train, y_train, batch_size=32, epochs=5, verbose=2)
model.evaluate(x_test, y_test, batch_size=32, verbose=2)
model = keras.Sequential()
model.add(keras.Input(shape=(784))),
model.add(layers.Dense(512, activation='relu', name='first_layer')),
model.add(layers.Dense(256, activation='relu')),
model.add(layers.Dense(128, activation='relu')),
model.add(layers.Dense(64, activation='relu')),
model.add(layers.Dense(10))


print(model.summary())
model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=0.001),
    metrics = ['accuracy'],
)

model.fit(x_train, y_train, batch_size=32, epochs=8, verbose=2)

model.evaluate(x_test, y_test, batch_size=32, verbose=2)<jupyter_output>Model: "sequential_6"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
first_layer (Dense)          (None, 512)               401920    
_________________________________________________________________
dense_19 (Dense)             (None, 256)               131328    
_________________________________________________________________
dense_20 (Dense)             (None, 128)               32896     
_________________________________________________________________
dense_21 (Dense)             (None, 64)                8256      
_________________________________________________________________
dense_22 (Dense)             (None, 10)                650       
=================================================================
Total params: 575,050
Trainable params: 575,050
Non-trainable params: 0
________________________________________________[...]<jupyter_text>CNN<jupyter_code>from tensorflow import keras
from tensorflow.keras import layers, regularizers
from tensorflow.keras.datasets import cifar10
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype("float32") / 255.0
x_test = x_test.astype("float32") / 255.0
model = keras.Sequential([
    keras.Input(shape=(32,32,3)),
    layers.Conv2D(32, 3, activation='relu'),
    layers.MaxPooling2D(pool_size=(2,2)),
    layers.Conv2D(64, 3, activation='relu'),
    layers.MaxPooling2D(),
    layers.Conv2D(128, 3, activation='relu'),
    layers.Flatten(),
    layers.Dense(64, activation='relu'),
    layers.Dense(10),
])
model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=3e-4),
    metrics = ["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=15, verbose=2)

model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output><empty_output><jupyter_text>FUNCTIONAL API<jupyter_code>def my_model():
    inputs = keras.Input(shape=(32,32,3))
    x = layers.Conv2D(32, 3, padding='same')(inputs)
    x = layers.BatchNormalization()(x)
    x = keras.activations.relu(x)
    x = layers.MaxPool2D()(x)
    x = layers.Conv2D(64, 3, padding='same')(x)
    x = layers.BatchNormalization()(x)
    x = keras.activations.relu(x)
    x = layers.MaxPool2D()(x)
    x = layers.Conv2D(128, 3, padding='same')(x)
    x = layers.BatchNormalization()(x)
    x = keras.activations.relu(x)
    x = layers.Flatten()(x)
    x = layers.Dense(64, activation='relu')(x)    
    outputs = layers.Dense(10)(x)
    model = keras.Model(inputs = inputs, outputs = outputs) 
    return model

model = my_model()

model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=3e-4),
    metrics = ["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=15, verbose=2)

model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output><empty_output><jupyter_text>REGULARIZATION FOR OVERFITTING DATA<jupyter_code>def my_model():
    inputs = keras.Input(shape=(32,32,3))
    x = layers.Conv2D(
        32, 3, padding='same', kernel_regularizer=regularizers.l2(0.01),
                      )(inputs)
    x = layers.BatchNormalization()(x)
    x = keras.activations.relu(x)
    x = layers.MaxPool2D()(x)
    x = layers.Conv2D(
        64, 3, padding='same', kernel_regularizer=regularizers.l2(0.01),
        )(x)
    x = layers.BatchNormalization()(x)
    x = keras.activations.relu(x)
    x = layers.MaxPool2D()(x)
    x = layers.Conv2D(
        128, 3, padding='same', kernel_regularizer=regularizers.l2(0.01),
        )(x)
    x = layers.BatchNormalization()(x)
    x = keras.activations.relu(x)
    x = layers.Flatten()(x)
    x = layers.Dense(
        64, activation='relu', kernel_regularizer=regularizers.l2(0.01),
        )(x)
    x = layers.Dropout(0.5)(x)    
    outputs = layers.Dense(10)(x)
    model = keras.Model(inputs = inputs, outputs = outputs) 
    return model

model = my_model()

model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=3e-4),
    metrics = ["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=150, verbose=2)

model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output><empty_output><jupyter_text>RNN, GRU, LSTM<jupyter_code>from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.datasets import mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
print(x_train.shape)

x_train = x_train.astype('float32') 
x_test = x_test.astype('float32') 
model = keras.Sequential()
model.add(keras.Input(shape=(None, 28)))
model.add(layers.SimpleRNN(512, return_sequences=True, activation='relu'))
model.add(layers.SimpleRNN(512, activation='relu'))
model.add(layers.Dense(10))

print(model.summary())
model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=0.001),
    metrics = ["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=10, verbose=2)

model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
simple_rnn (SimpleRNN)       (None, None, 512)         276992    
_________________________________________________________________
simple_rnn_1 (SimpleRNN)     (None, 512)               524800    
_________________________________________________________________
dense (Dense)                (None, 10)                5130      
=================================================================
Total params: 806,922
Trainable params: 806,922
Non-trainable params: 0
_________________________________________________________________
None
Epoch 1/10
938/938 - 34s - loss: 1.0213 - accuracy: 0.6945
Epoch 2/10
938/938 - 32s - loss: 0.3255 - accuracy: 0.8994
Epoch 3/10
938/938 - 31s - loss: 0.2171 - accuracy: 0.9409
Epoch 4/10
938/938 - 31s - loss: 0.1731 - accuracy: 0.9532
Epoch 5[...]<jupyter_text>GRUs<jupyter_code>model = keras.Sequential()
model.add(keras.Input(shape=(None, 28)))
model.add(layers.GRU(256, return_sequences=True, activation='relu'))
model.add(layers.GRU(256, activation='relu'))
model.add(layers.Dense(10))

print(model.summary())
model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=0.001),
    metrics = ["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=10, verbose=2)

model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output>WARNING:tensorflow:Layer gru will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer gru_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
gru (GRU)                    (None, None, 256)         219648    
_________________________________________________________________
gru_1 (GRU)                  (None, 256)               394752    
_________________________________________________________________
dense_1 (Dense)              (None, 10)                2570      
=================================================================
Total params: 616,970
Trainable params: 616,970
[...]<jupyter_text>LSTM<jupyter_code>model = keras.Sequential()
model.add(keras.Input(shape=(None, 28)))
model.add(layers.LSTM(256, return_sequences=True, activation='relu'))
model.add(layers.LSTM(256, activation='relu'))
model.add(layers.Dense(10))

print(model.summary())
model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=0.001),
    metrics = ["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=10, verbose=2)

model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output>WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
Model: "sequential_2"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm (LSTM)                  (None, None, 256)         291840    
_________________________________________________________________
lstm_1 (LSTM)                (None, 256)               525312    
_________________________________________________________________
dense_2 (Dense)              (None, 10)                2570      
=================================================================
Total params: 819,722
Trainable params: 819,72[...]<jupyter_text>BIDIRECTIONAL LSTM<jupyter_code>model = keras.Sequential()
model.add(keras.Input(shape=(None, 28)))
model.add(layers.Bidirectional(layers.LSTM(256, return_sequences=True, activation='relu')))
model.add(layers.Bidirectional(layers.LSTM(256, activation='relu')))
model.add(layers.Dense(10))

print(model.summary())
model.compile(
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adam(learning_rate=0.001),
    metrics = ["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=10, verbose=2)

model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output>WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when r[...]<jupyter_text>ADDITIONAL EXAMPLE OF FUNCTIONAL API ON MULTI DIGIT MNIST DATASET<jupyter_code>!unzip /content/drive/MyDrive/TFBASICS/MDMNIST.zip
#Change paths accordingly..
import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, regularizers
from tensorflow.keras.datasets import mnist

# Use Pandas to load dataset from csv file
import pandas as pd

# HYPERPARAMETERS
BATCH_SIZE = 64
WEIGHT_DECAY = 0.001
LEARNING_RATE = 0.001

# Make sure we don't get any GPU errors, no need if you are using Google Colab
physical_devices = tf.config.list_physical_devices("GPU")
tf.config.experimental.set_memory_growth(physical_devices[0], True)

train_df = pd.read_csv("/content/train.csv")
test_df = pd.read_csv("/content/test.csv")
train_images = "/content/train_images/" + train_df.iloc[:, 0].values
test_images = "/content/test_images/" + test_df.iloc[:, 0].values

train_labels = train_df.iloc[:, 1:].values
test_labels = test_df.iloc[:, 1:].values


def read_image(image_path, label):
    image = tf.io.read_file(image_path)
    image = tf.image.decode_image(image, channels=1, dtype=tf.float32)

    # In older versions you need to set shape in order to avoid error
    # on newer (2.3.0+) the following 3 lines can safely be removed
    image.set_shape((64, 64, 1))
    label[0].set_shape([])
    label[1].set_shape([])

    labels = {"first_num": label[0], "second_num": label[1]}
    return image, labels


AUTOTUNE = tf.data.experimental.AUTOTUNE
train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels))
train_dataset = (
    train_dataset.shuffle(buffer_size=len(train_labels))
    .map(read_image)
    .batch(batch_size=BATCH_SIZE)
    .prefetch(buffer_size=AUTOTUNE)
)

test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels))
test_dataset = (
    test_dataset.map(read_image)
    .batch(batch_size=BATCH_SIZE)
    .prefetch(buffer_size=AUTOTUNE)
)

inputs = keras.Input(shape=(64, 64, 1))
x = layers.Conv2D(
    filters=32,
    kernel_size=3,
    padding="same",
    kernel_regularizer=regularizers.l2(WEIGHT_DECAY),
)(inputs)
x = layers.BatchNormalization()(x)
x = keras.activations.relu(x)
x = layers.Conv2D(64, 3, kernel_regularizer=regularizers.l2(WEIGHT_DECAY),)(x)
x = layers.BatchNormalization()(x)
x = keras.activations.relu(x)
x = layers.MaxPooling2D()(x)
x = layers.Conv2D(
    64, 3, activation="relu", kernel_regularizer=regularizers.l2(WEIGHT_DECAY),
)(x)
x = layers.Conv2D(128, 3, activation="relu")(x)
x = layers.MaxPooling2D()(x)
x = layers.Flatten()(x)
x = layers.Dense(128, activation="relu")(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(64, activation="relu")(x)
output1 = layers.Dense(10, activation="softmax", name="first_num")(x)
output2 = layers.Dense(10, activation="softmax", name="second_num")(x)
model = keras.Model(inputs=inputs, outputs=[output1, output2])

model.compile(
    optimizer=keras.optimizers.Adam(LEARNING_RATE),
    loss=keras.losses.SparseCategoricalCrossentropy(),
    metrics=["accuracy"],
)

model.fit(train_dataset, epochs=5, verbose=2)
model.evaluate(test_dataset, verbose=2)<jupyter_output>Epoch 1/5
1000/1000 - 33s - loss: 1.7513 - first_num_loss: 0.8472 - second_num_loss: 0.8236 - first_num_accuracy: 0.7005 - second_num_accuracy: 0.7139
Epoch 2/5
1000/1000 - 27s - loss: 0.5215 - first_num_loss: 0.2282 - second_num_loss: 0.2246 - first_num_accuracy: 0.9253 - second_num_accuracy: 0.9282
Epoch 3/5
1000/1000 - 28s - loss: 0.3395 - first_num_loss: 0.1399 - second_num_loss: 0.1377 - first_num_accuracy: 0.9556 - second_num_accuracy: 0.9564
Epoch 4/5
1000/1000 - 28s - loss: 0.2717 - first_num_loss: 0.1091 - second_num_loss: 0.1050 - first_num_accuracy: 0.9650 - second_num_accuracy: 0.9666
Epoch 5/5
1000/1000 - 27s - loss: 0.2349 - first_num_loss: 0.0932 - second_num_loss: 0.0872 - first_num_accuracy: 0.9704 - second_num_accuracy: 0.9725
313/313 - 6s - loss: 0.9557 - first_num_loss: 0.2400 - second_num_loss: 0.6635 - first_num_accuracy: 0.9284 - second_num_accuracy: 0.8163<jupyter_text>MODEL SUBCLASSING WITH KERAS<jupyter_code>import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.datasets import mnist

physical_devices = tf.config.list_physical_devices("GPU")
tf.config.experimental.set_memory_growth(physical_devices[0], True)

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(-1, 28, 28, 1).astype("float32") / 255.0
x_test = x_test.reshape(-1, 28, 28, 1).astype("float32") / 255.0

# CNN -> BatchNorm -> ReLU (common structure)
# x10 (a lot of code to write!)
class CNNBlock(layers.Layer):
    def __init__(self, out_channels, kernel_size=3):
        super(CNNBlock, self).__init__()
        self.conv = layers.Conv2D(out_channels, kernel_size, padding="same")
        self.bn = layers.BatchNormalization()

    def call(self, input_tensor, training=False):
        x = self.conv(input_tensor)
        x = self.bn(x, training=training)
        x = tf.nn.relu(x)
        return x

# model = keras.Sequential(
#     [CNNBlock(32), CNNBlock(64), CNNBlock(128), layers.Flatten(), layers.Dense(10),]
# )
# model.compile(
#     optimizer=keras.optimizers.Adam(lr=0.001),
#     loss=keras.losses.SparseCategoricalCrossentropy(),
#     metrics=["accuracy"],
# )

# model.fit(x_train, y_train, batch_size=64, epochs=2, verbose =2)
# print(model.summary())
class ResBlock(layers.Layer):
    def __init__(self, channels):
        super(ResBlock, self).__init__()
        self.channels = channels
        self.cnn1 = CNNBlock(channels[0], 3)
        self.cnn2 = CNNBlock(channels[1], 3)
        self.cnn3 = CNNBlock(channels[2], 3)
        self.pooling = layers.MaxPooling2D()
        self.identity_mapping = layers.Conv2D(channels[1], 3, padding="same")

    def call(self, input_tensor, training=False):
        x = self.cnn1(input_tensor, training=training)
        x = self.cnn2(x, training=training)
        x = self.cnn3(x + self.identity_mapping(input_tensor), training=training,)
        x = self.pooling(x)
        return x
# keras.Model has additional functionalities than layers.Layer
class ResNet_Like(keras.Model):
    def __init__(self, num_classes=10):
        super(ResNet_Like, self).__init__()
        self.block1 = ResBlock([32, 32, 64])
        self.block2 = ResBlock([128, 128, 256])
        self.block3 = ResBlock([128, 256, 512])
        self.pool = layers.Flatten()
        self.classifier = layers.Dense(num_classes)

    def call(self, input_tensor, training=False):
        x = self.block1(input_tensor, training=training)
        x = self.block2(x, training=training)
        x = self.block3(x, training=training)
        x = self.pool(x, training=training)
        x = self.classifier(x)
        return x

    def model(self):
        x = keras.Input(shape=(28, 28, 1))
        return keras.Model(inputs=[x], outputs=self.call(x))

# we can use layers.Flatten() instead of layers.GlobalAveragePooling2D()
model = ResNet_Like().model()
base_input = model.layers[0].input
base_output = model.layers[2].output
output = layers.Dense(10)(layers.Flatten()(base_output))
model = keras.Model(base_input, output)

model.compile(
    optimizer=keras.optimizers.Adam(),
    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    metrics=["accuracy"],
)

model.fit(x_train, y_train, batch_size=64, epochs=20, verbose=2)
print(model.summary())
model.evaluate(x_test, y_test, batch_size=64, verbose=2)<jupyter_output>Epoch 1/20
938/938 - 17s - loss: 0.1054 - accuracy: 0.9672
Epoch 2/20
938/938 - 16s - loss: 0.0406 - accuracy: 0.9877
Epoch 3/20
938/938 - 16s - loss: 0.0286 - accuracy: 0.9911
Epoch 4/20
938/938 - 16s - loss: 0.0243 - accuracy: 0.9924
Epoch 5/20
938/938 - 16s - loss: 0.0196 - accuracy: 0.9939
Epoch 6/20
938/938 - 16s - loss: 0.0177 - accuracy: 0.9945
Epoch 7/20
938/938 - 16s - loss: 0.0159 - accuracy: 0.9947
Epoch 8/20
938/938 - 16s - loss: 0.0133 - accuracy: 0.9961
Epoch 9/20
938/938 - 16s - loss: 0.0122 - accuracy: 0.9964
Epoch 10/20
938/938 - 16s - loss: 0.0116 - accuracy: 0.9964
Epoch 11/20
938/938 - 16s - loss: 0.0074 - accuracy: 0.9978
Epoch 12/20
938/938 - 16s - loss: 0.0128 - accuracy: 0.9961
Epoch 13/20
938/938 - 16s - loss: 0.0074 - accuracy: 0.9978
Epoch 14/20
938/938 - 16s - loss: 0.0095 - accuracy: 0.9971
Epoch 15/20
938/938 - 16s - loss: 0.0120 - accuracy: 0.9966
Epoch 16/20
938/938 - 16s - loss: 0.0066 - accuracy: 0.9981
Epoch 17/20
938/938 - 16s - loss: 0.0092 - accura[...]<jupyter_text>CUSTOM LAYERS<jupyter_code># Here we are designing in built Dense or Relu layers in a custom way to gain additional flexibilty
import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.datasets import mnist

# To Avoid GPU errors
physical_devices = tf.config.list_physical_devices("GPU")
tf.config.experimental.set_memory_growth(physical_devices[0], True)

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(-1, 28 * 28).astype("float32") / 255.0
x_test = x_test.reshape(-1, 28 * 28).astype("float32") / 255.0


class Dense(layers.Layer):
    print("in dense1")
    def __init__(self, units, input_dim):
        super(Dense, self).__init__()
        self.w = self.add_weight(
            name="w",
            shape=(input_dim, units),
            initializer="random_normal",
            trainable=True,
        )

        self.b = self.add_weight(
            name="b", shape=(units,), initializer="zeros", trainable=True
        )

    def call(self, inputs):
        return tf.matmul(inputs, self.w) + self.b


class Dense(layers.Layer):
    print("in dense2")
    def __init__(self, units):
        super(Dense, self).__init__()
        self.units = units

    def build(self, input_shape):
        self.w = self.add_weight(
            name="w",
            shape=(input_shape[-1], self.units),
            initializer="random_normal",
            trainable=True,
        )
        self.b = self.add_weight(
            name="b", shape=(self.units,), initializer="random_normal", trainable=True,
        )

    def call(self, inputs):
        return tf.matmul(inputs, self.w) + self.b


class MyReLU(layers.Layer):
    def __init__(self):
        super(MyReLU, self).__init__()

    def call(self, x):
        return tf.math.maximum(x, 0)


class MyModel(keras.Model):  # model.fit, model.evalute, model.predict
    def __init__(self, num_classes=10):
        super(MyModel, self).__init__()
        self.dense1 = Dense(64)
        self.dense2 = Dense(num_classes)
        self.relu = MyReLU()

        # self.dense1 = layers.Dense(64)
        # self.dense3 = layers.Dense(num_classes)

    def call(self, x):
        x = self.relu(self.dense1(x))
        return self.dense2(x)


model = MyModel()
model.compile(
    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer=keras.optimizers.Adam(),
    metrics=["accuracy"],
)

model.fit(x_train, y_train, batch_size=32, epochs=2, verbose=2)
model.evaluate(x_test, y_test, batch_size=32, verbose=2)<jupyter_output>in dense1
in dense2
Epoch 1/2
1875/1875 - 4s - loss: 0.3468 - accuracy: 0.9036
Epoch 2/2
1875/1875 - 2s - loss: 0.1615 - accuracy: 0.9527
313/313 - 0s - loss: 0.1433 - accuracy: 0.9549<jupyter_text>SAVE & LOADING MODELS<jupyter_code># API(Sequential/Functional/subclassing) while saving and loading model should be same(as far as I know)


 import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.datasets import mnist

# To Avoid GPU errors
physical_devices = tf.config.list_physical_devices("GPU")
tf.config.experimental.set_memory_growth(physical_devices[0], True)

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(-1, 28 * 28).astype("float32") / 255.0
x_test = x_test.reshape(-1, 28 * 28).astype("float32") / 255.0


# 1. How to save and load model weights
# 2. Save and loading entire model (Serializing model)
#   - Saves weights
#   - Model architecture
#   - Training Configuration (model.compile())
#   - Optimizer and states


# method1: Sequential API
model1 = keras.Sequential([layers.Dense(64, activation="relu"), layers.Dense(10)])


# method1: Functional API
inputs = keras.Input(784)
x = layers.Dense(64, activation="relu")(inputs)
outputs = layers.Dense(10)(x)
model2 = keras.Model(inputs=inputs, outputs=outputs)


# Model subclassing
class MyModel(keras.Model):
    def __init__(self):
        super(MyModel, self).__init__()
        self.dense1 = layers.Dense(64, activation="relu")
        self.dense2 = layers.Dense(10)

    def call(self, input_tensor):
        x = tf.nn.relu(self.dense1(input_tensor))
        return self.dense2(x)


# SavedModel format or HDF5 format
model3 = MyModel()
# model = keras.models.load_model('saved_model/')
# model.load_weights('checkpoint_folder/')

model.compile(
    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer=keras.optimizers.Adam(),
    metrics=["accuracy"],
)

model.fit(x_train, y_train, batch_size=32, epochs=2, verbose=2)
model.evaluate(x_test, y_test, batch_size=32, verbose=2)
# model.save_weights('checkpoint_folder/')
model.save("saved_model/")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14762793079483522, 'fraction_numerical': 0.05739054098540546, 'mean_word_length': 4.235541937954807, 'pattern_counts': {'":': 2, '<': 49, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '138261', 'license': 'MIT', 'n_tokens_mistral': 10550, 'n_tokens_neox': 9489, 'n_words': 1982, 'path': 'Tensorflow_Basics_part1.ipynb', 'repo_name': 'chinya07/Tensorflow_Basics', 'size': '27339'}