<jupyter_start><jupyter_text>Normal model<jupyter_code>print(x_train.shape)
print(x_test.shape)
print(y_train[0:3])
from sklearn.model_selection import train_test_split
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=99)
print(x_train.shape)
print(x_valid.shape)
print(y_train[0:3])
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(512, activation=tf.nn.relu),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
%%time 
model.compile(optimizer=tf.train.AdamOptimizer(0.001),
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'])
fit = model.fit(x_train, y_train, epochs=5, validation_data=(x_valid, y_valid))
%%time 
y_train_one_hot = tf.keras.utils.to_categorical(y_train, 10)
model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
fit = model.fit(x_train, y_train_one_hot, epochs=5)
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
print(x_train.shape)
print(x_test.shape)
model = tf.keras.models.Sequential()
model.add(layers.Conv2D(32, kernel_size=(3,3), activation='relu', input_shape=(28, 28,1)))
model.add(layers.Conv2D(64, (3,3), activation='relu'))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))

%%time 
y_train_one_hot = tf.keras.utils.to_categorical(y_train, 10)
model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
fit = model.fit(x_train, y_train_one_hot, epochs=5)<jupyter_output>Epoch 1/5
60000/60000 [==============================] - 194s 3ms/step - loss: 0.1162 - acc: 0.9637
Epoch 2/5
60000/60000 [==============================] - 186s 3ms/step - loss: 0.0388 - acc: 0.9881
Epoch 3/5
60000/60000 [==============================] - 187s 3ms/step - loss: 0.0239 - acc: 0.9925
Epoch 4/5
60000/60000 [==============================] - 180s 3ms/step - loss: 0.0149 - acc: 0.9948
Epoch 5/5
60000/60000 [==============================] - 184s 3ms/step - loss: 0.0104 - acc: 0.9967<jupyter_text>Define modelAll layers must be defined in __init__() and run order in call()TODO: what is compute_output_shape() -> change output size if nescessary. See **Define layer > Flatten layer** for more information"[In case your layer modifies the shape of its input, you should specify here the shape transformation logic](https://keras.io/layers/writing-your-own-keras-layers/). This allows Keras to do automatic shape inference." (from keras page) - **mean, not necessary ?**<jupyter_code>class MyModel(tf.keras.Model):
    def __init__(self, num_classes=10):
        super(MyModel, self).__init__(name='my_model')
        self.num_classes = num_classes
        self.flatten_1= layers.Flatten(input_shape=(28, 28))
        self.dense_1 = layers.Dense(512, activation='relu')
        self.dropout_1 = layers.Dropout(0.2)
        self.dense_2 = layers.Dense(num_classes, activation='softmax')
        
    def call(self, inputs):
        x = self.flatten_1(inputs)
        x = self.dense_1(x)
        x = self.dropout_1(x)
        return self.dense_2(x)
    
#     def compute_output_shape(self, input_shape):
#         shape = tf.TensorShape(input_shape).as_list()
#         shape[-1] = self.num_classes
#         return tf.TensorShape(shape)
model = MyModel(num_classes=10)
model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
fit = model.fit(x_train, y_train, batch_size=32, epochs=5)<jupyter_output>Epoch 1/5
48000/48000 [==============================] - 9s 180us/step - loss: 0.3486 - acc: 0.8940
Epoch 2/5
48000/48000 [==============================] - 6s 123us/step - loss: 0.1184 - acc: 0.9653
Epoch 3/5
48000/48000 [==============================] - 6s 123us/step - loss: 0.0896 - acc: 0.9750
Epoch 4/5
48000/48000 [==============================] - 6s 129us/step - loss: 0.0727 - acc: 0.9795
Epoch 5/5
48000/48000 [==============================] - 6s 128us/step - loss: 0.0623 - acc: 0.9825<jupyter_text>Define layerDefine a layer is more more more difficult than ModelSome notes on this:* Some tensorflow function takes a tensor of int (double, float ..) as argument. This means we cannot pass None ( number of sample ) to it. (See MyFlattenLayer)* Always define compute_output_shape<jupyter_code>import numpy as np
class MyFlattenLayer(layers.Layer):
    def __init__(self, data_format=None, **kwargs):
        self.data_format = data_format
        super(MyFlattenLayer, self).__init__(**kwargs)
        
        
    def call(self, inputs):
        if self.data_format=="channels_first":
            transpose_order = [0]
            transpose_order.extend([i for i in range(2, len(inputs.get_shape()))])
            transpose_order.append(1)
            inputs = tf.transpose(inputs, perm=transpose_order)
        output_shape = self.compute_output_shape(inputs.get_shape())
#         NOTE : Error occurred with following code, because input size is None, not a tensor of int32
#         outputs = tf.reshape(inputs, output_shape)
        outputs = tf.reshape(inputs, (tf.shape(inputs)[0], output_shape[1]))
        return outputs
    
    def compute_output_shape(self, input_shape):
        input_shape = tf.TensorShape(input_shape).as_list()
        output_shape = [input_shape[0]]
        if all(input_shape[1:]):
            output_shape += [np.prod(input_shape[1:])]
        else :
            output_shape += [None]
        return tf.TensorShape(output_shape)
    
    def get_config(self):
        base_config = super(MyFlattenLayer, self).get_config()
        base_config['data_format'] = self.data_format
        return base_config
   
model = tf.keras.Sequential([
    MyFlattenLayer(),
    tf.keras.layers.Dense(128, activation=tf.nn.relu),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])

model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
fit = model.fit(x_train, y_train, batch_size=32, epochs=5, validation_data=(x_valid, y_valid))
class MyDenseLayer(layers.Layer):
    def __init__(self, output_dim, activation, **kwargs):
        self.output_dim = output_dim
        self.activation = activation
        super(MyDenseLayer, self).__init__(**kwargs)
        
    def build(self, input_shape):
        shape = tf.TensorShape((input_shape[1], self.output_dim)).as_list()
        self.kernel = self.add_weight(name='kernel',
                                  shape=shape,
                                  initializer='uniform', 
                                      dtype=tf.float32,
                                  trainable=True)
        super(MyDenseLayer, self).build(input_shape)
        
    def call(self, inputs):
        y = layers.Activation(self.activation)(tf.matmul(inputs, self.kernel))
        return y

    def compute_output_shape(self, input_shape):
        return (input_shape[0], self.output_dim)
    
    def get_config(self):
        base_config = super(MyDenseLayer, self).get_config()
        base_config['output_dim'] = self.output_dim
        base_config['activation'] = self.activation
        return base_config
model = tf.keras.Sequential([
    MyFlattenLayer(),
    MyDenseLayer(128, 'relu'),
    MyDenseLayer(10, 'softmax')
])

model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
fit = model.fit(x_train, y_train, batch_size=32, epochs=5, validation_data=(x_valid, y_valid))
model = tf.keras.Sequential([
    MyFlattenLayer(),
    MyDenseLayer(128, 'relu'),
    MyDenseLayer(10, 'softmax')
])
model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

fit = model.fit(x_train, y_train, batch_size=32, epochs=5, validation_data=(x_valid, y_valid))
model.summary()
estimator = tf.keras.estimator.model_to_estimator(model)
fit = model.fit(x_train, y_train, batch_size=32, epochs=5, validation_data=(x_valid, y_valid))
callbacks = [
  # Interrupt training if `val_loss` stops improving for over 2 epochs
  tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),
  # Write TensorBoard logs to `./logs` directory
  tf.keras.callbacks.TensorBoard(log_dir='./logs')
]
fit = model.fit(x_train, y_train, batch_size=32, epochs=5, callbacks=callbacks, validation_data=(x_valid, y_valid))<jupyter_output>Train on 48000 samples, validate on 12000 samples
Epoch 1/5
48000/48000 [==============================] - 5s 104us/step - loss: 0.0343 - acc: 0.9913 - val_loss: 0.1090 - val_acc: 0.9734
Epoch 2/5
48000/48000 [==============================] - 6s 116us/step - loss: 0.0320 - acc: 0.9916 - val_loss: 0.1046 - val_acc: 0.9771
Epoch 3/5
48000/48000 [==============================] - 6s 122us/step - loss: 0.0281 - acc: 0.9927 - val_loss: 0.1155 - val_acc: 0.9760
Epoch 4/5
48000/48000 [==============================] - 5s 94us/step - loss: 0.0253 - acc: 0.9932 - val_loss: 0.1163 - val_acc: 0.9759
{'fraction_non_alphanumeric': 0.16994232001709036, 'fraction_numerical': 0.059923093356120484, 'mean_word_length': 3.73836032388664, 'pattern_counts': {'":': 1, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '655979', 'license': 'MIT', 'n_tokens_mistral': 3556, 'n_tokens_neox': 3157, 'n_words': 616, 'path': 'mnist.ipynb', 'repo_name': 'kuThang/study-tensor-keras', 'size': '9362'}