<jupyter_start><jupyter_text>Test output and input shape of a layer **Custom Layer for understanding keras input-/output shapes**<jupyter_code>class DummyLayer(LSTM):
    def __init__(self, units, **kwargs):
        super().__init__(units, **kwargs)
    
    def build(self, input_shape):
        print(f'build:   input_shape={input_shape}')
        return super().build(input_shape)
    
    def call(self, x, **kwargs):
        y = super().call(x, **kwargs)
        print(f'call:    input_shape={x.shape}, output_shape={y.shape}')
        sleep(1)
        return y
        
    
    def compute_output_shape(self, input_shape):
        output_shape = super().compute_output_shape(input_shape)
        print(f'compute: input_shape={input_shape}, output_shape={output_shape}')
        return output_shape
def get_model():
    model = SequentialModel()
    model.add(Embedding(input_dim=15000, output_dim=200))
    model.add(LSTM(units=128, dropout=0.2, recurrent_dropout=0.15, return_sequences=True))
    model.add(DummyLayer(units=200))
    model.add(Dense(units=100))
    model.add(Dense(3, activation='softmax'))

    model.compile(optimizer='Adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])
    return model
model = get_model()
model.fit(x_train, 
          y_train,
          epochs=4,
          batch_size=32,
          validation_split=0.3)<jupyter_output>C:\Users\Sgt-Peppers\Anaconda3\envs\keras\lib\site-packages\tensorflow_core\python\framework\indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.
  "Converting sparse IndexedSlices to a dense Tensor of unknown shape. "<jupyter_text>Expected Input/Output: ``` LSTM-output as inputbuild:   input_shape=(None, None, 128)call:    input_shape=(None, None, 128), output_shape=(None, 200)compute: input_shape=(None, None, 128), output_shape=(None, 200)```  Custom Layers included in model:  Layer Normalization<jupyter_code>def get_model():
    model = SequentialModel()
    model.add(Embedding(input_dim=15000, output_dim=200))
    model.add(LSTM(units=128, dropout=0.2, recurrent_dropout=0.15, return_sequences=False))
    model.add(Dense(units=100))
    model.add(LayerNormalization(units=100))
    # model.add(Dense(units=100))
    model.add(Dense(3, activation='softmax'))

    model.compile(optimizer='Adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])
    return model
model = get_model()
model.fit(x_train, 
          y_train,
          epochs=8,
          batch_size=32,
          validation_split=0.3)
model.summary()<jupyter_output>Model: "sequential_23"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_22 (Embedding)     (None, None, 200)         3000000   
_________________________________________________________________
lstm_27 (LSTM)               (None, 128)               168448    
_________________________________________________________________
dense_31 (Dense)             (None, 100)               12900     
_________________________________________________________________
layer_normalization_5 (Layer (None, 100)               0         
_________________________________________________________________
dense_32 (Dense)             (None, 3)                 303       
=================================================================
Total params: 3,181,651
Trainable params: 3,181,651
Non-trainable params: 0
___________________________________________[...]<jupyter_text>Scaled Dot Product Attention<jupyter_code>def get_model():
    # x = Input()
    model = SequentialModel()
    model.add(Embedding(input_dim=15000, output_dim=128))
    # model.add(LSTM(units=128, dropout=0.2, recurrent_dropout=0.15, return_sequences=True))
    model.add(ScaledDotProductAttention(d_model=128, d_k=16, d_v=128))
    model.add(Flatten())
    model.add(Dense(units=100, name='hidden_0'))
    model.add(LayerNormalization(units=100))
    model.add(Dense(3, activation='softmax', name='output_layer'))

    model.compile(optimizer='Adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])
    return model


model = get_model()
model.summary()
model.fit(x_train, 
          y_train,
          epochs=8,
          batch_size=32,
          validation_split=0.3)<jupyter_output>C:\Users\Sgt-Peppers\Anaconda3\envs\keras\lib\site-packages\tensorflow_core\python\framework\indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.
  "Converting sparse IndexedSlices to a dense Tensor of unknown shape. "<jupyter_text>Multi-Head Attention<jupyter_code>def get_model(number_of_heads=2):
    input_layer = Input(shape=(128,))

    _0_x = Embedding(input_dim=15000, output_dim=128)(input_layer)

    _1_s = [ScaledDotProductAttention(d_model=128, d_k=16, d_v=128)(_0_x) for _ in range(number_of_heads)]
    _1_m = MultiHeadAttention(d_heads=2, d_model=128, d_k=16, d_v=128)(_1_s)

    _2_f = Flatten()(_1_m)
    _2_h_0 = Dense(units=10, name='hidden_0')(_2_f)
    _2_hL_0 = LayerNormalization(units=10)(_2_h_0)

    output_layer = Dense(3, activation='softmax', name='output_layer')(_2_hL_0)

    model = Model(inputs=[input_layer],
                  outputs=[output_layer])
    model.compile(optimizer='Adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    return model

model = get_model()
model.summary()
model.fit(x_train, 
          y_train,
          epochs=8,
          batch_size=32,
          validation_split=0.3)<jupyter_output>C:\Users\Sgt-Peppers\Anaconda3\envs\keras\lib\site-packages\tensorflow_core\python\framework\indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.
  "Converting sparse IndexedSlices to a dense Tensor of unknown shape. "<jupyter_text>Subclassing Model<jupyter_code>class MHAModel(Model):
    def __init__(self, *args, **kwargs):
        input_layer = Input(shape=(128,))

        _0_x = Embedding(input_dim=15000, output_dim=128)(input_layer)

        _1_s = [ScaledDotProductAttention(d_model=128, d_k=16, d_v=128)(_0_x) for _ in range(2)]
        _1_m = MultiHeadAttention(d_heads=2, d_model=128, d_k=16, d_v=128)(_1_s)

        _2_f = Flatten()(_1_m)
        _2_h_0 = Dense(units=10, name='hidden_0')(_2_f)
        _2_hL_0 = LayerNormalization(units=10)(_2_h_0)
        
        output_layer = Dense(3, activation='softmax', name='output_layer')(_2_hL_0)
    
        super().__init__(inputs=[input_layer], 
                         outputs=[output_layer], 
                         *args, 
                         **kwargs)

def get_model():
    model = MHAModel(name='Easy keras, EASY.')
    return model


model = get_model()
model.compile(optimizer='Adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
model.summary()<jupyter_output>Model: "Easy keras, EASY."
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            (None, 128)          0                                            
__________________________________________________________________________________________________
embedding_1 (Embedding)         (None, 128, 128)     1920000     input_1[0][0]                    
__________________________________________________________________________________________________
scaled_dot_product_attention_1  (None, 128, 128)     20480       embedding_1[0][0]                
__________________________________________________________________________________________________
scaled_dot_product_attention_2  (None, 128, 128)     20480       embedding_1[0][0][...]<jupyter_text>Model within a Model<jupyter_code>class ModelCeption(Model):
    def __init__(self, *args, **kwargs):
        
        # Model
        input_layer = Input(shape=(128,))

        _0_x = Embedding(input_dim=15000, output_dim=128)

        _1_s = [ScaledDotProductAttention(d_model=128, d_k=16, d_v=128) for _ in range(2)]
        _1_m = MultiHeadAttention(d_heads=2, d_model=128, d_k=16, d_v=128)

        _2_f = Flatten()
        _2_h_0 = Dense(units=10, name='hidden_0')
        _2_hL_0 = LayerNormalization(units=10)
        
        output_layer = Dense(3, activation='softmax', name='output_layer')
    
        
        _0_x_tensor = _0_x(input_layer)
        _1_s_tensor = [head(_0_x_tensor) for head in _1_s]
        _1_m_tensor = _1_m(_1_s_tensor)
        _2_f_tensor = _2_f(_1_m_tensor)
        _2_h_0_tensor = _2_h_0(_2_f_tensor)
        _2_hL_0_tensor = _2_hL_0(_2_h_0_tensor)
        output_tensor = output_layer(_2_hL_0_tensor)
    
        super().__init__(inputs=input_layer,
                         outputs=output_tensor,
                         *args, 
                         **kwargs)
        self.inner_model = {'inner_model': InnerModel(),
                            'inner_model_input': _2_f}
        self.input_layer = input_layer
        self._0_x = _0_x
        self._1_s = _1_s
        self._1_m = _1_m
        self._2_f = _2_f
        self._2_h_0 = _2_h_0
        self._2_hL_0 = _2_hL_0
        self.output_layer = output_layer
        
    def __call__(self, *args, **kwargs):
        print('CALLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL')
        super().__call__(*args, **kwargs)
        
#     def fit():
#         pass

#     def train_on_batch:
#         pass
    
    def call(self, inputs, *args, **kwargs):
        print()
        print('INPUTS')
        print(inputs)
        # sleep(2)
        exit(-1)
        super().call(inputs, *args, **kwargs)
    
    def compile(self, *args, **kwargs):
        super().compile(*args, **kwargs)
        self.inner_model['inner_model'].compile(optimizer='Adam',
                                                loss='categorical_crossentropy',
                                                metrics=['accuracy'])


class InnerModel(Model):
    def __init__(self, *args, **kwargs):
        _inputs = Input((16384,))
        # z = Lambda(lambda x: K.stop_gradient(x))(_inputs)
        z = Dense(units=10, name='inner_0')(_inputs)
        z = LayerNormalization(units=10)(z)
        
        output_layer = Dense(3, activation='softmax', name='inner_output')(z)
        
        super().__init__(*args,
                         inputs=_inputs,
                         outputs=output_layer,
                         **kwargs)

class TrainInnerModelCallback(Callback):
    def __init__(self, inner_model, pseudo_input,*args, **kwags):
        self.inner_model = inner_model
        self.pseudo_input = pseudo_input
        super().__init__()

    def on_batch_end(self, batch, logs=None):
        print()
        print(self.pseudo_input.output)
        sleep(2)
        
        
    def on_epoch_end(self, epoch, logs=None):
        pass
    
    
        
def get_model():
    model = ModelCeption(name='ModelCeption')
    return model


model = get_model()
model.compile(optimizer='Adam',
              loss=['categorical_crossentropy'],
              metrics=['accuracy']
             )
model.summary()
callback = TrainInnerModelCallback(inner_model=model.inner_model['inner_model'], 
                                   pseudo_input=model._2_f)

model.fit(x_train, 
          y_train,
          epochs=8,
          batch_size=32,
          validation_split=0.3,
          callbacks=[callback])
model.train_function
x_t = x_train[:int(0.7*len(x_train))]
y_t = y_train[:int(0.7*len(x_train))]

x_v = x_train[int(0.7*len(x_train)):]
y_v = y_train[int(0.7*len(x_train)):]

output = []
for e in range(5):
    print(f'epoch: {e}')
    epoch_output = []
    for i in range(0, len(x_t), 16):
        batch_output = model.train_on_batch(x_t[i: i+16], y_t[i: i+16])
        epoch_output.append(batch_output)
    output.append(epoch_output)
model.evaluate(x_v, y_v, batch_size=16)
input_layer = Input(shape=(128,))
_0_x = Embedding(input_dim=15000, output_dim=128)(input_layer)
_1_s = [ScaledDotProductAttention(d_model=128, d_k=16, d_v=128)(_0_x) for _ in range(2)]
_1_m = MultiHeadAttention(d_heads=2, d_model=128, d_k=16, d_v=128)(_1_s)
_2_f = Flatten(name='flattened_intermediate_layer')(_1_m)
_2_h_0 = Dense(units=10, name='hidden_0')(_2_f)
_2_hL_0 = LayerNormalization(units=10)(_2_h_0)
output_layer = Dense(3, activation='softmax', name='output_layer')(_2_hL_0)

model = Model(inputs=[input_layer], outputs=[output_layer])

model.compile(optimizer='Adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])

batch = x_train[:16]
batch_labels = y_train[:16]

flatten_output_function = K.function([model.input], _2_f)
flatten_output = flatten_output_function(batch)
flatten_output
flatten_output.shape
inner = InnerModel()
inner.compile(optimizer='Adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
inner.train_on_batch(x=flatten_output,
                     y=batch_labels)
# Model
input_layer = Input(shape=(128,))
_0_x = Embedding(input_dim=15000, output_dim=128)
_1_s = [ScaledDotProductAttention(d_model=128, d_k=16, d_v=128) for _ in range(2)]
_1_m = MultiHeadAttention(d_heads=2, d_model=128, d_k=16, d_v=128)
_2_f = Flatten()
_2_h_0 = Dense(units=10, name='hidden_0')
_2_hL_0 = LayerNormalization(units=10)
output_layer = Dense(3, activation='softmax', name='output_layer')

_0_x_tensor = _0_x(input_layer)
_1_s_tensor = [head(_0_x_tensor) for head in _1_s]
_1_m_tensor = _1_m(_1_s_tensor)
_2_f_tensor = _2_f(_1_m_tensor)
_2_h_0_tensor = _2_h_0(_2_f_tensor)
_2_hL_0_tensor = _2_hL_0(_2_h_0_tensor)
output_tensor = output_layer(_2_hL_0_tensor)

model = Model(inputs=[input_layer], outputs=[output_tensor])

model.compile(optimizer='Adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])

_2_f.output
_2_f_tensor<jupyter_output><empty_output><jupyter_text>Custom Loss function<jupyter_code>class test_model(Model):
    def __init__(self, *args, **kwargs):
        
        inp1 = Input(shape=(10,))
        inp2 = Input(shape=(10,))
        
        x = Concatenate(inputs=[inp1, inp2], axis=1)
        z = Dense(10)(x)
        
        y = Dense(3, activation='softmax')(z)
        
        super().__init__(inputs=[inp1, inp2], outputs=[x], *args, **kwargs)
        
    def train_on_batch(self, x, y, *args,**kwargs):
        print(x)
        inp1, inp2 = x
        
        print()
        
        super().train_on_batch(x, y, *args, **kwargs)
m = test_model()
m.summary()
m.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy'])

m.train_on_batch(x=[K.zeros((16,10)), K.zeros((16, 10))],
                 y=K.zeros((16, 3)))
K.eval(m([K.zeros((16,10)), K.zeros((16, 10))]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1328956007879186, 'fraction_numerical': 0.04110308601444517, 'mean_word_length': 3.201655172413793, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '264545', 'license': 'MIT', 'n_tokens_mistral': 5374, 'n_tokens_neox': 4995, 'n_words': 808, 'path': 'notebooks/Test Custom Layers.ipynb', 'repo_name': 'ViktorStagge/CompressiveTransformer', 'size': '15230'}