<jupyter_start><jupyter_text>Model 1: Conv Layers, (sigmoid)<jupyter_code>x1 = np.random.beta(0.5, 1, (1000,100))
y1 = np.zeros((1000,))
print(x1.shape)
print(y1.shape)

x2 = np.random.beta(0.7, 1.25, (1000,100))
y2 = np.ones((1000,))
print(x2.shape)
print(y2.shape)

xz = np.concatenate((x1, x2), axis=0)
yz = np.concatenate((y1, y2), axis=0)
print(xz.shape)
print(yz.shape)

z = list(zip(xz, yz))
np.random.shuffle(z)
xz = np.array([a[0] for a in z])
yz = np.array([a[1] for a in z])

xz = np.reshape(xz, (2000, 100, 1))
yz = np.reshape(yz, (2000,))

print(xz.shape)
print(yz.shape)
import matplotlib.pyplot as plt
plt.plot(x1[0])
plt.show()
plt.plot(x2[0])
plt.show()
from aug_attn import *
from tensorflow.keras.layers import Input, Dense, Conv1D, Flatten, Activation, Embedding
from tensorflow.keras.models import Model




ip = Input(shape=(100,1))
cnn1 = Conv1D(filters = 8, kernel_size=3, strides=1,padding='same', activation = 'relu')(ip)

fl = Flatten()(cnn1)
fc = Dense(1, activation='sigmoid')(fl)

basic_cnn = Model(ip, fc)

basic_cnn.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])

basic_cnn.summary()
cnn_hist = basic_cnn.fit(xz, yz, epochs = 10, batch_size = 16, validation_split=0.5, verbose = 1)
import matplotlib.pyplot as plt
plt.plot(cnn_hist.history['acc'])
plt.plot(cnn_hist.history['val_acc'])
plt.plot(cnn_hist.history['loss'])
plt.plot(cnn_hist.history['val_loss'])
plt.legend(['acc', 'val_acc', 'loss', 'val_loss'])
plt.title('basic conv (833 params)')
plt.show()
basic_cnn.evaluate(xz, yz)
ip = Input(shape=(100,1))

x = augmented_conv1d(ip, shape = (100, 1), filters=6, kernel_size=3,
                     strides = 1,
                     padding = 'same', # if causal convolution is needed
                     depth_k=2, depth_v=2,  
                     num_heads=2, relative_encodings=True)
x = Activation('relu')(x)
fl = Flatten()(x)
fc = Dense(1, activation='sigmoid')(fl)

aac_cnn = Model(ip, fc)

aac_cnn.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])

aac_cnn.summary()
aac_hist = aac_cnn.fit(xz, yz, epochs = 10, batch_size = 16, validation_split = 0.5, verbose = 0)
import matplotlib.pyplot as plt
plt.plot(aac_hist.history['acc'])
plt.plot(aac_hist.history['val_acc'])
plt.plot(aac_hist.history['loss'])
plt.plot(aac_hist.history['val_loss'])
plt.title('attention augmented conv (835 params)')
plt.legend(['acc', 'val_acc', 'loss', 'val_loss'])
plt.show()
aac_cnn.evaluate(xz, yz)
from aug_attn import *
from keras.layers import Input
from keras.models import Model

ip = Input(shape=(32, 32, 10))
cnn1 = Conv2D(filters = 10, kernel_size=3, strides=1,padding='same')(ip)
x = augmented_conv2d(cnn1, filters=20, kernel_size=5, # shape parameter is not needed
                     strides = 1,
                     depth_k=4, depth_v=4,  # padding is by default, same
                     num_heads=4, relative_encodings=True)

# depth_k | filters, depth_v | filters,  Nh | depth_k, Nh | filters-depth_v

model = Model(ip, x)
model.summary()

x = tf.ones((1, 32, 32, 10))
print(x.shape)
y = model(x)
print(y.shape)
# extra, delete later
from modules import *
from tensorflow.keras.layers import Activation, Flatten, Dense
ip = Input(shape=(100,1))

x = augmented_conv1d(ip, shape = (100, 1), filters=6, kernel_size=3,
                     strides = 1,
                     padding = 'same', # if causal convolution is needed
                     depth_k=2, depth_v=2,  
                     num_heads=2, relative_encodings=True)

x = dha(x, training = 1)
x = Activation('relu')(x)
fl = Flatten()(x)
fc = Dense(1, activation='sigmoid')(fl)

aac_cnn = Model(ip, fc)

aac_cnn.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])

aac_cnn.summary()
aac_hist = aac_cnn.fit(xz, yz, epochs = 50, batch_size = 16, validation_split = 0.5, verbose = 0)
import matplotlib.pyplot as plt
plt.plot(aac_hist.history['acc'])
plt.plot(aac_hist.history['val_acc'])
plt.plot(aac_hist.history['loss'])
plt.plot(aac_hist.history['val_loss'])
plt.title('attention augmented conv (835 params)')
plt.legend(['acc', 'val_acc', 'loss', 'val_loss'])
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1677063339731286, 'fraction_numerical': 0.03982725527831094, 'mean_word_length': 4.428385416666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '535820', 'license': 'MIT', 'n_tokens_mistral': 1791, 'n_tokens_neox': 1619, 'n_words': 339, 'path': 'aug_attn.ipynb', 'repo_name': 'zabir-nabil/keras-attn_aug_cnn', 'size': '4168'}