<jupyter_start><jupyter_text>one experiment...early_stopping=EarlyStopping(monitor='val_loss', patience=15)reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,                              patience=5, min_lr=0.00001,                             verbose=True)checkpoint = ModelCheckpoint('weights.lstm.best.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')model = Sequential()model.add(Embedding(max_features, 32,                     embeddings_regularizer=regularizers.l2(0.001)))model.add(LSTM(32,                dropout=0.2, recurrent_dropout=0.2))model.add(Dense(1, activation='sigmoid', kernel_regularizer=regularizers.l2(0.001), bias_regularizer=regularizers.l2(0.001)))model.compile(optimizer=Adam(lr=0.002),            loss='binary_crossentropy',            metrics=['acc'])history = model.fit(input_train, y_train,            epochs=100,            batch_size=128,            validation_split=0.2,            callbacks=[early_stopping, reduce_lr, checkpoint])<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.show()
from keras.layers import LSTM, Bidirectional
early_stopping=EarlyStopping(monitor='val_loss', patience=15)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
                              patience=5, min_lr=0.00001)
checkpoint = ModelCheckpoint('data/weights.lstm.best.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')


model = Sequential()
model.add(Embedding(max_features, 32, 
                    embeddings_regularizer=regularizers.l2(0.001)))
model.add(LSTM(32, 
               dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid', kernel_regularizer=regularizers.l2(0.001), bias_regularizer=regularizers.l2(0.001)))

model.compile(optimizer=Adam(lr=0.002),
            #loss='binary_crossentropy',
            loss='mean_squared_error',
            metrics=['acc'])
history = model.fit(input_train, y_train,
            epochs=100,
            batch_size=128,
            validation_split=0.2,
            callbacks=[early_stopping, reduce_lr, checkpoint])
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>BiLSTM<jupyter_code>early_stopping=EarlyStopping(monitor='val_loss', patience=15)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
                              patience=5, min_lr=0.00001,
                             verbose=True)
checkpoint = ModelCheckpoint('data/weights.bilstm.best.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')


model = Sequential()
model.add(Embedding(max_features, 32, 
                    embeddings_regularizer=regularizers.l2(0.001)))
model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, dropout=0.2)))
model.add(Dense(1, activation='sigmoid', 
                kernel_regularizer=regularizers.l2(0.001), bias_regularizer=regularizers.l2(0.001)))

model.compile(optimizer=Adam(lr=0.001),
            #loss='binary_crossentropy',
            loss='mean_squared_error',
            metrics=['acc'])
history = model.fit(input_train, y_train,
            epochs=100,
            batch_size=128,
            validation_split=0.2,
            callbacks=[early_stopping, reduce_lr, checkpoint])
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>CNN<jupyter_code>from keras.models import Sequential
from keras import layers
from keras.optimizers import RMSprop, Adam
early_stopping=EarlyStopping(monitor='val_loss', patience=15)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
                              patience=5, min_lr=0.00001,
                             verbose=True)
checkpoint = ModelCheckpoint('data/weights.cnn.best.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')

l2 = 0.001
dout = 0.4

model = Sequential()
model.add(layers.Embedding(max_features, 64, input_length=maxlen, embeddings_regularizer=regularizers.l2(l2)))
model.add(layers.Dropout(dout))
model.add(layers.Conv1D(64, 7, activation='relu', 
                        kernel_regularizer=regularizers.l2(l2)))
model.add(layers.MaxPooling1D(5))
model.add(layers.Dropout(dout))
model.add(layers.Conv1D(32, 7, activation='relu', kernel_regularizer=regularizers.l2(l2)))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dropout(dout))
model.add(layers.Dense(1, kernel_regularizer=regularizers.l2(l2)))
model.summary()
model.compile(optimizer=Adam(lr=1e-4),
            #loss='binary_crossentropy',
            loss='mean_squared_error',
            metrics=['acc'])
history = model.fit(input_train, y_train,
            epochs=100,
            batch_size=16,
            validation_split=0.2,
            callbacks=[early_stopping, reduce_lr, checkpoint])
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14812620352540365, 'fraction_numerical': 0.03051399792623315, 'mean_word_length': 4.507340946166395, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '88722', 'license': 'MIT', 'n_tokens_mistral': 2432, 'n_tokens_neox': 2253, 'n_words': 358, 'path': 'Practice-2019-05-04/RNN.ipynb', 'repo_name': 'serge-sotnyk/nlp-practice', 'size': '6751'}