<jupyter_start><jupyter_text>Embed codons, learn w/ single Long Short Term Memory Layer  4 dimensional embedding<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers.embeddings import Embedding

# create the model
embedding_vecor_length = 4
model = Sequential()
model.add(Embedding(top_words, embedding_vecor_length, input_length=max_seq_length))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
model.fit(X_train, y_train, epochs=3, batch_size=100)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_1 (Embedding)      (None, 149, 4)            84        
_________________________________________________________________
lstm_1 (LSTM)                (None, 100)               42000     
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 101       
=================================================================
Total params: 42,185
Trainable params: 42,185
Non-trainable params: 0
_________________________________________________________________
None
Epoch 1/3
15654/15654 [==============================] - 107s 7ms/step - loss: 0.6933 - acc: 0.5211
Epoch 2/3
15654/15654 [==============================] - 106s 7ms/step - loss: 0.6703 - acc: 0.5973
Epoch 3/3
15654/15654 [==============================] - 107s 7ms[...]<jupyter_text>8 dimensional embedding<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers.embeddings import Embedding

# create the model
embedding_vecor_length = 8
model = Sequential()
model.add(Embedding(top_words, embedding_vecor_length, input_length=max_seq_length))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
model.fit(X_train, y_train, epochs=3, batch_size=100)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_2 (Embedding)      (None, 149, 8)            168       
_________________________________________________________________
lstm_2 (LSTM)                (None, 100)               43600     
_________________________________________________________________
dense_2 (Dense)              (None, 1)                 101       
=================================================================
Total params: 43,869
Trainable params: 43,869
Non-trainable params: 0
_________________________________________________________________
None
Epoch 1/3
15654/15654 [==============================] - 107s 7ms/step - loss: 0.6864 - acc: 0.5418
Epoch 2/3
15654/15654 [==============================] - 106s 7ms/step - loss: 0.6619 - acc: 0.6056
Epoch 3/3
15654/15654 [==============================] - 106s 7ms[...]<jupyter_text>16 dimensional embedding<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers.embeddings import Embedding

# create the model
embedding_vecor_length = 16
model = Sequential()
model.add(Embedding(top_words, embedding_vecor_length, input_length=max_seq_length))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
model.fit(X_train, y_train, epochs=3, batch_size=100)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_3 (Embedding)      (None, 149, 16)           336       
_________________________________________________________________
lstm_3 (LSTM)                (None, 100)               46800     
_________________________________________________________________
dense_3 (Dense)              (None, 1)                 101       
=================================================================
Total params: 47,237
Trainable params: 47,237
Non-trainable params: 0
_________________________________________________________________
None
Epoch 1/3
15654/15654 [==============================] - 111s 7ms/step - loss: 0.6812 - acc: 0.5611
Epoch 2/3
15654/15654 [==============================] - 110s 7ms/step - loss: 0.6439 - acc: 0.6286
Epoch 3/3
15654/15654 [==============================] - 111s 7ms[...]<jupyter_text>32 dimensional embedding<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers.embeddings import Embedding

# create the model
embedding_vecor_length = 32
model = Sequential()
model.add(Embedding(top_words, embedding_vecor_length, input_length=max_seq_length))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
model.fit(X_train, y_train, epochs=3, batch_size=100)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_4 (Embedding)      (None, 149, 32)           672       
_________________________________________________________________
lstm_4 (LSTM)                (None, 100)               53200     
_________________________________________________________________
dense_4 (Dense)              (None, 1)                 101       
=================================================================
Total params: 53,973
Trainable params: 53,973
Non-trainable params: 0
_________________________________________________________________
None
Epoch 1/3
15654/15654 [==============================] - 119s 8ms/step - loss: 0.6807 - acc: 0.5616
Epoch 2/3
15654/15654 [==============================] - 119s 8ms/step - loss: 0.6464 - acc: 0.6315
Epoch 3/3
15654/15654 [==============================] - 118s 8ms[...]<jupyter_text>Hook LSTM to basic CNN<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D
from keras.layers import Dropout
from keras.layers.embeddings import Embedding

# create the model
embedding_vecor_length = 16
model = Sequential()
model.add(Embedding(top_words, embedding_vecor_length, input_length=max_seq_length))
model.add(Conv1D(filters=128, kernel_size=3, padding='same', activation='selu'))
model.add(MaxPooling1D(pool_size=2))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())


# record training progress
history = model.fit(X_train, y_train, epochs=15, batch_size=64,
                    validation_data=(X_test, y_test))

# plot loss vs. epoch
# https://machinelearningmastery.com/diagnose-overfitting-underfitting-lstm-models/
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model train vs validation loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper right')
plt.show()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_5 (Embedding)      (None, 149, 16)           336       
_________________________________________________________________
conv1d_1 (Conv1D)            (None, 149, 128)          6272      
_________________________________________________________________
max_pooling1d_1 (MaxPooling1 (None, 74, 128)           0         
_________________________________________________________________
lstm_5 (LSTM)                (None, 100)               91600     
_________________________________________________________________
dense_5 (Dense)              (None, 1)                 101       
=================================================================
Total params: 98,309
Trainable params: 98,309
Non-trainable params: 0
_________________________________________________________________
None
T[...]
{'fraction_non_alphanumeric': 0.19702238643773093, 'fraction_numerical': 0.05955227124538144, 'mean_word_length': 4.979857050032488, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '324149', 'license': 'MIT', 'n_tokens_mistral': 2857, 'n_tokens_neox': 2365, 'n_words': 553, 'path': 'archive/dev_jay/LSTM_aa_seq.ipynb', 'repo_name': 'cnyambura/NovoNordisk_Capstone', 'size': '9202'}