<jupyter_start><jupyter_text>Hyperparameters<jupyter_code>max_length = 32
embedding_dim = 300
num_epochs = 6
batch_size = 512

trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"
val_portion=.05<jupyter_output><empty_output><jupyter_text>Training/Validation data preparation<jupyter_code>corpus = []

train_pd = pd.read_csv("../train.csv").fillna(' ')

train_text = train_pd['question_text']
labels_text = train_pd['target']

list_item = list(zip(train_text,labels_text))
num_sentences = len(list_item)
corpus = list_item
print(num_sentences)
print(len(corpus))
print(corpus[1])
training_size = len(corpus)

sentences=[]
labels=[]
random.shuffle(corpus)
for x in range(training_size):
    sentences.append(corpus[x][0])
    labels.append(corpus[x][1])

tokenizer = Tokenizer()
tokenizer.fit_on_texts(sentences)

word_index = tokenizer.word_index
vocab_size=len(word_index)

sequences = tokenizer.texts_to_sequences(sentences)
padded = pad_sequences(sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)

split = int(val_portion * training_size)

val_sequences = padded[0:split]
training_sequences = padded[split:training_size]
val_labels = labels[0:split]
training_labels = labels[split:training_size]
print(vocab_size)
print(word_index['the'])<jupyter_output>222161
1<jupyter_text>Loading glove pretrained weights matrix<jupyter_code>glove_txt_path = "../embeddings/glove.840B.300d/glove.840B.300d.txt"

embeddings_index = {};
with open(glove_txt_path, encoding="utf-8") as f:
    for line in f:
        values = line.split();
        
        if len(values) != 301:
            continue
        
        word = values[0];
        coefs = np.asarray(values[1:], dtype='float32');
        embeddings_index[word] = coefs;
embeddings_matrix = np.zeros((vocab_size+1, embedding_dim));
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word);
    if embedding_vector is not None:
        embeddings_matrix[i] = embedding_vector;
print(len(embeddings_matrix))<jupyter_output>222162<jupyter_text>Model<jupyter_code>model = tf.keras.Sequential([
    tf.keras.layers.Embedding(vocab_size+1, embedding_dim, input_length=max_length, weights=[embeddings_matrix], trainable=False),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv1D(64, 5, activation='relu'),
    tf.keras.layers.MaxPooling1D(pool_size=4),
    tf.keras.layers.LSTM(64),
    tf.keras.layers.Dense(1, activation='sigmoid')
])
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
model.summary()
training_step = len(training_labels)//batch_size
validation_step = len(val_labels)//batch_size

history = model.fit(training_sequences, 
                    training_labels, 
                    epochs=num_epochs, 
                    batch_size=batch_size,
                    validation_data=(val_sequences, val_labels), 
                    verbose=1)

model.save("model.h5")
model.save_weights("weights.h5")
model_str = model.to_json()
f = open("model_json.json", "w")
f.write(model_str)
f.close()

print("Training Complete")
from keras.models import load_model
from keras.models import model_from_json

#model = load_model("model.h5")
json_file = open('model_json.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
model.load_weights("weights.h5")<jupyter_output><empty_output><jupyter_text>Drawing Accuracy & Loss<jupyter_code>import matplotlib.image  as mpimg
import matplotlib.pyplot as plt
%matplotlib inline

#-----------------------------------------------------------
# Retrieve a list of list results on training and test data
# sets for each training epoch
#-----------------------------------------------------------
acc=history.history['acc']
val_acc=history.history['val_acc']
loss=history.history['loss']
val_loss=history.history['val_loss']

epochs=range(len(acc)) # Get number of epochs

#------------------------------------------------
# Plot training and validation accuracy per epoch
#------------------------------------------------
plt.plot(epochs, acc, 'r')
plt.plot(epochs, val_acc, 'b')
plt.title('Training and validation accuracy')
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend(["Accuracy", "Validation Accuracy"])

plt.figure()

#------------------------------------------------
# Plot training and validation loss per epoch
#------------------------------------------------
plt.plot(epochs, loss, 'r')
plt.plot(epochs, val_loss, 'b')
plt.title('Training and validation loss')
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.legend(["Loss", "Validation Loss"])

plt.figure()
plt.show()

# Expected Output
# A chart where the validation loss does not increase sharply!<jupyter_output><empty_output><jupyter_text>Detection of best threshold<jupyter_code>def get_best_thresh(predictions, labels):
    max_pred = max(predictions)
    min_pred = min(predictions)
    print(min_pred, max_pred)
    
    thresholds = np.linspace(min_pred, max_pred, num=20, endpoint=False)
    thresholds = np.around(thresholds, decimals=2)
    
    max_count = 0
    best_thresh = 0

    print("total count: ", len(predictions))
    for thresh in thresholds:
        int_labels = (predictions>thresh).astype(int)
        boolarr = int_labels == np.array(labels).reshape(len(labels), 1)
        count_true = np.sum(boolarr)
        print("thresh: ", thresh, " count true: ", count_true)
        if count_true > max_count:
            max_count = count_true
            best_thresh = thresh

    print("best thresh: ", best_thresh)
    return best_thresh
val_predictions = model.predict(val_sequences, batch_size=batch_size, verbose=2)
best_val_thresh = get_best_thresh(val_predictions, val_labels)
val_int_labels = (val_predictions>best_val_thresh).astype(int)
count_1_true = 0
for i in range(len(val_int_labels)):
    if (val_int_labels[i] == val_labels[i]) and (val_labels[i] == 1):
        count_1_true += 1
        
print("count Insincere questions for best_thresh: ", count_1_true)
print("count correct Insincere questions: ", np.sum(np.array(val_labels) == 1))<jupyter_output>count Insincere questions for best_thresh:  2574
count correct Insincere questions:  4083<jupyter_text>Prediction for test data<jupyter_code>test_pd = pd.read_csv("../test.csv").fillna(' ')
test_corpus = test_pd['question_text']

test_sentences=[]
for x in range(len(test_corpus)):
    test_sentences.append(test_corpus[x][0])
    
sequences_test = tokenizer.texts_to_sequences(test_sentences)
test_padded = pad_sequences(sequences_test, maxlen=max_length, padding=padding_type, truncating=trunc_type)

submit_test_sequences = test_padded

pred_prob = model.predict(submit_test_sequences, batch_size=batch_size, verbose=2)
max_pred = max(pred_prob)
min_pred = min(pred_prob)
print(min_pred, max_pred, best_val_thresh)
best_test_threshold = (max_pred - min_pred)*best_val_thresh
print("the threshold is 0.53 from the test pred range: ", best_test_threshold)
submission = pd.DataFrame.from_dict({'qid': test_pd['qid']})
submission['prediction'] = (pred_prob>best_test_threshold).astype(int)
submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>Visualization If you train an Embedding layer (here we didn't train. we use a pretrained 'glove' trainable=False)you can project these vectors and visualize them, it's nice to seeyou need two inputs: vecs.tsv and meta.tsv https://projector.tensorflow.org/<jupyter_code>reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])

e = model.layers[0]
weights = e.get_weights()[0]
print(weights.shape) # shape: (vocab_size, embedding_dim)
import io

out_v = io.open('vecs.tsv', 'w', encoding='utf-8')
out_m = io.open('meta.tsv', 'w', encoding='utf-8')
for word_num in range(1, vocab_size):
    word = reverse_word_index[word_num]
    embeddings = weights[word_num]
    out_m.write(word + "\n")
    out_v.write('\t'.join([str(x) for x in embeddings]) + "\n")
out_v.close()
out_m.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16123392031972025, 'fraction_numerical': 0.012364181341326339, 'mean_word_length': 4.378106111484217, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '570075', 'license': 'MIT', 'n_tokens_mistral': 2822, 'n_tokens_neox': 2708, 'n_words': 497, 'path': 'Insincere_Questions_Classification.ipynb', 'repo_name': 'anas-899/Binary_Questions_Classification', 'size': '8007'}