<jupyter_start><jupyter_text><jupyter_code>import os
import time
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from google.colab import drive
drive.mount('/gdrive')
# mount nguyentrunghieu
cd "/gdrive/My Drive"
EMBEDDING_FILE = 'embeddings/GoogleNews-vectors-negative300.bin'
path_train = "nlp/data/train.csv"
path_test = "nlp/data/test.csv"
train = pd.read_csv(path_train)
# test = pd.read_csv(path_test)
print(train.shape)
# print(test.shape)
train, val = train_test_split(train, test_size = 0.1, random_state = 2019) 
embed_size = 300
max_features = 50000
maxlen = 100
## fill up the missing values
train_X = train["question_text"].fillna("_na_").values
val_X = val["question_text"].fillna("_na_").values
# test_X = test["question_text"].fillna("_na_").values

#Tokenize the sentences
tokenizer = Tokenizer(num_words = max_features)
tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
val_X = tokenizer.texts_to_sequences(val_X)
# test_X = tokenizer.texts_to_sequences(test_X)
#padding
train_X = pad_sequences(train_X, maxlen = maxlen)
val_X = pad_sequences(val_X, maxlen = maxlen)
# test_X = pad_sequences(test_X, maxlen = maxlen)

train_Y = train['target'].values
val_Y = val['target'].values
print(train_Y.shape)
print(val_Y.shape)
from gensim.models import KeyedVectors

embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)

word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros((50000, 300))
for word, i in word_index.items():
    if i >= max_features: continue
    if word in embeddings_index:
        embedding_vector = embeddings_index.get_vector(word)
        embedding_matrix[i] = embedding_vector
!pip install h5py pyyaml
# !pip install tf_nightly
from __future__ import absolute_import, division, print_function, unicode_literals

import os

import tensorflow as tf
from tensorflow import keras

tf.__version__
import tensorflow as tf
from keras import backend as K

def f1(y_true, y_pred):
    def recall(y_true, y_pred):
        """Recall metric.

        Only computes a batch-wise average of recall.

        Computes the recall, a metric for multi-label classification of
        how many relevant items are selected.
        """
        true_positives = K.sum(y_true * y_pred)
        possible_positives = K.sum(y_true)
        recall = true_positives / (possible_positives + K.epsilon())
        return recall

    def precision(y_true, y_pred):
        """Precision metric.

        Only computes a batch-wise average of precision.

        Computes the precision, a metric for multi-label classification of
        how many selected items are relevant.
        """
        true_positives = K.sum(y_true * y_pred)
        predicted_positives = K.sum(y_pred)
        precision = true_positives / (predicted_positives + K.epsilon())
        return precision
    
    Y_pred = tf.cast(tf.less(0.4, y_pred), tf.float32)
    
    precision = precision(y_true, Y_pred)
    recall = recall(y_true, Y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))
from keras.models import Sequential
from keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation
from keras.layers.embeddings import Embedding

def create_model():
    model = Sequential()
    model.add(Embedding(50000, 300, input_length=100, weights=[embedding_matrix], trainable=False))
    model.add(Dropout(0.2))
    model.add(Conv1D(64, 5, activation='relu'))
    model.add(MaxPooling1D(pool_size=4))
    model.add(LSTM(300))
    model.add(Dense(1, activation='sigmoid'))
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
    
    return model
model = create_model()

model.summary()


checkpoint_path = "training_2/cp-{epoch:04d}.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)

cp_callback = tf.keras.callbacks.ModelCheckpoint(
    checkpoint_path, verbose=1, save_weights_only=True,
    # Save weights, every 5-epochs.
    period=2)
model.save_weights(checkpoint_path.format(epoch=0))
history = model.fit(train_X, train_Y, batch_size=512, epochs=10, callbacks = [cp_callback],validation_data=(val_X, val_Y))
model.save_weights("Naive_weights_GoogleNews.h5")
predict_val_y = model.predict(val_X, batch_size = 1024, verbose = 1)


history.history.keys()
import matplotlib.pyplot as plt

plt.plot(history.history['f1'])
plt.plot(history.history['val_f1'])
plt.title('model performance')
plt.ylabel('f1')
plt.xlabel('epoch')
plt.legend(['train','val'], loc='upper left')

plt.show()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train','test'], loc='upper right')

plt.show()
scores = []
for thresh in np.arange(0.1, 0.901, 0.01):
    thresh = np.round(thresh, 2)
    score = metrics.f1_score(val_Y, (predict_val_y>thresh).astype(int))
    scores.append(score)
    print("F1 score at threshold {0} is {1}".format(thresh, score))
    
print("Best F1 score:", max(scores))<jupyter_output>F1 score at threshold 0.1 is 0.5242086648983201
F1 score at threshold 0.11 is 0.5336279019231465
F1 score at threshold 0.12 is 0.542773517098292
F1 score at threshold 0.13 is 0.5491320754716981
F1 score at threshold 0.14 is 0.5566455087516831
F1 score at threshold 0.15 is 0.5626346086071191
F1 score at threshold 0.16 is 0.5702436495593571
F1 score at threshold 0.17 is 0.5760447035957241
F1 score at threshold 0.18 is 0.5808626999465439
F1 score at threshold 0.19 is 0.5867023053792182
F1 score at threshold 0.2 is 0.5908436997545908
F1 score at threshold 0.21 is 0.5947614352467098
F1 score at threshold 0.22 is 0.5989058700937826
F1 score at threshold 0.23 is 0.6028745989187289
F1 score at threshold 0.24 is 0.6066018037229552
F1 score at threshold 0.25 is 0.6101603557472038
F1 score at threshold 0.26 is 0.612827034883721
F1 score at threshold 0.27 is 0.6162112514351321
F1 score at threshold 0.28 is 0.6191979205347197
F1 score at threshold 0.29 is 0.6220014992503747
F1 score at threshold 0.[...]
{'fraction_non_alphanumeric': 0.10464940178706648, 'fraction_numerical': 0.0784491897622293, 'mean_word_length': 4.833922261484099, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '399008', 'license': 'MIT', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2276, 'n_words': 575, 'path': 'group_Relu_GoogleNews_embedding.ipynb', 'repo_name': 'Tungthanhlee/Quora-Insincere-Question-Classification', 'size': '6603'}