<jupyter_start><jupyter_text>Functions and constants<jupyter_code>TARGETS = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
def multi_roc_auc(y_true, y_score, verbose=False):
    ''' Compute roc auc for each target and then average them
    y_true - dataframe of true targets
    y_score - dataframe of predicted target
    '''
    roc_scores = dict()
    for target in TARGETS:
        roc_score = roc_auc_score(y_true=y_true[target], y_score=y_score[target])
        roc_scores[target] = roc_score

    mean_roc_score = np.mean(list(roc_scores.values()))
    
    if verbose: 
        print('Mean ROC AUC overall all targets: {}'.format(mean_roc_score))
    
    return mean_roc_score, roc_scores<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>data_dir = '../data/external/'
train_df = pd.read_csv(data_dir + 'train.csv')
test_df = pd.read_csv(data_dir + 'test.csv')
X_train = train_df.comment_text
y_train = train_df.loc[:, TARGETS]

X_test = test_df.comment_text<jupyter_output><empty_output><jupyter_text>Word embeddings<jupyter_code>vocab_size = 100000
max_sequence_len = 200
embedding_dim = 100
glove_file = '../glove.6B/glove.6B.100d.txt'
tk = text.Tokenizer(num_words=vocab_size, lower=True)
tk.fit_on_texts(list(X_train))
# Rectify issue with num_words in Keras
# See issue: https://github.com/keras-team/keras/issues/8583#issuecomment-346981336
tk.word_index = {e:i for e,i in tk.word_index.items() if i <= vocab_size} # <= because tokenizer is 1 indexed
tk.word_index[tk.oov_token] = vocab_size + 1
X_train_processed = tk.texts_to_sequences(X_train)
X_test_processed = tk.texts_to_sequences(X_test)

X_train_processed = sequence.pad_sequences(X_train_processed, maxlen=max_sequence_len)
X_test_processed = sequence.pad_sequences(X_test_processed, maxlen=max_sequence_len)
# Load embeddings into dictionary
embeddings_dict = {}
with open(glove_file) as f:
    for line in f:
        values = line.split()
        word = values[0]
        vector = np.asarray(values[1:], dtype='float32')
        embeddings_dict[word] = vector
word_index = tk.word_index

embedding_matrix = np.zeros((vocab_size+1, embedding_dim))
for word, i in word_index.items():
    embedding_vector = embeddings_dict.get(word)
    if embedding_vector is not None:
        embedding_matrix[i, :] = embedding_vector<jupyter_output><empty_output><jupyter_text>Build and train bidirectional LSTM<jupyter_code>input_seq = Input(shape=(max_sequence_len,), dtype='int32')
x = Embedding(input_dim=vocab_size+1,
              output_dim=embedding_dim,
              weights=[embedding_matrix],
              input_length=max_sequence_len,
              trainable=False)(input_seq)

x = SpatialDropout1D(0.3)(x)
x = Bidirectional(LSTM(128, return_sequences=True, dropout=0.1))(x)

max_pool = GlobalMaxPooling1D()(x)
avg_pool = GlobalAveragePooling1D()(x)
x = concatenate([max_pool, avg_pool])

output = Dense(6, activation='sigmoid')(x)
model = Model(input_seq, output)
model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(X_train_processed[:100], y_train.values[:100], batch_size=32, epochs=1, verbose=1)<jupyter_output>Epoch 1/1
100/100 [==============================] - 3s 25ms/step - loss: 0.1866 - acc: 0.9450<jupyter_text>Score on train and test set<jupyter_code>preds_train = model.predict(X_train_processed[:10])
preds_test = model.predict(X_test_processed[:10])
mean_roc_score, roc_scores = multi_roc_auc(y_score=preds_train, y_true=y_train.values[:10])<jupyter_output><empty_output><jupyter_text>Output predictions for submission<jupyter_code>output_df = pd.concat([test_df['id'], preds_test_df], axis=1)
output_df.head()
output_df.to_csv('../data/processed/lstm_glove_submission.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13010271266789572, 'fraction_numerical': 0.023439557545430603, 'mean_word_length': 5.830935251798561, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '183965', 'license': 'MIT', 'n_tokens_mistral': 1474, 'n_tokens_neox': 1377, 'n_words': 230, 'path': 'notebooks/4-bidrectional-LSTMs-glove.ipynb', 'repo_name': 'dkn22/kishan-dat-toxic-comment-challenge', 'size': '3797'}