<jupyter_start><jupyter_text>Do2Vec<jupyter_code>d2v = Doc2Vec.load(f"{os.getcwd()}/../../models/d2v.model")

n_textos = []
for texto in textos:
    doc_vec = d2v.infer_vector(texto[0].split(), epochs=500)
    n_textos.append(doc_vec)
n_textos = np.asarray(n_textos)
X = n_textos
y = textos[:, 1].astype(np.int).ravel()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
# # For Conv & LSTM
print(X_train.shape)
print(X_test.shape)
X_train = X_train.reshape(2048, 300, -1)
X_test = X_test.reshape(512, 300, -1)
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
model = Sequential(name='Doc2Vec')
# model.add(Conv1D(128, 2, padding='valid', activation='elu'))
# model.add(Conv1D(128, 2, padding='valid', activation='elu'))
# model.add(Conv1D(128, 2, padding='valid', activation='elu'))
# model.add(Conv1D(32, 5, padding='same', activation='elu'))
# model.add(MaxPooling1D())
# model.add(Dropout(0.5))
# model.add(BatchNormalization())
# model.add(Dropout(0.5))
model.add(Bidirectional(LSTM(128, return_sequences=True)))
model.add(Bidirectional(LSTM(128)))
# model.add(Dropout(0.5))
# model.add(GlobalMaxPooling1D())
# model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# model.summary()
epochs = 10
checkpoint_filepath_d2v = './weights/weights_d2v.hdf5'
model_checkpoint_callback = ModelCheckpoint(
    filepath=checkpoint_filepath_d2v,
    save_weights_only=True,
    monitor='val_accuracy',
    mode='max',
    verbose=0,
    save_best_only=True)

history = model.fit(
    X_train, y_train, 
    batch_size=batch_size, 
    epochs=epochs, 
    verbose=0,
    shuffle=True,
    validation_data=(X_test, y_test),
    callbacks=[model_checkpoint_callback]
)

model.load_weights(checkpoint_filepath_d2v)
score = model.evaluate(X_test, y_test, batch_size=batch_size, verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', round(score[1] * 100))
pred = (model.predict(X_test) > 0.5).astype(np.int).ravel()
print(classification_report(y_test, pred))

df_cm = confusion_matrix(y_test, pred)
plt.figure(figsize=(6, 4))
sn.heatmap(df_cm, annot=True, fmt="d")
plt.tight_layout()
plt.show()
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)

plt.plot(history.history["accuracy"], label="Acc", c="C0")
plt.plot(history.history["val_accuracy"], label="Val. Acc", c="C2")
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(history.history["loss"], label="Loss", c="C1")
plt.plot(history.history["val_loss"], label="Val. Loss", c="C3")
plt.xticks(range(0, epochs, 2))
plt.legend()
plt.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18492901346923918, 'fraction_numerical': 0.0389515835456862, 'mean_word_length': 6.806818181818182, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '161885', 'license': 'MIT', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1106, 'n_words': 160, 'path': 'code/notebooks/validacoes/alpha_avaliacao_deep_learning_classificacao_comentarios.ipynb', 'repo_name': 'victorleal/Covid19-Twitter-Tracker', 'size': '2747'}