<jupyter_start><jupyter_text><jupyter_code>!pip install xgboost
from tensorflow import keras
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

# los REALES 😎
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import plotly.express as px

import sklearn as sk
import xgboost as xgb<jupyter_output><empty_output><jupyter_text>Reading data<jupyter_code>data = pd.read_csv('train.csv.zip')
def add_y_col(dataframe):
    d_frame = dataframe.copy()
    # columns that are more or less toxic
    cols_info = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']

    # If the row has one or more negative categories, it is considered as Y = negative 
    d_frame['Y'] = d_frame[cols_info].sum(axis=1).apply(lambda x: 1 if x > 0 else 0)
    num_neg = d_frame['Y'].value_counts()[1]
    positive = d_frame[d_frame['Y'] == 0].sample(num_neg)[['comment_text', 'Y']].reset_index(drop=True)
    negative = d_frame[d_frame['Y'] == 1][['comment_text', 'Y']].sample(frac=1).reset_index(drop=True)

    # intercalate rows
    return pd.concat([positive, negative]).sort_index(kind='merge').reset_index(drop=True)

print(f'Total number of rows in the training dataset: {data.shape[0]}')<jupyter_output>Total number of rows in the training dataset: 159571<jupyter_text>Clean and transform data<jupyter_code>df = add_y_col(data)
X_raw = df['comment_text']
y = df['Y'].to_numpy().reshape((-1, 1)).astype(float)
X_train_raw, X_test_raw, y_train, y_test = train_test_split(X_raw, 
                                                    y,
                                                    test_size = 0.20)
# Taken from documentation 
# https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer
MAX_COLS = 2000
# Create a Tokenizer object
## This tokenizer will learn the 'num_words' most popular words in the vocab and
## assign a tfidf metric to it
token = Tokenizer(num_words=MAX_COLS,
                  filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
                  lower = True, split = ' ', oov_token = "XDDD",
                  document_count = 0
                  )

token.fit_on_texts(X_train_raw)

X_train_matrix = token.texts_to_matrix(X_train_raw, mode='tfidf')
X_test_matrix = token.texts_to_matrix(X_test_raw, mode='tfidf')
# Taken from documentation 
# https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer
MAX_WORDS = 3000
MAX_LEN = 120

# Create a Tokenizer object
## This tokenizer will learn the most MAX_WORDS words and
## assign an integer in order to those words
token_seq = Tokenizer(
                  num_words=MAX_WORDS,
                  filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
                  lower = True, split = ' ', oov_token = "XDDD",
                  )

token_seq.fit_on_texts(X_train_raw)

X_train_seq = token_seq.texts_to_sequences(X_train_raw)
X_test_seq = token_seq.texts_to_sequences(X_test_raw)

X_train_final = pad_sequences(X_train_seq, padding = 'post', maxlen = MAX_LEN)
X_test_final = pad_sequences(X_test_seq, padding = 'post', maxlen = MAX_LEN)
import io 
import json

## TO SAVE THE VOCABULARY LEARNED BY THE TOKENIZER
tokenizer_json = token_seq.to_json()
with io.open('/content/config_tokenizer.json', 'w', encoding='utf-8') as f:
    f.write(json.dumps(tokenizer_json, ensure_ascii=False))
# vocab = {} 
# for i in X_train_seq:
#     for n in i:
#         if n not in vocab.keys():
#             vocab[n] = 1
#         else:
#             vocab[n] += 1
# (np.array([len(i) for i in X_train_seq]).astype(int) > 300).sum()<jupyter_output><empty_output><jupyter_text>Training the model<jupyter_code>resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')
tf.config.experimental_connect_to_cluster(resolver)
# This is the TPU initialization code that has to be at the beginning.
tf.tpu.experimental.initialize_tpu_system(resolver)
print("All devices: ", tf.config.list_logical_devices('TPU'))
strategy = tf.distribute.TPUStrategy(resolver)
 with strategy.scope():
    model = keras.models.Sequential()
    model.add(keras.layers.Embedding(MAX_WORDS, 4, input_length=X_train_final.shape[1]))
    model.add(keras.layers.Bidirectional(keras.layers.LSTM(5, return_sequences=True, recurrent_dropout=0.5, dropout=0.5)))
    model.add(keras.layers.GlobalMaxPooling1D())
    model.add(keras.layers.Dense(8, activation='relu'))
    model.add(keras.layers.Dense(1, activation='sigmoid'))

    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

print(model.summary())

epochs = 15
# batch_size = int(X_train_final.shape[0] / 9)
batch_size = 512

history = model.fit(X_train_final, 
                    y_train, 
                    validation_data=(X_test_final, y_test),
                    epochs=epochs, batch_size=batch_size)
model.save('/content/modelo_entrenado_file.h5')
predsss = model.predict(X_test_final)
np.apply_along_axis(lambda x: 1 if x > 0.5 else 0, 1, predsss)
predsss<jupyter_output><empty_output><jupyter_text>Trying with non-neural networks<jupyter_code>model2 = xgb.XGBClassifier(objetive='binary:logistic', seed=1234,verbosity=3, n_jobs=-1)
model2.fit(X_train_matrix, np.ravel(y_train))
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_jobs=-1, verbose=3)
rf.fit(X_train_matrix, np.ravel(y_train))
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score

# Reporting most important scores
def report_scores(y_true, y_pred):

    precision = precision_score(y_true, y_pred, average='weighted')
    recall = recall_score(y_true, y_pred, average='weighted')
    f1 = f1_score(y_true, y_pred, average='weighted')
    accuracy = accuracy_score(y_true, y_pred)

    print(
        f'''\n\nThe weighted average precision is {precision:.{3}}\n
The weighted average recall is {recall:.{3}}\n
The weighted average f1-score is {f1:.{3}}\n
The accuracy is {accuracy:.{3}}
        '''

    )
report_scores(np.ravel(y_test), rf.predict(X_test_matrix))
report_scores(np.ravel(y_test), model2.predict(X_test_matrix))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12948014207297384, 'fraction_numerical': 0.013077171456247982, 'mean_word_length': 4.281329923273657, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '522829', 'license': 'MIT', 'n_tokens_mistral': 2273, 'n_tokens_neox': 2156, 'n_words': 431, 'path': '02_NLP_model.ipynb', 'repo_name': 'oswaldochan/COVID19-Tweet-Toxic-Classification', 'size': '6194'}