<jupyter_start><jupyter_text>0.0 - Imports<jupyter_code>import re
import io
import os
import tensorflow        as tf
import numpy             as np
import pandas            as pd
import seaborn           as sns
import matplotlib.pyplot as plt


from tensorflow.keras.preprocessing.text     import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences<jupyter_output><empty_output><jupyter_text>0.1 Caminhos dos diretórios e arquivos<jupyter_code>BASE_DIR = os.path.abspath('..')
DATA_DIR = os.path.join(BASE_DIR, 'data')
DATA_INTERIM_DIR = os.path.join(DATA_DIR, 'interim')
DATA_PROCESSED_DIR = os.path.join(DATA_DIR, 'external')
DATA_TRAIN = os.path.join(DATA_INTERIM_DIR, 'train_v1.csv')
DATA_VALIDATION =os.path.join(DATA_INTERIM_DIR, 'validation_v1.csv')<jupyter_output><empty_output><jupyter_text>0.1 loading Data<jupyter_code>df_train = pd.read_csv(DATA_TRAIN)<jupyter_output><empty_output><jupyter_text>0.2 Helper Functions<jupyter_code># validação da acurácia do modelo
def plot_graphs(history, string):
    sns.set()
    plt.plot(history.history[string])
    plt.plot(history.history['val_'+string])
    plt.xlabel('Epochs')
    plt.ylabel(string)
    plt.legend([string, 'val_'+string])
    plt.show()<jupyter_output><empty_output><jupyter_text>1.0 Preprocessing<jupyter_code>df=df_train.copy()
df['review_text'] = df['review_text'].apply(str)<jupyter_output><empty_output><jupyter_text>1.1 Configuration to model and tokenization<jupyter_code>vocab_size = 30000 # Quantidade de palavras
embedding_dim = 16
max_length = 3000 # Tamanho de cada frase
trunc_type = 'post'
padding_type = 'post'
oov_tok = 'XXXX'
training_size = 85000<jupyter_output><empty_output><jupyter_text>1.2 Train and Test<jupyter_code>len(df)
df.columns
sentences = df['review_text']
labels = df['recommend_to_a_friend_num']
# Divisão em 85 / 15 (treino e teste)
training_sentences = sentences[0:training_size]
training_labels = labels[0:training_size]
testing_sentences = sentences[training_size:]
testing_labels = labels[training_size:]
testing_labels_final = np.array(testing_labels)
training_labels_final = np.array(training_labels)<jupyter_output><empty_output><jupyter_text>Distribuition of the data train and test<jupyter_code>training_labels.value_counts(normalize=True)
testing_labels.value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>1- Sim (Recomendaria o produto para um amigo)2 - Não (Não recomendaria um produto para um amigo)  1.3 Tokenização<jupyter_code>tokenizer = Tokenizer(  num_words = vocab_size,
                        oov_token=oov_tok)
tokenizer.fit_on_texts(training_sentences) # Tokenizando as palavras do treinamento

word_index = tokenizer.word_index

# Criando a sequência de palavras para o treinamento, Substituindo as palavras pelos seus tokens.
training_sequences = tokenizer.texts_to_sequences(training_sentences) 
training_padded = pad_sequences(training_sequences,
                                maxlen= max_length,
                                padding=padding_type,
                                truncating= trunc_type)

# Repetindo o processo para os dados de teste.
testing_sequences = tokenizer.texts_to_sequences(testing_sentences) 
testing_padded = pad_sequences(testing_sequences,
                                maxlen= max_length,
                                padding=padding_type,
                                truncating= trunc_type)<jupyter_output><empty_output><jupyter_text>2.0 Model  2.1 Build Model - Model_1<jupyter_code># Buid Model
model = tf.keras.Sequential([
    tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length = max_length), # Embedding das palavras tokenizadas
    tf.keras.layers.LSTM(32),
    tf.keras.layers.Dense(64, activation = 'relu'),
    tf.keras.layers.Dense(6, activation = 'relu'),
    tf.keras.layers.Dense(1, activation = 'sigmoid')
    ])

# Compile
loss = 'binary_crossentropy'
optimizer = 'adam'
metrics = ['accuracy']

model.compile(  loss= loss,
                optimizer = optimizer,
                metrics = metrics)

model.summary()
# Mostrar modelo
dot_img_file = os.path.join(os.path.join(BASE_DIR,'docs'),'image_model.png')
tf.keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True)<jupyter_output>You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model/model_to_dot to work.<jupyter_text>2.1.1 Model Fit<jupyter_code>n_epochs = 15
history = model.fit(training_padded,
                    training_labels_final,
                    epochs = n_epochs,
                    validation_data = (testing_padded, testing_labels_final),
                    verbose = 2)<jupyter_output>Epoch 1/15<jupyter_text>2.1.3 Save Model<jupyter_code>VERSION_MODEL = '1'
path_model = os.path.join(os.path.join(BASE_DIR,'models'),VERSION_MODEL)
model.save(path_model)<jupyter_output>INFO:tensorflow:Assets written to: D:\projetos\Projetos_NLP\text_classifier\models\model00_a\assets<jupyter_text>3.0 Performance model  3.1 Load Model<jupyter_code>reconstructed_model = tf.keras.models.load_model(path_model)<jupyter_output><empty_output><jupyter_text>3.2 Results<jupyter_code>plot_graphs(history, 'accuracy')
plot_graphs(history, 'loss')<jupyter_output><empty_output><jupyter_text>4.0 Embeddings<jupyter_code># Obtendo os resultados da camada de embedding (primeira camada)
e = model.layers[0]
weights = e.get_weights()[0]
print(weights.shape) # Shape: (Vocab_size, embedding_dim)
# 20000 Palavras e 16 dimensões
# Para sermos capazes de plotar as palavras como vetores, devemos obter o índice de palavras ao contrário
reverse_word_index = dict([(value, key) for (key,value) in word_index.items()])<jupyter_output><empty_output><jupyter_text>4.1 Escrevendo os vetores em formatos de metadados<jupyter_code>path_vecs = os.path.join(os.getenv('PATH_BASE'),'data\\external\\vecs.tsv')
path_meta = os.path.join(os.getenv('PATH_BASE'),'data\\external\\meta.tsv')

out_v = io.open(path_vecs, 'w', encoding='utf-8')
out_m = io.open(path_meta, 'w', encoding='utf-8')
for word_num in range(1, vocab_size):
    word = reverse_word_index[word_num]
    embeddings = weights[word_num]
    out_m.write(word + '\n')
    out_v.write('\t'.join([str(x) for x in embeddings]) + '\n')
out_m.close()
out_v.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10962145110410094, 'fraction_numerical': 0.014195583596214511, 'mean_word_length': 4.893122676579925, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 62, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '443642', 'license': 'MIT', 'n_tokens_mistral': 2320, 'n_tokens_neox': 2240, 'n_words': 391, 'path': 'notebooks/1-Model-RNN.ipynb', 'repo_name': 'KaissonFerreira/text_classifier', 'size': '6340'}