<jupyter_start><jupyter_text>Recomendaciones para mejorar la calidad de los datos:1. Realizar lemmatizacion (spacy, clips-pattern.es)2. Usar diccionarios (lista de palabras)3. Usar word embeddings pre-entrenados y ajustarlos (vert)<jupyter_code>## Tensorflow Keras Conversational Model 
# Define an input sequence and process it.
encoder_inputs = Input(shape=(None,))

# Set up encoder, output lstm states
encoder_embed_layer = Embedding(vocab_size_input, embed_size, mask_zero=True)
encoder_embed = encoder_embed_layer(encoder_inputs)

encoder_layers = [LSTM(encoder_hidden_sizes[i], return_sequences=True, go_backwards=True) for i in range(n_encoder_layers)]
encoder_lstms_outputs = []
for i in range(n_decoder_layers):
    if i == 0:
        encoder_lstms_outputs.append(encoder_layers[i](encoder_embed))
    else:
        encoder_lstms_outputs.append(encoder_layers[i](encoder_lstms_outputs[i-1]))

encoder_lstm, state_h, state_c = LSTM(hidden_size, return_state=True,
                                      go_backwards=True)(encoder_lstms_outputs[-1])
encoder_states = [state_h, state_c]

# Set up the decoder, using `encoder_states` as initial state.
decoder_inputs = Input(shape=(None,))

decoder_embed_layer = Embedding(vocab_size_output, embed_size, mask_zero=True)
decoder_embed = decoder_embed_layer(decoder_inputs)

decoder_lstm = LSTM(hidden_size, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embed, initial_state=encoder_states)

decoder_layers = [LSTM(lstm_hidden_sizes[i], return_sequences=True) for i in range(n_decoder_layers)]
decoder_lstms_outputs = []
for i in range(n_decoder_layers):
    if i == 0:
        decoder_lstms_outputs.append(LSTM(lstm_hidden_sizes[i], return_sequences=True)(decoder_outputs))
    elif i == n_decoder_layers-1:
        decoder_lstms_outputs.append(LSTM(lstm_hidden_sizes[i], return_sequences=False)(decoder_lstms_outputs[i-1]))
    else:
        decoder_lstms_outputs.append(LSTM(lstm_hidden_sizes[i], return_sequences=True)(decoder_lstms_outputs[i-1]))

# Create dense vector with next word probability 
decoder_dense = Dense(len(index2word_out), activation='relu')
decoder_outputs = decoder_dense(decoder_lstms_outputs[-1])

# Define the model that will turn 'X_input' and 'y_input' into 'y_target'
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
# Compile model
model.compile(optimizer=Nadam(), loss='sparse_categorical_crossentropy', metrics=[perplexity])
# Model Estructure Summary
model.summary()
## Inference Model
# Encoder
encoder_model = Model(encoder_inputs, encoder_states)

# Decoder inference inputs
decoder_state_input_h = Input(shape=(hidden_size,))
decoder_state_input_c = Input(shape=(hidden_size,))
decoder_states_input = [decoder_state_input_h, decoder_state_input_c]

# Decoder inference
decoder_embed = decoder_embed_layer(decoder_inputs)
decoder_outputs, state_h, state_c = decoder_lstm(decoder_embed, initial_state=decoder_states_input)
decoder_states = [state_h, state_c]

decoder_layers = [LSTM(lstm_hidden_sizes[i], return_sequences=True) for i in range(n_decoder_layers)]
decoder_lstms_outputs = []
for i in range(n_decoder_layers):
    if i == 0:
        decoder_lstms_outputs.append(decoder_layers[i](decoder_outputs))
    else:
        decoder_lstms_outputs.append(decoder_layers[i](decoder_lstms_outputs[i-1]))

decoder_outputs = decoder_dense(decoder_lstms_outputs[-1])
decoder_model = Model(
    [decoder_inputs] + decoder_states_input,
    [decoder_outputs] + decoder_states)

# Save models
encoder_model.save(os.path.join(finalDir,'encoder_model_{}_{}_{}_{}.h5'.format(hidden_size,batch_size,num_epochs,embed_size)))
decoder_model.save(os.path.join(finalDir,'decoder_model_{}_{}_{}_{}.h5'.format(hidden_size,batch_size,num_epochs,embed_size)))
# Define callbacks
model_checkpoint = ModelCheckpoint(os.path.join(chptsDir,'{epoch:02d}_{val_loss:.2f}.chpt'),
                                   monitor='val_loss', verbose=0, save_best_only=True,
                                   save_weights_only=False, mode='auto', period=1)

early_stopping = EarlyStopping(monitor='val_loss', patience=50)

tensorboard = TensorBoard(log_dir=logsDir, histogram_freq=20, batch_size=32, write_graph=True,
                          write_grads=True, embeddings_freq=0, embeddings_layer_names=None,
                          embeddings_metadata=None, embeddings_data=None)

# Fit model
model_history = model.fit([X, Y], targets,
                          batch_size=batch_size,
                          epochs=num_epochs,
                          validation_split=0.05,
                          callbacks=[early_stopping, model_checkpoint, tensorboard])

# Save model history
with open(os.path.join(finalDir,'history_{}_{}_{}_{}.pk'.format(hidden_size,batch_size,num_epochs,embed_size)),'wb') as f:
    pk.dump(model_history.history, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10563955864323661, 'fraction_numerical': 0.006538618716796077, 'mean_word_length': 5.535380507343124, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '560380', 'license': 'MIT', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1585, 'n_words': 264, 'path': 'deep_learning/models/combine_processes/Model_Training.ipynb', 'repo_name': 'Claudio9701/mailbot', 'size': '4894'}