<jupyter_start><jupyter_text>TESTING MODEL<jupyter_code>results = []
for index in range(len(test)):
    
    choice = index
    test_source_text = test[choice][0]
    target_text = test[choice][2]
    test_source_seq = tokenizer.texts_to_sequences([test_source_text])

    en_initial_states = encoder.init_states(1)
    en_outputs = encoder(tf.constant(test_source_seq), en_initial_states)
    
    de_input = tf.constant([[tokenizer.word_index['<start>']]])
      
    de_state_h, de_state_c = en_outputs[1:]
    out_words = []
    alignments = []

    while True:
        de_output, de_state_h, de_state_c, alignment = decoder(
            de_input, (de_state_h, de_state_c), en_outputs[0])
        de_input = tf.expand_dims(tf.argmax(de_output, -1), 0)
        out_words.append(tokenizer.index_word[de_input.numpy()[0][0]])
        alignments.append(alignment.numpy())
        
        if out_words[-1] == '<end>' or len(out_words) >= 100:
              break

    results.append([test_source_text,target_text, ' '.join(out_words)])


dataframe_results = pd.DataFrame(results, columns = ["Text","Target", "Predicted"])
# if(SAVE):
#     dataframe_results.to_csv(f"Experiment_2/Results/LSTM_results_{NUM_EPOCHS}epochs.csv")
dataframe_results
dataframe_results.to_csv(f"Results/LSTM_{e}epochs.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1361926260346125, 'fraction_numerical': 0.010534236267870579, 'mean_word_length': 4.175097276264592, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '661171', 'license': 'Apache-2.0', 'n_tokens_mistral': 493, 'n_tokens_neox': 482, 'n_words': 61, 'path': 'Experiment_1/Encoder-decoder_exp1.ipynb', 'repo_name': 'DEIB-GECO/GEO-metadata-translator', 'size': '1329'}