<jupyter_start><jupyter_text>Load models Load `NeuralTranslationModel`<jupyter_code>max_len_in_chinese_tokenized=46
max_word_index=3438
translation_model = NeuralTranslationModel(encoder_input_shape=(max_len_in_chinese_tokenized,128),
                                           decoder_input_shape=(max_word_index + 1, 128))

# build the model by calling it
encoder_in=tf.zeros([1,max_len_in_chinese_tokenized,128])
decoder_in=tf.Variable([[1]])
print(f'encoder_in shape: {encoder_in.shape}')
print(f'decoder_in shape: {decoder_in.shape}')
_ = translation_model((encoder_in,decoder_in))
translation_model.load_weights('models/eng-to-ch/checkpoint_best/checkpoint')<jupyter_output><empty_output><jupyter_text>Load pre-trained English embedding<jupyter_code>embedding_layer = load_model('models/tf2-preview_nnlm-en-dim128_1')<jupyter_output>WARNING:tensorflow:SavedModel saved prior to TF 2.5 detected when loading Keras model. Please ensure that you are saving the model with model.save() or tf.keras.models.save_model(), *NOT* tf.saved_model.save(). To confirm, there should be a file named "keras_metadata.pb" in the SavedModel directory.<jupyter_text>---  Load dataset<jupyter_code>df = pd.read_json('data/cmn-processed-tokenized.json')
df.head()
chinese = df['chinese']
english = df['english']
english_split = df['english_split']<jupyter_output><empty_output><jupyter_text>---  Load tokenizer<jupyter_code>tokenizer=[]
with open('data/tokenizer.json') as f:
    data = json.load(f)
    tokenizer = tokenizer_from_json(data)

tokenizer_config = tokenizer.get_config()
word_index = json.loads(tokenizer_config['word_index'])
start_token = word_index['<start>']
end_token = word_index['<end>']
inv_chinese_index = {value:key for key,value in tokenizer.word_index.items()}<jupyter_output><empty_output><jupyter_text>---  Making translation<jupyter_code>def translate(english_split_in):
    eng_embedding = embedding_layer(english_split_in)
    eng_padded = tf.pad(eng_embedding, 
                        [[max_len_in_chinese_tokenized-len(eng_embedding), 0], 
                         [0, 0]], 
                        constant_values = 0)
    english_expand = tf.expand_dims(eng_padded, 0)
    hidden_state, cell_state = translation_model.encoder(english_expand)

    current_translation = []
    current_token = tf.Variable([[start_token]])

    while (len(current_translation) <= max_len_in_chinese_tokenized):
        out1, hidden_state, cell_state = translation_model.decoder(current_token,hidden_state,cell_state)
        out2 = tf.argmax(out1, axis=2).numpy()[0,0]
        current_token = tf.Variable([[out2]])
        if out2 == end_token:
            break
        else:
            current_translation.append(out2)
    inv_tokenized = [inv_chinese_index[w] for w in current_translation]
    inv_tokenized_string = ' '.join(inv_tokenized)
    return inv_tokenized_string
full_test_count = 10
test_indices = np.random.choice(english_split.index,full_test_count)
test_indices
full_translations=[]
for idx in test_indices:
    tmp_result = translate(english_split[idx])
    full_translations.append(tmp_result)
df_full_results = pd.DataFrame(data={'English-Dataset':english.loc[test_indices].reset_index(drop=True), 
                                     'Chinese-Dataset':chinese.loc[test_indices].reset_index(drop=True),
                                     'Chinese-Translated':full_translations}, 
                               index=range(len(test_indices)))
df_full_results<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1109846328969835, 'fraction_numerical': 0.011667615253272624, 'mean_word_length': 4.570522979397781, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '334437', 'license': 'MIT', 'n_tokens_mistral': 1181, 'n_tokens_neox': 1146, 'n_words': 170, 'path': '04-TranslateEnglishToChinese.ipynb', 'repo_name': 'raymondngiam/neural-translation-model-eng-to-ch', 'size': '3514'}