<jupyter_start><jupyter_text>Restore the model<jupyter_code>device_name = "cuda:0" if torch.cuda.is_available() else "cpu"
device = torch.device(device_name)

# Load the original hyperparameters.
model_dir = Path("./flickr/models/aevnmt/supervised/")
hparams = Hyperparameters(check_required=False)
hparams_file = model_dir / "hparams"
hparams.update_from_file(hparams_file, override=False)

# Load the vocabularies.
hparams.vocab_prefix = model_dir / "vocab"
hparams.share_vocab = False
vocab_de, vocab_en = load_vocabularies(hparams)

# Restore the model.
model, _, _, translate = create_model(hparams, vocab_de, vocab_en)
model.load_state_dict(torch.load(model_dir / "model/bleu/de-en.pt", map_location=device_name))
model = model.to(device)
print(model)<jupyter_output>AEVNMT(
  (encoder): RNNEncoder(
    (rnn): LSTM(256, 256, batch_first=True, bidirectional=True)
  )
  (decoder): BahdanauDecoder(
    (rnn): LSTM(768, 256, batch_first=True)
    (dropout_layer): Dropout(p=0.5)
    (pre_output_layer): Linear(in_features=1024, out_features=256, bias=True)
    (attention): BahdanauAttention(
      (key_layer): Linear(in_features=512, out_features=256, bias=False)
      (query_layer): Linear(in_features=256, out_features=256, bias=False)
      (scores_layer): Linear(in_features=256, out_features=1, bias=False)
    )
  )
  (language_model): RNNLM(
    (embedder): Embedding(9726, 256, padding_idx=1)
    (rnn): LSTM(256, 256, batch_first=True)
    (dropout_layer): Dropout(p=0.5)
  )
  (tgt_embedder): Embedding(9726, 256, padding_idx=1)
  (dropout_layer): Dropout(p=0.5)
  (encoder_init_layer): Sequential(
    (0): Linear(in_features=32, out_features=256, bias=True)
    (1): Tanh()
  )
  (decoder_init_layer): Sequential(
    (0): Linear(in_features=32, out_fea[...]<jupyter_text>Sample source sentences from the latent space (greedy decoding)<jupyter_code>num_samples = 5

prior = model.prior()
z = prior.sample(sample_shape=[num_samples])

# Construct the LM initial hidden state and inputs.
hidden_lm = tile_rnn_hidden(model.lm_init_layer(z), model.language_model.rnn)
x_init = z.new([vocab_de[SOS_TOKEN] for _ in range(num_samples)]).long()
x_embed = model.language_model.embedder(x_init)

# Keep track of model samples.
x_samples = [x_init.unsqueeze(-1)] # List of [num_samples, 1] integers.

# Sample num_samples source sentences conditioned on z.
for _ in range(hparams.max_decoding_length):
    hidden_lm, logits = model.language_model.step(x_embed, hidden_lm)
    next_word_dist = torch.distributions.categorical.Categorical(logits=logits)
    x = next_word_dist.sample()
    x_embed = model.language_model.embedder(x.squeeze())
    x_samples.append(x)

# Concatenate the samples and convert to sentences.
x_samples = torch.cat(x_samples, dim=-1)
x_samples = batch_to_sentences(x_samples, vocab_de)

# Construct a post-processing pipeline for German.
postprocess = [Detokenizer("de"),
               Recaser("de"), 
               WordDesegmenter(separator=hparams.subword_token)] # Executed in reverse order.
pipeline_de = Pipeline(pre=[], post=postprocess)

# Print the samples.
pp_x_samples = [pipeline_de.post(x) for x in x_samples]
for idx, x in enumerate(pp_x_samples, 1): print(f"{idx}: {x}")<jupyter_output>1: Zwei hunde rennen in einer wiese während new york.
2: Ein kleiner junge liegt auf einem holzzaun über dem strand.
3: Diese menschen lesen verschiedene pinsame tag etwas in der pfanne.
4: Ein obdachloser mann sitzt auf einem hocker vor einer skulptur und liest seinen fahrrad.
5: Ein mann mit stock, der den kamm in seinem fahrrad fährt.<jupyter_text>Sample translations from the approximate posterior (greedy decoding)<jupyter_code>num_samples = 5
x_samples = ["in kleines blondes mädchen hält ein sandwich ."] * num_samples
x_in, _, seq_mask_x, seq_len_x = create_batch(x_samples, vocab_de, device)

# Infer q(z|x). 
qz = model.approximate_posterior(x_in, seq_mask_x, seq_len_x)
z = qz.sample()

# Encode the source sentences
encoder_outputs, encoder_final = model.encode(x_in, seq_len_x, z)

# Create the initial hidden state of the TM.
hidden_tm = model.init_decoder(encoder_outputs, encoder_final, z)

# Sample target sentences conditional on the source and z.
y_samples = ancestral_sample(model.decoder,
                             model.tgt_embed,
                             model.generate,
                             hidden_tm,
                             encoder_outputs, encoder_final,
                             seq_mask_x,
                             vocab_en[SOS_TOKEN],
                             vocab_en[EOS_TOKEN],
                             vocab_en[PAD_TOKEN],
                             hparams.max_decoding_length,
                             greedy=False)["sample"]
y_samples = batch_to_sentences(y_samples, vocab_en)

# Construct a post-processing pipeline for English.
postprocess = [Detokenizer("en"),
               Recaser("en"), 
               WordDesegmenter(separator=hparams.subword_token)] # Executed in reverse order.
pipeline_en = Pipeline(pre=[], post=postprocess)

# Print the samples.
pp_y_samples = [pipeline_en.post(y) for y in y_samples]
for idx, y in enumerate(pp_y_samples, 1): print(f"{idx}: {y}")<jupyter_output>1: Young blond children, holding a sandwich.
2: In pink and little girls holding a pair of sandwich.
3: Shirt to short blond-hair, holding a sandwich
4: A little girl with blond-hair is holding a sandwich.
5: At young blond girl holding a sandwich.
{'fraction_non_alphanumeric': 0.10624656530500091, 'fraction_numerical': 0.016669719728888074, 'mean_word_length': 4.064935064935065, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '167475', 'license': 'MIT', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1870, 'n_words': 435, 'path': 'demo/AEVNMT.ipynb', 'repo_name': 'vitaka/AEVNMT.pt', 'size': '5459'}