<jupyter_start><jupyter_text>Encoder-Decoder Analysis  Model Architecture<jupyter_code>report_file = '/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_6k_trV_200_512_04drb/encdec_noing15_6k_trV_200_512_04drb.json'
log_file = '/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_6k_trV_200_512_04drb/encdec_noing15_6k_trV_200_512_04drb_logs.json'

import json
import matplotlib.pyplot as plt
with open(report_file) as f:
    report = json.loads(f.read())
with open(log_file) as f:
    logs = json.loads(f.read())
print'Encoder: \n\n', report['architecture']['encoder']
print'Decoder: \n\n', report['architecture']['decoder']<jupyter_output>Encoder: 

nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 

nn.gModule<jupyter_text>Perplexity on Each Dataset<jupyter_code>print('Train Perplexity: ', report['train_perplexity'])
print('Valid Perplexity: ', report['valid_perplexity'])
print('Test Perplexity: ', report['test_perplexity'])<jupyter_output>('Train Perplexity: ', 31.727393439486)
('Valid Perplexity: ', 44.894794722518)
('Test Perplexity: ', 46.800850181676)<jupyter_text>Loss vs. Epoch<jupyter_code>%matplotlib inline
for k in logs.keys():
    plt.plot(logs[k][0], logs[k][1], label=str(k) + ' (train)')
    plt.plot(logs[k][0], logs[k][2], label=str(k) + ' (valid)')
plt.title('Loss v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Perplexity vs. Epoch<jupyter_code>%matplotlib inline
for k in logs.keys():
    plt.plot(logs[k][0], logs[k][3], label=str(k) + ' (train)')
    plt.plot(logs[k][0], logs[k][4], label=str(k) + ' (valid)')
plt.title('Perplexity v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Euclidean Distances (mean) vs. Epoch<jupyter_code>%matplotlib inline
for k in logs.keys():
    plt.plot(logs[k][0], logs[k][5], label=str(k) + ' (pairwise)')
    plt.plot(logs[k][0], logs[k][9], label=str(k) + ' (against-k)')
plt.title('Euclidean Distance (mean) v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Euclidean Distance (mean)')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Euclidean Distances (std-dev) vs. Epoch<jupyter_code>%matplotlib inline
for k in logs.keys():
    plt.plot(logs[k][0], logs[k][6], label=str(k) + ' (pairwise)')
    plt.plot(logs[k][0], logs[k][10], label=str(k) + ' (against-k)')
plt.title('Euclidean Distance (std-dev) v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Euclidean Distance (std-dev)')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Cosine Distances (mean) vs. Epoch<jupyter_code>%matplotlib inline
for k in logs.keys():
    plt.plot(logs[k][0], logs[k][7], label=str(k) + ' (pairwise)')
    plt.plot(logs[k][0], logs[k][11], label=str(k) + ' (against-k)')
plt.title('Cosine Distance (mean) v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Cosine Distance (mean)')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Cosine Distances (std-dev) vs. Epoch<jupyter_code>%matplotlib inline
for k in logs.keys():
    plt.plot(logs[k][0], logs[k][8], label=str(k) + ' (pairwise)')
    plt.plot(logs[k][0], logs[k][12], label=str(k) + ' (against-k)')
plt.title('Cosine Distance (std-dev) v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Cosine Distance (std-dev)')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Generations<jupyter_code>def print_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    print('Input: '+ enc_input + '\n')
    print('Gend: ' + sample['generated'] + '\n')
    print('True: ' + gold + '\n')
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        print('Closest BLEU Match: ' + cbm + '\n')
        print('Closest BLEU Score: ' + str(best_bleu['best_score']) + '\n')
    print('\n')
    
for i, sample in enumerate(report['train_samples']):
    print_sample(sample, report['best_bleu_matches_train'][i] if 'best_bleu_matches_train' in report else None)
for i, sample in enumerate(report['valid_samples']):
    print_sample(sample, report['best_bleu_matches_valid'][i] if 'best_bleu_matches_valid' in report else None)
for i, sample in enumerate(report['test_samples']):
    print_sample(sample, report['best_bleu_matches_test'][i] if 'best_bleu_matches_test' in report else None)<jupyter_output>Input:  smoked salmon , avocado , dill and parsley mayo sandwich

Gend:  <beg> to make the upper of pork sort and shortening to cool a salmon until well temperature hours completely . warm are except . <step> in

True:  mash the lemon juice and avocado with dill and parsley mayo . spoon over a slice of bread and <end>



Input:  fancy hot dogs

Gend:  <beg> combine first first ingredients except . <step> 2 dry in a paper small sized ( from in the bottomed until smooth are low off halves

True:  <step> 1 melt butter in a large skillet ( cast iron works well for this purpose <end>



Input:  healthy oatmeal cookies

Gend:  <beg> recipe a large pot of boiling water with several adjust . <step> bring to simmer sauce , mix to a boil and tofu aside to

True:  preheat oven to 350 degrees . in a medium bowl , whisk together flours and baking powder ; set aside . <step> in <end>



Input:  mexican hummus

Gend:  <beg> preheat oven to 400 degrees . <step> fry the coriander , olive oil in a small[...]<jupyter_text>BLEU Analysis<jupyter_code>def print_bleu(blue_struct):
    print 'Overall Score: ', blue_struct['score'], '\n'
    print '1-gram Score: ', blue_struct['components']['1']
    print '2-gram Score: ', blue_struct['components']['2']
    print '3-gram Score: ', blue_struct['components']['3']
    print '4-gram Score: ', blue_struct['components']['4']
# Training Set BLEU Scores
print_bleu(report['train_bleu'])
# Validation Set BLEU Scores
print_bleu(report['valid_bleu'])
# Test Set BLEU Scores
print_bleu(report['test_bleu'])
# All Data BLEU Scores
print_bleu(report['combined_bleu'])<jupyter_output>Overall Score:  0 

1-gram Score:  19.5
2-gram Score:  2.9
3-gram Score:  0.4
4-gram Score:  0<jupyter_text>N-pairs BLEU AnalysisThis analysis randomly samples 1000 pairs of generations/ground truths and treats them as translations, giving their BLEU score. We can expect very low scores in the ground truth and high scores can expose hyper-common generations<jupyter_code># Training Set BLEU n-pairs Scores
print_bleu(report['n_pairs_bleu_train'])
# Validation Set n-pairs BLEU Scores
print_bleu(report['n_pairs_bleu_valid'])
# Test Set n-pairs BLEU Scores
print_bleu(report['n_pairs_bleu_test'])
# Combined n-pairs BLEU Scores
print_bleu(report['n_pairs_bleu_all'])
# Ground Truth n-pairs BLEU Scores
print_bleu(report['n_pairs_bleu_gold'])<jupyter_output>Overall Score:  11.34 

1-gram Score:  25.9
2-gram Score:  12.5
3-gram Score:  8
4-gram Score:  6.4<jupyter_text>Alignment AnalysisThis analysis computs the average Smith-Waterman alignment score for generations, with the same intuition as N-pairs BLEU, in that we expect low scores in the ground truth and hyper-common generations to raise the scores<jupyter_code>print 'Average (Train) Generated Score: ', report['average_alignment_train']
print 'Average (Valid) Generated Score: ', report['average_alignment_valid']
print 'Average (Test) Generated Score: ', report['average_alignment_test']
print 'Average (All) Generated Score: ', report['average_alignment_all']
print 'Average Gold Score: ', report['average_alignment_gold']<jupyter_output>Average (Train) Generated Score:  10.8444444444
Average (Valid) Generated Score:  9.97777777778
Average (Test) Generated Score:  11.5777777778
Average (All) Generated Score:  10.8965517241
Average Gold Score:  24.1172413793
{'fraction_non_alphanumeric': 0.15035531729210821, 'fraction_numerical': 0.03129285625233762, 'mean_word_length': 5.752525252525253, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '348346', 'license': 'MIT', 'n_tokens_mistral': 3174, 'n_tokens_neox': 2901, 'n_words': 706, 'path': 'report_notebooks/encdec_noing15_6k_trV_200_512_04drb.ipynb', 'repo_name': 'kingb12/recipeGenerationRNN', 'size': '8021'}