<commit_before>from mc_memn2n import MemN2N
import tensorflow as tf
import numpy as np
import os

import pickle
"""
Stores the configuration for loading the model
max_memory_size is the maximum size allowed during training
memory_size is the actual memory size used based on max story size
"""

with open('../save/vocab_data.pickle', 'rb') as handle:
  vocab_data = pickle.load(handle)


config = {
    'batch': 64,
    'vocab_size': len(vocab_data['vocab']),
    'sentence_size': vocab_data['sentence_size'],
    'max_memory_size': vocab_data['memory_size'],
    'memory_size': vocab_data['memory_size'],
    'embedding_size': 50,
    'hops': 3,
    'max_grad_norm': 40.0,
    'regularization': 0.1,
    'epsilon': 1e-8,
    'lr': 0.001
}

restore_location = '../save/checkpoint'
print(restore_location)

sess = tf.Session()

model = MemN2N(batch_size=config["batch"],
               vocab_size=config["vocab_size"],
               sentence_size=config["sentence_size"],
               memory_size=config["memory_size"],
               embedding_size=config["embedding_size"],
               answer_size=vocab_data['answer_size'],
               label_size=4,
               glove_embedding=np.random.standard_normal([config['vocab_size'], config['embedding_size']]),
               session=sess,
               hops=config["hops"],
               max_grad_norm=config["max_grad_norm"],
               l2=config["regularization"],
               lr=config["lr"],
               epsilon=config["epsilon"],
               nonlin=tf.nn.relu,
               restoreLoc=restore_location)

print('finished load model')

# Uncomment to see if the weights were loaded correctly
# print(sess.run(model.A))

def get_pred(testS, testQ, testAS):
    ps = model.predict_proba(testS, testQ, testAS)
    op = model.predict_test(testS, testQ, testAS)

    answer = op[0][0]<commit_msg>model_helper<commit_after>from mc_memn2n import MemN2N
import tensorflow as tf
import numpy as np
import os

import pickle
"""
Stores the configuration for loading the model
max_memory_size is the maximum size allowed during training
memory_size is the actual memory size used based on max story size
"""

with open('../save/vocab_data.pickle', 'rb') as handle:
  vocab_data = pickle.load(handle)


config = {
    'batch': 64,
    'vocab_size': len(vocab_data['vocab']),
    'sentence_size': vocab_data['sentence_size'],
    'max_memory_size': vocab_data['memory_size'],
    'memory_size': vocab_data['memory_size'],
    'embedding_size': 50,
    'hops': 3,
    'max_grad_norm': 40.0,
    'regularization': 0.1,
    'epsilon': 1e-8,
    'lr': 0.001
}

restore_location = '../save/lr=0.01_eps=1e-08_mgn=40.0_hp=3_es=50_ms=100_reg=0.1'
print(restore_location)

sess = tf.Session()

model = MemN2N(batch_size=config["batch"],
               vocab_size=config["vocab_size"],
               sentence_size=config["sentence_size"],
               memory_size=config["memory_size"],
               embedding_size=config["embedding_size"],
               answer_size=vocab_data['answer_size'],
               label_size=4,
               glove_embedding=np.random.standard_normal([config['vocab_size'], config['embedding_size']]),
               session=sess,
               hops=config["hops"],
               max_grad_norm=config["max_grad_norm"],
               l2=config["regularization"],
               lr=config["lr"],
               epsilon=config["epsilon"],
               nonlin=tf.nn.relu,
               restoreLoc=restore_location)

print('finished load model')

# Uncomment to see if the weights were loaded correctly
# print(sess.run(model.A))

def get_pred(testS, testQ, testAS):
    ps = model.predict_proba(testS, testQ, testAS)
    op = model.predict_test(testS, testQ, testAS)

    answer = op[0][0]
{'fraction_non_alphanumeric': 0.12774398307326104, 'fraction_numerical': 0.0166622586617297, 'mean_word_length': 3.1514818880351263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6419191', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1232, 'n_words': 245}