<commit_before>import glob
import os
import random
import sys
import gzip
import pickle

import argparse
import numpy as np

from config import BabiConfig, BabiConfigJoint
from train_test import train, train_linear_start, test
from util import parse_babi_task, build_model

import fasttext

seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)  # for reproducing


def run_tableQA(data_path, model_file):
    """
    Train and test for table QA
    """

    # Parse data
    train_files = glob.glob(data_path.format('train'))
    test_files  = glob.glob(data_path.format('test'))
    # SV: init dict with pre-trained vectors, e.g. from fastText
    # dictionary = fasttext.load_model('./embeddings/fil9.bin')
    dictionary = fasttext.load_model('./embeddings/enwik9')
    # dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(train_files, dictionary, False)
    test_story, test_questions, test_qstory    = parse_babi_task(test_files, dictionary, False)
    # print train_story
    general_config = BabiConfig(train_story, train_questions, dictionary)

    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)<commit_msg>chg: add loggers for loading fastText model<commit_after>import glob
import os
import random
import sys
import gzip
import pickle

import argparse
import numpy as np

from config import BabiConfig, BabiConfigJoint
from train_test import train, train_linear_start, test
from util import parse_babi_task, build_model

import fasttext

seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)  # for reproducing

EMBEDDINGS_MODEL = 'enwik9'


def run_tableQA(data_path, model_file):
    """
    Train and test for table QA
    """

    # Parse data
    train_files = glob.glob(data_path.format('train'))
    test_files  = glob.glob(data_path.format('test'))
    # SV: init dict with pre-trained vectors, e.g. from fastText
    # dictionary = fasttext.load_model('./embeddings/fil9.bin')
    print "Loading", EMBEDDINGS_MODEL
    dictionary = fasttext.load_model('./embeddings/'+EMBEDDINGS_MODEL)
    print "Finished loading"
    # dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(train_files, dictionary, False)
    test_story, test_questions, test_qstory    = parse_babi_task(test_files, dictionary, False)
    # print train_story
    general_config = BabiConfig(train_story, train_questions, dictionary)

    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)
{'fraction_non_alphanumeric': 0.08566243194192377, 'fraction_numerical': 0.003629764065335753, 'mean_word_length': 4.753653444676409, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '463015', 'n_tokens_mistral': 886, 'n_tokens_neox': 857, 'n_words': 221}