<commit_before># pylint: disable=invalid-name,protected-access
from copy import deepcopy
from unittest import TestCase
import codecs
import gzip
import logging
import os
import shutil

import numpy
from numpy.testing import assert_allclose

from deep_qa.common.checks import log_keras_version_info
from deep_qa.models.memory_networks.memory_network import MemoryNetwork
from deep_qa.models.multiple_choice_qa.multiple_true_false_similarity import MultipleTrueFalseSimilarity


class DeepQaTestCase(TestCase):  # pylint: disable=too-many-public-methods
    TEST_DIR = './TMP_TEST/'
    TRAIN_FILE = TEST_DIR + 'train_file'
    VALIDATION_FILE = TEST_DIR + 'validation_file'
    TRAIN_BACKGROUND = TEST_DIR + 'train_background'
    VALIDATION_BACKGROUND = TEST_DIR + 'validation_background'
    SNLI_FILE = TEST_DIR + 'snli_file'
    PRETRAINED_VECTORS_FILE = TEST_DIR + 'pretrained_glove_vectors_file'
    PRETRAINED_VECTORS_GZIP = TEST_DIR + 'pretrained_glove_vectors_file.gz'

    def setUp(self):
        logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
                            level=logging.INFO)
        log_keras_version_info()
        os.makedirs(self.TEST_DIR, exist_ok=True)

    def tearDown(self):
        shutil.rmtree(self.TEST_DIR)

    def get_model_params(self, model_class, additional_arguments=None):
        params = {}
        params['save_models'] = False
        params['model_serialization_prefix'] = self.TEST_DIR
        params['train_files'] = [self.TRAIN_FILE]
        params['validation_files'] = [self.VALIDATION_FILE]
        params['embedding_dim'] = {'words': 6, 'characters': 2}
        params['encoder'] = {"default": {'type': 'bow'}}
        params['num_epochs'] = 1
        params['keras_validation_split'] = 0.0
        if self.is_model_with_background(model_class):
            params['train_files'].append(self.TRAIN_BACKGROUND)
            params['validation_files'].append(self.VALIDATION_BACKGROUND)
        if self.is_memory_network(model_class):
            params['knowledge_selector'] = {'type': 'dot_product'}
            params['memory_updater'] = {'type': 'sum'}
            params['entailment_input_combiner'] = {'type': 'memory_only'}
        if additional_arguments:
            for key, value in additional_arguments.items():
                params[key] = deepcopy(value)
        return params

    def get_model(self, model_class, additional_arguments=None):
        params = self.get_model_params(model_class, additional_arguments)<commit_msg>Clear tensorflow session between tests to reduce memory consumption (#275)<commit_after># pylint: disable=invalid-name,protected-access
from copy import deepcopy
from unittest import TestCase
import codecs
import gzip
import logging
import os
import shutil

from keras import backend as K
import numpy
from numpy.testing import assert_allclose

from deep_qa.common.checks import log_keras_version_info
from deep_qa.models.memory_networks.memory_network import MemoryNetwork
from deep_qa.models.multiple_choice_qa.multiple_true_false_similarity import MultipleTrueFalseSimilarity


class DeepQaTestCase(TestCase):  # pylint: disable=too-many-public-methods
    TEST_DIR = './TMP_TEST/'
    TRAIN_FILE = TEST_DIR + 'train_file'
    VALIDATION_FILE = TEST_DIR + 'validation_file'
    TRAIN_BACKGROUND = TEST_DIR + 'train_background'
    VALIDATION_BACKGROUND = TEST_DIR + 'validation_background'
    SNLI_FILE = TEST_DIR + 'snli_file'
    PRETRAINED_VECTORS_FILE = TEST_DIR + 'pretrained_glove_vectors_file'
    PRETRAINED_VECTORS_GZIP = TEST_DIR + 'pretrained_glove_vectors_file.gz'

    def setUp(self):
        logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
                            level=logging.INFO)
        log_keras_version_info()
        os.makedirs(self.TEST_DIR, exist_ok=True)

    def tearDown(self):
        if K.backend() == 'tensorflow':
            K.clear_session()
        shutil.rmtree(self.TEST_DIR)

    def get_model_params(self, model_class, additional_arguments=None):
        params = {}
        params['save_models'] = False
        params['model_serialization_prefix'] = self.TEST_DIR
        params['train_files'] = [self.TRAIN_FILE]
        params['validation_files'] = [self.VALIDATION_FILE]
        params['embedding_dim'] = {'words': 6, 'characters': 2}
        params['encoder'] = {"default": {'type': 'bow'}}
        params['num_epochs'] = 1
        params['keras_validation_split'] = 0.0
        if self.is_model_with_background(model_class):
            params['train_files'].append(self.TRAIN_BACKGROUND)
            params['validation_files'].append(self.VALIDATION_BACKGROUND)
        if self.is_memory_network(model_class):
            params['knowledge_selector'] = {'type': 'dot_product'}
            params['memory_updater'] = {'type': 'sum'}
            params['entailment_input_combiner'] = {'type': 'memory_only'}
        if additional_arguments:
            for key, value in additional_arguments.items():
                params[key] = deepcopy(value)
        return params

    def get_model(self, model_class, additional_arguments=None):
        params = self.get_model_params(model_class, additional_arguments)
{'fraction_non_alphanumeric': 0.10359327217125382, 'fraction_numerical': 0.002484709480122324, 'mean_word_length': 3.7702825888787603, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5687228', 'n_tokens_mistral': 1632, 'n_tokens_neox': 1572, 'n_words': 294}