<commit_before>from candidate_generation.candidate_generator_cache import CandidateGeneratorCache
from candidate_generation.neighborhood_candidate_generator import NeighborhoodCandidateGenerator
from candidate_selection.baselines.oracle_candidate import OracleCandidate
from candidate_selection.models.dumb_entity_embedding_vs_bag_of_words import DumbEntityEmbeddingVsBagOfWords
from candidate_selection.models.dumb_entity_embedding_vs_lstm import DumbEntityEmbeddingVsLstm
from candidate_selection.tensorflow_candidate_selector import TensorflowCandidateSelector
from database_interface.data_interface.CsvInterface import CsvInterface
from database_interface.data_interface.FreebaseInterface import FreebaseInterface
from database_interface.entity_cache_interface import EntityCacheInterface
from database_interface.expansion_strategies.all_through_expansion_strategy import AllThroughExpansionStrategy
from database_interface.expansion_strategies.only_freebase_element_strategy import OnlyFreebaseExpansionStrategy
from database_interface.hypergraph_interface import HypergraphInterface
from facts.database_facts.csv_facts import CsvFacts
from facts.database_facts.freebase_facts import FreebaseFacts
from helpers.read_conll_files import ConllReader
from model_construction.settings_reader import SettingsReader
import itertools

class ModelBuilder:

    settings_reader = None
    settings = None

    def __init__(self):
        self.settings_reader = SettingsReader()

    def build(self, settings, version=None):
        self.settings = settings
        return self.create_model()

    def create_model(self):
        model_class = self.retrieve_class_name(self.settings["algorithm"]["model"]["stack_name"])
        model = model_class()
        model = self.wrap_model(model)
        for k, v in self.settings["algorithm"]["model"].items():
            if k == "stack_name":
                continue
            else:
                model.update_setting(k, v)
        return model

    def wrap_model(self, model):
        if "prefix" in self.settings["dataset"]["database"]:
            prefix = self.settings["dataset"]["database"]["prefix"]
        else:
            prefix = ""

        if self.settings["dataset"]["database"]["endpoint"] == "sparql":
            database_interface = FreebaseInterface()
            expansion_strategy = OnlyFreebaseExpansionStrategy()
            facts = FreebaseFacts()
        elif self.settings["dataset"]["database"]["endpoint"] == "csv":
            database_interface = CsvInterface(self.settings["dataset"]["database"]["file"])
            expansion_strategy = AllThroughExpansionStrategy()
            facts = CsvFacts(self.settings["dataset"]["database"]["file"])
            prefix = ""

        database = HypergraphInterface(database_interface, expansion_strategy, prefix=prefix)
        database = EntityCacheInterface(database)
        disk_cache = self.settings["dataset"]["database"]["disk_cache"] if "disk_cache" in self.settings["dataset"]["database"] else None
        candidate_generator = NeighborhoodCandidateGenerator(database, neighborhood_search_scope=1,
                                                             extra_literals=True)
        candidate_generator = CandidateGeneratorCache(candidate_generator,
                                                      disk_cache=disk_cache)<commit_msg>disabled inner cache is memory runs out<commit_after>from candidate_generation.candidate_generator_cache import CandidateGeneratorCache
from candidate_generation.neighborhood_candidate_generator import NeighborhoodCandidateGenerator
from candidate_selection.baselines.oracle_candidate import OracleCandidate
from candidate_selection.models.dumb_entity_embedding_vs_bag_of_words import DumbEntityEmbeddingVsBagOfWords
from candidate_selection.models.dumb_entity_embedding_vs_lstm import DumbEntityEmbeddingVsLstm
from candidate_selection.tensorflow_candidate_selector import TensorflowCandidateSelector
from database_interface.data_interface.CsvInterface import CsvInterface
from database_interface.data_interface.FreebaseInterface import FreebaseInterface
from database_interface.entity_cache_interface import EntityCacheInterface
from database_interface.expansion_strategies.all_through_expansion_strategy import AllThroughExpansionStrategy
from database_interface.expansion_strategies.only_freebase_element_strategy import OnlyFreebaseExpansionStrategy
from database_interface.hypergraph_interface import HypergraphInterface
from facts.database_facts.csv_facts import CsvFacts
from facts.database_facts.freebase_facts import FreebaseFacts
from helpers.read_conll_files import ConllReader
from model_construction.settings_reader import SettingsReader
import itertools

class ModelBuilder:

    settings_reader = None
    settings = None

    def __init__(self):
        self.settings_reader = SettingsReader()

    def build(self, settings, version=None):
        self.settings = settings
        return self.create_model()

    def create_model(self):
        model_class = self.retrieve_class_name(self.settings["algorithm"]["model"]["stack_name"])
        model = model_class()
        model = self.wrap_model(model)
        for k, v in self.settings["algorithm"]["model"].items():
            if k == "stack_name":
                continue
            else:
                model.update_setting(k, v)
        return model

    def wrap_model(self, model):
        if "prefix" in self.settings["dataset"]["database"]:
            prefix = self.settings["dataset"]["database"]["prefix"]
        else:
            prefix = ""

        if self.settings["dataset"]["database"]["endpoint"] == "sparql":
            database_interface = FreebaseInterface()
            expansion_strategy = OnlyFreebaseExpansionStrategy()
            facts = FreebaseFacts()
        elif self.settings["dataset"]["database"]["endpoint"] == "csv":
            database_interface = CsvInterface(self.settings["dataset"]["database"]["file"])
            expansion_strategy = AllThroughExpansionStrategy()
            facts = CsvFacts(self.settings["dataset"]["database"]["file"])
            prefix = ""

        database = HypergraphInterface(database_interface, expansion_strategy, prefix=prefix)
        #database = EntityCacheInterface(database)
        disk_cache = self.settings["dataset"]["database"]["disk_cache"] if "disk_cache" in self.settings["dataset"]["database"] else None
        candidate_generator = NeighborhoodCandidateGenerator(database, neighborhood_search_scope=1,
                                                             extra_literals=True)
        candidate_generator = CandidateGeneratorCache(candidate_generator,
                                                      disk_cache=disk_cache)
{'fraction_non_alphanumeric': 0.07783193029094669, 'fraction_numerical': 0.0002953773445576724, 'mean_word_length': 4.020014825796887, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2252664', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1666, 'n_words': 319}