<commit_before>import copy
import glob
import importlib
import inspect
import logging
import os
import pkg_resources
import traceback
import typing
from imp import find_module, load_module

from openfisca_core import commons, periods, variables
from openfisca_core.entities import Entity
from openfisca_core.errors import VariableNameConflictError, VariableNotFoundError
from openfisca_core.parameters import ParameterNode
from openfisca_core.periods import Instant, Period
from openfisca_core.populations import Population, GroupPopulation
from openfisca_core.simulations import SimulationBuilder
from openfisca_core.variables import Variable

log = logging.getLogger(__name__)


class TaxBenefitSystem:
    """
    Represents the legislation.

    It stores parameters (values defined for everyone) and variables (values defined for some given entity e.g. a person).

    :param entities: Entities used by the tax benefit system.
    :param string parameters: Directory containing the YAML parameter files.


    .. attribute:: parameters

       :obj:`.ParameterNode` containing the legislation parameters
    """
    _base_tax_benefit_system = None
    _parameters_at_instant_cache = None
    person_key_plural = None
    preprocess_parameters = None
    baseline = None  # Baseline tax-benefit system. Used only by reforms. Note: Reforms can be chained.
    cache_blacklist = None
    decomposition_file_path = None

    def __init__(self, entities):
        # TODO: Currently: Don't use a weakref, because they are cleared by Paste (at least) at each call.
        self.parameters = None
        self._parameters_at_instant_cache = {}  # weakref.WeakValueDictionary()
        self.variables = {}
        self.open_api_config = {}
        # Tax benefit systems are mutable, so entities (which need to know about our variables) can't be shared among them
        if entities is None or len(entities) == 0:
            raise Exception("A tax and benefit sytem must have at least an entity.")
        self.entities = [copy.copy(entity) for entity in entities]
        self.person_entity = [entity for entity in self.entities if entity.is_person][0]
        self.group_entities = [entity for entity in self.entities if not entity.is_person]
        for entity in self.entities:
            entity.set_tax_benefit_system(self)

    @property
    def base_tax_benefit_system(self):
        base_tax_benefit_system = self._base_tax_benefit_system
        if base_tax_benefit_system is None:
            baseline = self.baseline
            if baseline is None:
                return self
            self._base_tax_benefit_system = base_tax_benefit_system = baseline.base_tax_benefit_system
        return base_tax_benefit_system

    def instantiate_entities(self):
        person = self.person_entity
        members = Population(person)
        entities: typing.Dict[Entity.key, Entity] = {person.key: members}

        for entity in self.group_entities:
            entities[entity.key] = GroupPopulation(entity, members)

        return entities

    # Deprecated method of constructing simulations, to be phased out in favor of SimulationBuilder
    def new_scenario(self):
        class ScenarioAdapter(object):
            def __init__(self, tax_benefit_system):
                self.tax_benefit_system = tax_benefit_system

            def init_from_attributes(self, **attributes):
                self.attributes = attributes
                return self

            def init_from_dict(self, dict):
                self.attributes = None
                self.dict = dict
                self.period = dict.pop('period')
                return self

            def new_simulation(self, debug = False, opt_out_cache = False, use_baseline = False, trace = False):
                # Legacy from scenarios, used in reforms
                tax_benefit_system = self.tax_benefit_system
                if use_baseline:
                    while True:
                        baseline = tax_benefit_system.baseline
                        if baseline is None:
                            break
                        tax_benefit_system = baseline

                builder = SimulationBuilder()
                if self.attributes:
                    variables = self.attributes.get('input_variables') or {}
                    period = self.attributes.get('period')
                    builder.set_default_period(period)
                    simulation = builder.build_from_variables(tax_benefit_system, variables)
                else:
                    builder.set_default_period(self.period)
                    simulation = builder.build_from_entities(tax_benefit_system, self.dict)

                simulation.trace = trace
                simulation.debug = debug
                simulation.opt_out_cache = opt_out_cache

                return simulation

        return ScenarioAdapter(self)

    def prefill_cache(self):
        pass

    def load_variable(self, variable_class, update = False):
        name = variable_class.__name__

        # Check if a Variable with the same name is already registered.
        baseline_variable = self.get_variable(name)
        if baseline_variable and not update:
            raise VariableNameConflictError(
                'Variable "{}" is already defined. Use `update_variable` to replace it.'.format(name))

        variable = variable_class(baseline_variable = baseline_variable)
        self.variables[variable.name] = variable

        return variable

    def add_variable(self, variable):
        """
        Adds an OpenFisca variable to the tax and benefit system.

        :param .Variable variable: The variable to add. Must be a subclass of Variable.

        :raises: :exc:`.VariableNameConflictError` if a variable with the same name have previously been added to the tax and benefit system.
        """
        return self.load_variable(variable, update = False)

    def replace_variable(self, variable):
        """
        Replaces an existing OpenFisca variable in the tax and benefit system by a new one.

        The new variable must have the same name than the replaced one.

        If no variable with the given name exists in the tax and benefit system, no error will be raised and the new variable will be simply added.

        :param Variable variable: New variable to add. Must be a subclass of Variable.
        """
        name = variable.__name__
        if self.variables.get(name) is not None:
            del self.variables[name]
        self.load_variable(variable, update = False)

    def update_variable(self, variable):
        """
        Updates an existing OpenFisca variable in the tax and benefit system.

        All attributes of the updated variable that are not explicitely overridden by the new ``variable`` will stay unchanged.

        The new variable must have the same name than the updated one.

        If no variable with the given name exists in the tax and benefit system, no error will be raised and the new variable will be simply added.

        :param Variable variable: Variable to add. Must be a subclass of Variable.
        """
        return self.load_variable(variable, update = True)

    def add_variables_from_file(self, file_path):
        """
        Adds all OpenFisca variables contained in a given file to the tax and benefit system.
        """
        try:
            file_name = os.path.splitext(os.path.basename(file_path))[0]

            #  As Python remembers loaded modules by name, in order to prevent collisions, we need to make sure that:
            #  - Files with the same name, but located in different directories, have a different module names. Hence the file path hash in the module name.
            #  - The same file, loaded by different tax and benefit systems, has distinct module names. Hence the `id(self)` in the module name.
            module_name = '{}_{}_{}'.format(id(self), hash(os.path.abspath(file_path)), file_name)

            module_directory = os.path.dirname(file_path)
            try:
                module = load_module(module_name, *find_module(file_name, [module_directory]))
            except NameError as e:
                logging.error(str(e) + ": if this code used to work, this error might be due to a major change in OpenFisca-Core. Checkout the changelog to learn more: <https://github.com/openfisca/openfisca-core/blob/master/CHANGELOG.md>")
                raise
            potential_variables = [getattr(module, item) for item in dir(module) if not item.startswith('__')]
            for pot_variable in potential_variables:
                # We only want to get the module classes defined in this module (not imported)
                if inspect.isclass(pot_variable) and issubclass(pot_variable, Variable) and pot_variable.__module__ == module_name:
                    self.add_variable(pot_variable)
        except Exception:
            log.error('Unable to load OpenFisca variables from file "{}"'.format(file_path))
            raise

    def add_variables_from_directory(self, directory):
        """
        Recursively explores a directory, and adds all OpenFisca variables found there to the tax and benefit system.
        """
        py_files = glob.glob(os.path.join(directory, "*.py"))
        for py_file in py_files:
            self.add_variables_from_file(py_file)
        subdirectories = glob.glob(os.path.join(directory, "*/"))
        for subdirectory in subdirectories:
            self.add_variables_from_directory(subdirectory)

    def add_variables(self, *variables):
        """
        Adds a list of OpenFisca Variables to the `TaxBenefitSystem`.

        See also :any:`add_variable`
        """
        for variable in variables:<commit_msg>Apply @maukoquiroga's suggested fix to #1053<commit_after>import copy
import glob
import importlib
import inspect
import logging
import os
import pkg_resources
import sys
import traceback
import typing

from openfisca_core import commons, periods, variables
from openfisca_core.entities import Entity
from openfisca_core.errors import VariableNameConflictError, VariableNotFoundError
from openfisca_core.parameters import ParameterNode
from openfisca_core.periods import Instant, Period
from openfisca_core.populations import Population, GroupPopulation
from openfisca_core.simulations import SimulationBuilder
from openfisca_core.variables import Variable

log = logging.getLogger(__name__)


class TaxBenefitSystem:
    """
    Represents the legislation.

    It stores parameters (values defined for everyone) and variables (values defined for some given entity e.g. a person).

    :param entities: Entities used by the tax benefit system.
    :param string parameters: Directory containing the YAML parameter files.


    .. attribute:: parameters

       :obj:`.ParameterNode` containing the legislation parameters
    """
    _base_tax_benefit_system = None
    _parameters_at_instant_cache = None
    person_key_plural = None
    preprocess_parameters = None
    baseline = None  # Baseline tax-benefit system. Used only by reforms. Note: Reforms can be chained.
    cache_blacklist = None
    decomposition_file_path = None

    def __init__(self, entities):
        # TODO: Currently: Don't use a weakref, because they are cleared by Paste (at least) at each call.
        self.parameters = None
        self._parameters_at_instant_cache = {}  # weakref.WeakValueDictionary()
        self.variables = {}
        self.open_api_config = {}
        # Tax benefit systems are mutable, so entities (which need to know about our variables) can't be shared among them
        if entities is None or len(entities) == 0:
            raise Exception("A tax and benefit sytem must have at least an entity.")
        self.entities = [copy.copy(entity) for entity in entities]
        self.person_entity = [entity for entity in self.entities if entity.is_person][0]
        self.group_entities = [entity for entity in self.entities if not entity.is_person]
        for entity in self.entities:
            entity.set_tax_benefit_system(self)

    @property
    def base_tax_benefit_system(self):
        base_tax_benefit_system = self._base_tax_benefit_system
        if base_tax_benefit_system is None:
            baseline = self.baseline
            if baseline is None:
                return self
            self._base_tax_benefit_system = base_tax_benefit_system = baseline.base_tax_benefit_system
        return base_tax_benefit_system

    def instantiate_entities(self):
        person = self.person_entity
        members = Population(person)
        entities: typing.Dict[Entity.key, Entity] = {person.key: members}

        for entity in self.group_entities:
            entities[entity.key] = GroupPopulation(entity, members)

        return entities

    # Deprecated method of constructing simulations, to be phased out in favor of SimulationBuilder
    def new_scenario(self):
        class ScenarioAdapter(object):
            def __init__(self, tax_benefit_system):
                self.tax_benefit_system = tax_benefit_system

            def init_from_attributes(self, **attributes):
                self.attributes = attributes
                return self

            def init_from_dict(self, dict):
                self.attributes = None
                self.dict = dict
                self.period = dict.pop('period')
                return self

            def new_simulation(self, debug = False, opt_out_cache = False, use_baseline = False, trace = False):
                # Legacy from scenarios, used in reforms
                tax_benefit_system = self.tax_benefit_system
                if use_baseline:
                    while True:
                        baseline = tax_benefit_system.baseline
                        if baseline is None:
                            break
                        tax_benefit_system = baseline

                builder = SimulationBuilder()
                if self.attributes:
                    variables = self.attributes.get('input_variables') or {}
                    period = self.attributes.get('period')
                    builder.set_default_period(period)
                    simulation = builder.build_from_variables(tax_benefit_system, variables)
                else:
                    builder.set_default_period(self.period)
                    simulation = builder.build_from_entities(tax_benefit_system, self.dict)

                simulation.trace = trace
                simulation.debug = debug
                simulation.opt_out_cache = opt_out_cache

                return simulation

        return ScenarioAdapter(self)

    def prefill_cache(self):
        pass

    def load_variable(self, variable_class, update = False):
        name = variable_class.__name__

        # Check if a Variable with the same name is already registered.
        baseline_variable = self.get_variable(name)
        if baseline_variable and not update:
            raise VariableNameConflictError(
                'Variable "{}" is already defined. Use `update_variable` to replace it.'.format(name))

        variable = variable_class(baseline_variable = baseline_variable)
        self.variables[variable.name] = variable

        return variable

    def add_variable(self, variable):
        """
        Adds an OpenFisca variable to the tax and benefit system.

        :param .Variable variable: The variable to add. Must be a subclass of Variable.

        :raises: :exc:`.VariableNameConflictError` if a variable with the same name have previously been added to the tax and benefit system.
        """
        return self.load_variable(variable, update = False)

    def replace_variable(self, variable):
        """
        Replaces an existing OpenFisca variable in the tax and benefit system by a new one.

        The new variable must have the same name than the replaced one.

        If no variable with the given name exists in the tax and benefit system, no error will be raised and the new variable will be simply added.

        :param Variable variable: New variable to add. Must be a subclass of Variable.
        """
        name = variable.__name__
        if self.variables.get(name) is not None:
            del self.variables[name]
        self.load_variable(variable, update = False)

    def update_variable(self, variable):
        """
        Updates an existing OpenFisca variable in the tax and benefit system.

        All attributes of the updated variable that are not explicitely overridden by the new ``variable`` will stay unchanged.

        The new variable must have the same name than the updated one.

        If no variable with the given name exists in the tax and benefit system, no error will be raised and the new variable will be simply added.

        :param Variable variable: Variable to add. Must be a subclass of Variable.
        """
        return self.load_variable(variable, update = True)

    def add_variables_from_file(self, file_path):
        """
        Adds all OpenFisca variables contained in a given file to the tax and benefit system.
        """
        try:
            file_name = os.path.splitext(os.path.basename(file_path))[0]

            #  As Python remembers loaded modules by name, in order to prevent collisions, we need to make sure that:
            #  - Files with the same name, but located in different directories, have a different module names. Hence the file path hash in the module name.
            #  - The same file, loaded by different tax and benefit systems, has distinct module names. Hence the `id(self)` in the module name.
            module_name = f"{id(self)}_{hash(os.path.abspath(file_path))}_{file_name}"

            try:
                spec = importlib.util.spec_from_file_location(module_name, file_path)
                module = importlib.util.module_from_spec(spec)
                sys.modules[module_name] = module
                spec.loader.exec_module(module)
            except NameError as e:
                logging.error(str(e) + ": if this code used to work, this error might be due to a major change in OpenFisca-Core. Checkout the changelog to learn more: <https://github.com/openfisca/openfisca-core/blob/master/CHANGELOG.md>")
                raise
            potential_variables = [getattr(module, item) for item in dir(module) if not item.startswith('__')]
            for pot_variable in potential_variables:
                # We only want to get the module classes defined in this module (not imported)
                if inspect.isclass(pot_variable) and issubclass(pot_variable, Variable) and pot_variable.__module__ == module_name:
                    self.add_variable(pot_variable)
        except Exception:
            log.error('Unable to load OpenFisca variables from file "{}"'.format(file_path))
            raise

    def add_variables_from_directory(self, directory):
        """
        Recursively explores a directory, and adds all OpenFisca variables found there to the tax and benefit system.
        """
        py_files = glob.glob(os.path.join(directory, "*.py"))
        for py_file in py_files:
            self.add_variables_from_file(py_file)
        subdirectories = glob.glob(os.path.join(directory, "*/"))
        for subdirectory in subdirectories:
            self.add_variables_from_directory(subdirectory)

    def add_variables(self, *variables):
        """
        Adds a list of OpenFisca Variables to the `TaxBenefitSystem`.

        See also :any:`add_variable`
        """
        for variable in variables:
{'fraction_non_alphanumeric': 0.06426132599500586, 'fraction_numerical': 0.0005096060745044081, 'mean_word_length': 2.4980392156862745, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5624655', 'n_tokens_mistral': 5154, 'n_tokens_neox': 4884, 'n_words': 1721}