<commit_before># -*- coding: utf-8 -*-


from __future__ import division
import logging
import shutil
import os
import warnings

import numpy as np
import psutil

from commons import empty_clone
import periods
from periods import MONTH, YEAR, ETERNITY
from columns import make_column_from_variable
from indexed_enums import Enum, EnumArray

log = logging.getLogger(__name__)


class DatedHolder(object):
    """
        A wrapper of the value of a variable for a given period (and possibly a given set of extra parameters).
    """
    holder = None
    period = None
    extra_params = None

    def __init__(self, holder, period, value, extra_params = None):
        self.holder = holder
        self.period = period
        self.extra_params = extra_params
        self.value = value

    @property
    def array(self):
        return self.value

    @array.setter
    def array(self, array):
        raise ValueError('Impossible to modify DatedHolder.array. Please use Holder.put_in_cache.')

    @property
    def variable(self):
        return self.holder.variable

    @property
    def entity(self):
        return self.holder.entity

    def to_value_json(self, use_label = False):
        column = make_column_from_variable(self.holder.variable)
        transform_dated_value_to_json = column.transform_dated_value_to_json
        return [
            transform_dated_value_to_json(cell, use_label = use_label)
            for cell in self.array.tolist()
            ]


class Holder(object):
    """
        A holder keeps tracks of a variable values after they have been calculated, or set as an input.
    """
    _array = None  # Only used when variable.definition_period == ETERNITY
    _array_by_period = None  # Only used when variable.definition_period != ETERNITY
    variable = None
    entity = None
    formula = None
    formula_output_period_by_requested_period = None

    def __init__(self, simulation = None, variable = None, entity = None):
        assert variable is not None
        assert self.variable is None
        if simulation is not None:
            warnings.warn(
                u"The Holder(simulation, variable) constructor has been deprecated. "
                u"Please use Holder(entity = entity, variable = variable) instead.",
                Warning
                )
            self.simulation = simulation
            self.entity = simulation.get_entity(variable.entity)
        else:
            self.entity = entity
            self.simulation = entity.simulation
        self.variable = variable
        self.buffer = {}
        self._memory_storage = InMemoryStorage(is_eternal = (self.variable.definition_period == ETERNITY))

        # By default, do not activate on-disk storage, or variable dropping
        self._disk_storage = None
        self._on_disk_storable = False
        self._do_not_store = False
        if self.simulation.memory_config:
            if self.variable.name not in self.simulation.memory_config.priority_variables:
                storage_dir = os.path.join(self.simulation.data_storage_dir, self.variable.name)
                if not os.path.isdir(storage_dir):
                    os.mkdir(storage_dir)
                self._disk_storage = OnDiskStorage(
                    storage_dir, is_eternal = (self.variable.definition_period == ETERNITY))
                self._on_disk_storable = True
            if self.variable.name in self.simulation.memory_config.variables_to_drop:
                self._do_not_store = True

    # Sould probably be deprecated
    @property
    def array(self):
        return self.get_array(self.simulation.period)

    # Sould probably be deprecated
    @array.setter
    def array(self, array):
        self.put_in_cache(array, self.simulation.period)

    def calculate(self, period, **parameters):
        dated_holder = self.compute(period = period, **parameters)
        return dated_holder.array
<commit_msg>Fixup: typo<commit_after># -*- coding: utf-8 -*-


from __future__ import division
import logging
import shutil
import os
import warnings

import numpy as np
import psutil

from commons import empty_clone
import periods
from periods import MONTH, YEAR, ETERNITY
from columns import make_column_from_variable
from indexed_enums import Enum, EnumArray

log = logging.getLogger(__name__)


class DatedHolder(object):
    """
        A wrapper of the value of a variable for a given period (and possibly a given set of extra parameters).
    """
    holder = None
    period = None
    extra_params = None

    def __init__(self, holder, period, value, extra_params = None):
        self.holder = holder
        self.period = period
        self.extra_params = extra_params
        self.value = value

    @property
    def array(self):
        return self.value

    @array.setter
    def array(self, array):
        raise ValueError('Impossible to modify DatedHolder.array. Please use Holder.put_in_cache.')

    @property
    def variable(self):
        return self.holder.variable

    @property
    def entity(self):
        return self.holder.entity

    def to_value_json(self, use_label = False):
        column = make_column_from_variable(self.holder.variable)
        transform_dated_value_to_json = column.transform_dated_value_to_json
        return [
            transform_dated_value_to_json(cell, use_label = use_label)
            for cell in self.array.tolist()
            ]


class Holder(object):
    """
        A holder keeps tracks of a variable values after they have been calculated, or set as an input.
    """
    _array = None  # Only used when variable.definition_period == ETERNITY
    _array_by_period = None  # Only used when variable.definition_period != ETERNITY
    variable = None
    entity = None
    formula = None
    formula_output_period_by_requested_period = None

    def __init__(self, simulation = None, variable = None, entity = None):
        assert variable is not None
        assert self.variable is None
        if simulation is not None:
            warnings.warn(
                u"The Holder(simulation, variable) constructor has been deprecated. "
                u"Please use Holder(entity = entity, variable = variable) instead.",
                Warning
                )
            self.simulation = simulation
            self.entity = simulation.get_entity(variable.entity)
        else:
            self.entity = entity
            self.simulation = entity.simulation
        self.variable = variable
        self.buffer = {}
        self._memory_storage = InMemoryStorage(is_eternal = (self.variable.definition_period == ETERNITY))

        # By default, do not activate on-disk storage, or variable dropping
        self._disk_storage = None
        self._on_disk_storable = False
        self._do_not_store = False
        if self.simulation.memory_config:
            if self.variable.name not in self.simulation.memory_config.priority_variables:
                storage_dir = os.path.join(self.simulation.data_storage_dir, self.variable.name)
                if not os.path.isdir(storage_dir):
                    os.mkdir(storage_dir)
                self._disk_storage = OnDiskStorage(
                    storage_dir, is_eternal = (self.variable.definition_period == ETERNITY))
                self._on_disk_storable = True
            if self.variable.name in self.simulation.memory_config.variables_to_drop:
                self._do_not_store = True

    # Should probably be deprecated
    @property
    def array(self):
        return self.get_array(self.simulation.period)

    # Should probably be deprecated
    @array.setter
    def array(self, array):
        self.put_in_cache(array, self.simulation.period)

    def calculate(self, period, **parameters):
        dated_holder = self.compute(period = period, **parameters)
        return dated_holder.array

{'fraction_non_alphanumeric': 0.07280813214739518, 'fraction_numerical': 0.00025412960609911054, 'mean_word_length': 2.488918439716312, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1847041', 'n_tokens_mistral': 2151, 'n_tokens_neox': 2080, 'n_words': 599}