<commit_before># -*- coding: utf-8 -*-
"""
This is the Simulation module. The Simulation layer takes of creating output
variables, writing data to disk, iterating over data and calculations at
each interval in the simulation and setting any parameters required to perform
the simulation. It gets all its info from the model, which in turn gets it from
each layer which gets info from the layers' sources.
"""

from carousel.core import logging, CommonBase, Registry, UREG
from carousel.core.exceptions import CircularDependencyError
import json
import os
import sys
import numpy as np
import Queue
import functools

LOGGER = logging.getLogger(__name__)


def topological_sort(dag):
    """
    topological sort

    :param dag: directed acyclic graph
    :type dag: dict

    .. seealso:: `Topographical Sorting
        <http://en.wikipedia.org/wiki/Topological_sorting>`_,
        `Directed Acyclic Graph (DAG)
        <https://en.wikipedia.org/wiki/Directed_acyclic_graph>`_
    """
    # find all edges of dag
    topsort = [node for node, edge in dag.iteritems() if not edge]
    # loop through nodes until topologically sorted
    while len(topsort) < len(dag):
        num_nodes = len(topsort)  # number of nodes
        # unsorted nodes
        for node in dag.viewkeys() - set(topsort):
            # nodes with no incoming edges
            if set(dag[node]) <= set(topsort):
                topsort.append(node)
                break
        # circular dependencies
        if len(topsort) == num_nodes:
            raise CircularDependencyError(dag.viewkeys() - set(topsort))
    return topsort


class SimRegistry(Registry):
    #: meta names
    _meta_names = ['commands']

    def __init__(self):
        super(SimRegistry, self).__init__()
        #: simulation commands
        self.commands = {}

    def register(self, sim, *args, **kwargs):
        kwargs.update(zip(self._meta_names, args))
        # call super method, now meta can be passed as args or kwargs.
        super(SimRegistry, self).register(sim, **kwargs)


class SimBase(CommonBase):
    """
    Meta class for simulations.
    """
    _path_attr = 'sim_path'
    _file_attr = 'sim_file'

    def __new__(mcs, name, bases, attr):
        # use only with Simulation subclasses
        if not CommonBase.get_parents(bases, SimBase):
            return super(SimBase, mcs).__new__(mcs, name, bases, attr)
        # set param file full path if simulations path and file specified or
        # try to set parameters from class attributes except private/magic
        attr = mcs.set_param_file_or_parameters(attr)
        return super(SimBase, mcs).__new__(mcs, name, bases, attr)


class Simulation(object):
    """
    A class for simulations.

    :param simfile: Filename of simulation configuration file.
    :type simfile: str
    """
    __metaclass__ = SimBase

    def __init__(self, *args, **kwargs):
        # save arguments, might need them later
        self.args = args  #: positional arguments
        self.kwargs = kwargs  #: keyword arguments
        # check if simulation file is in keyword arguments or is first argument
        simfile = kwargs.get('simfile')  # defaults to None
        if simfile is None and args:
            simfile = args[0]
        simfile = simfile or getattr(self, 'param_file', None)
        # read and load JSON parameter map file as "parameters"
        if simfile is not None and os.path.isfile(simfile):
            with open(simfile, 'r') as fp:
                #: parameters from file for simulation
                self.sim_params = json.load(fp)
                # FIXME: remove either sim_params or parameters
                self.parameters = self.sim_params
        else:
            #: parameter file
            self.param_file = None
            self.sim_params = getattr(self, 'parameters')
        _path = self.sim_params.get('path', "~\\Carousel_Simulations\\")
        #: path where all Carousel simulation files are stored
        self.path = os.path.expandvars(os.path.expanduser(_path))
        #: ID for this particular simulation, used for path & file names
        self.ID = self.sim_params['ID']
        #: thresholds for calculations
        self.thresholds = self.sim_params.get('thresholds', {})
        # simulation intervals
        _interval = self.sim_params.get('interval_length', [1, 'hour'])
        #: length of each interval
        self.interval = _interval[0] * UREG[str(_interval[1])]
        _sim_length = self.sim_params.get('simulation_length', [25, 'years'])
        #: simulation length
        self.sim_length = _sim_length[0] * UREG[str(_sim_length[1])]
        # rescale simulation length to interval units to calc no. of intervals
        _sim_length = self.sim_length.to(self.interval.units)
        #: total number of intervals simulated
        self.number_intervals = np.ceil(_sim_length / self.interval)
        #: frequency output is displayed
        self.display_frequency = self.sim_params.get('display_frequency', 12)
        #: output fields displayed
        self.display_fields = self.sim_params.get('display_fields')
        # data dump
        #: frequency output is saved
        self.write_frequency = self.sim_params.get('write_frequency', 8760)
        #: output fields written to disk
        self.write_fields = self.sim_params.get('write_fields')
        #: interval index, start at zero
        self.interval_idx = 0
        #: pause status
        self._ispaused = False
        #: finished status
        self._iscomplete = False
        #: initialized status
        self._isinitialized = False<commit_msg>only positional arg is simfile, defaults to None<commit_after># -*- coding: utf-8 -*-
"""
This is the Simulation module. The Simulation layer takes of creating output
variables, writing data to disk, iterating over data and calculations at
each interval in the simulation and setting any parameters required to perform
the simulation. It gets all its info from the model, which in turn gets it from
each layer which gets info from the layers' sources.
"""

from carousel.core import logging, CommonBase, Registry, UREG
from carousel.core.exceptions import CircularDependencyError
import json
import os
import sys
import numpy as np
import Queue
import functools

LOGGER = logging.getLogger(__name__)


def topological_sort(dag):
    """
    topological sort

    :param dag: directed acyclic graph
    :type dag: dict

    .. seealso:: `Topographical Sorting
        <http://en.wikipedia.org/wiki/Topological_sorting>`_,
        `Directed Acyclic Graph (DAG)
        <https://en.wikipedia.org/wiki/Directed_acyclic_graph>`_
    """
    # find all edges of dag
    topsort = [node for node, edge in dag.iteritems() if not edge]
    # loop through nodes until topologically sorted
    while len(topsort) < len(dag):
        num_nodes = len(topsort)  # number of nodes
        # unsorted nodes
        for node in dag.viewkeys() - set(topsort):
            # nodes with no incoming edges
            if set(dag[node]) <= set(topsort):
                topsort.append(node)
                break
        # circular dependencies
        if len(topsort) == num_nodes:
            raise CircularDependencyError(dag.viewkeys() - set(topsort))
    return topsort


class SimRegistry(Registry):
    #: meta names
    _meta_names = ['commands']

    def __init__(self):
        super(SimRegistry, self).__init__()
        #: simulation commands
        self.commands = {}

    def register(self, sim, *args, **kwargs):
        kwargs.update(zip(self._meta_names, args))
        # call super method, now meta can be passed as args or kwargs.
        super(SimRegistry, self).register(sim, **kwargs)


class SimBase(CommonBase):
    """
    Meta class for simulations.
    """
    _path_attr = 'sim_path'
    _file_attr = 'sim_file'

    def __new__(mcs, name, bases, attr):
        # use only with Simulation subclasses
        if not CommonBase.get_parents(bases, SimBase):
            return super(SimBase, mcs).__new__(mcs, name, bases, attr)
        # set param file full path if simulations path and file specified or
        # try to set parameters from class attributes except private/magic
        attr = mcs.set_param_file_or_parameters(attr)
        return super(SimBase, mcs).__new__(mcs, name, bases, attr)


class Simulation(object):
    """
    A class for simulations.

    :param simfile: Filename of simulation configuration file.
    :type simfile: str
    """
    __metaclass__ = SimBase

    def __init__(self, simfile=None, **kwargs):
        # check if simulation file is in keyword arguments or is first argument
        # simfile = simfile or kwargs.get('simfile')
        if simfile is None:
            simfile = kwargs.get('simfile')  # defaults to None
        # simfile = simfile or getattr(self, 'param_file', None)
        if simfile is None:
            simfile = getattr(self, 'param_file', None)
        # read and load JSON parameter map file as "parameters"
        if simfile is not None:
            with open(simfile, 'r') as fp:
                #: parameters from file for simulation
                self.parameters = json.load(fp)
        else:
            #: parameter file
            self.param_file = None
        # use any keyword arguments instead of parameters
        self.parameters.update(kwargs)
        _path = self.parameters.get('path', "~\\Carousel_Simulations\\")
        #: path where all Carousel simulation files are stored
        self.path = os.path.expandvars(os.path.expanduser(_path))
        #: ID for this particular simulation, used for path & file names
        self.ID = self.parameters['ID']
        #: thresholds for calculations
        self.thresholds = self.parameters.get('thresholds', {})
        # simulation intervals
        _interval = self.parameters.get('interval_length', [1, 'hour'])
        #: length of each interval
        self.interval = _interval[0] * UREG[str(_interval[1])]
        _sim_length = self.parameters.get('simulation_length', [25, 'years'])
        #: simulation length
        self.sim_length = _sim_length[0] * UREG[str(_sim_length[1])]
        # rescale simulation length to interval units to calc no. of intervals
        _sim_length = self.sim_length.to(self.interval.units)
        #: total number of intervals simulated
        self.number_intervals = np.ceil(_sim_length / self.interval)
        #: frequency output is displayed
        self.display_frequency = self.parameters.get('display_frequency', 12)
        #: output fields displayed
        self.display_fields = self.parameters.get('display_fields')
        # data dump
        #: frequency output is saved
        self.write_frequency = self.parameters.get('write_frequency', 8760)
        #: output fields written to disk
        self.write_fields = self.parameters.get('write_fields')
        #: interval index, start at zero
        self.interval_idx = 0
        #: pause status
        self._ispaused = False
        #: finished status
        self._iscomplete = False
        #: initialized status
        self._isinitialized = False
{'fraction_non_alphanumeric': 0.09124482075301747, 'fraction_numerical': 0.002792289677535579, 'mean_word_length': 2.63913470993117, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6316535', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2986, 'n_words': 979}