<commit_before>import errno
import jinja2
import json
import logging
import os
import shutil

from collections import Mapping
from copy import deepcopy

from neckbeard.scaling import MinScalingBackend  # TODO: Don't hardcode this

logger = logging.getLogger('configuration')


class CircularSeedEnvironmentError(Exception):
    pass


class InfiniteEmptyStringDict(object):
    """
    An infinitely-nested dictionary where all individual values evaluate to ""
    when converted to a string.
    """
    def __str__(self):
        return ''

    def __unicode__(self):
        return ''

    def __len__(self):
        return 0

    def __getitem__(self, key):
        return InfiniteEmptyStringDict()

    def __contains__(self, key):
        return True

    def get(self, key, default=None):
        return self[key]


def mkdir_p(path):
    """
    Borrowed from: http://stackoverflow.com/a/600612/386925
    """
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno != errno.EEXIST or not os.path.isdir(path):
            raise


class ConfigurationManager(object):
    """
    ConfigurationManager accepts the already-parsed JSON configuration
    (probably from ``NeckbeardLoader``) and creates an expanded configuration
    for each individual `CloudResource` in an environment. It:
        * Validates the configuration for required options and internal
          consistency
        * Allows `resource_addons` and `environment_addons` to validate their
          specific slice of configuration
        * Generates and properly scopes the template context required for
          expanding individual configuration options. eg.
            * environment.constants
            * environment.secrets
            * seed_environment.constants
            * seed_environment.secrets
            * node
            * seed_node
        * Determines the current scaling state for each `scaling_group` to
          determine how many nodes should exist
          (using the `ScalingCoordinator`).
        * The end results combines the scaling state, configuration, and
          context to generate an individual expanded configuration for each
          `CloudResource` that should exist.
    """
    def __init__(
        self,
        scaling_backend,
        environments,
        constants=None,
        secrets=None,
        secrets_tpl=None,
        node_templates=None,
    ):
        self.scaling_backend = scaling_backend
        self.environments = environments

        self.constants = constants or {}
        self.secrets = secrets or {}
        self.secrets_tpl = secrets_tpl or {}
        self.node_templates = node_templates or {}

        self._expanded_configuration = {}

    @classmethod
    def from_loader(cls, loader):
        """
        Create a new `ConfigurationManager` from an existing
        ``NeckbeardLoader``.
        """
        raw_config = loader.raw_configuration
        configuration = cls(
            environments=raw_config['environments'],
            scaling_backend=MinScalingBackend(),
            constants=raw_config.get('constants', {}),
            secrets=raw_config.get('secrets', {}),
            secrets_tpl=raw_config.get('secrets_tpl', {}),
            node_templates=raw_config.get('node_templates', {}),
        )

        return configuration

    def is_valid(self):
        # TODO: Actually do some like, you know, validation
        return True

    def print_validation_errors(self):
        pass

    def _get_environment_constants(self, environment_name):
        environments = self.constants.get('environments', {})
        return environments.get(environment_name, {})

    def _get_environment_secrets(self, environment_name):
        environments = self.secrets.get('environments', {})
        return environments.get(environment_name, {})

    def _get_seed_environment_constants(self, environment_name):
        seed_environment_name = self._get_seed_environment_name(
            environment_name,
        )
<commit_msg>Refs #2. Pull the neckbeard info from the loader.<commit_after>import errno
import jinja2
import json
import logging
import os
import shutil

from collections import Mapping
from copy import deepcopy

from neckbeard.scaling import MinScalingBackend  # TODO: Don't hardcode this

logger = logging.getLogger('configuration')


class CircularSeedEnvironmentError(Exception):
    pass


class InfiniteEmptyStringDict(object):
    """
    An infinitely-nested dictionary where all individual values evaluate to ""
    when converted to a string.
    """
    def __str__(self):
        return ''

    def __unicode__(self):
        return ''

    def __len__(self):
        return 0

    def __getitem__(self, key):
        return InfiniteEmptyStringDict()

    def __contains__(self, key):
        return True

    def get(self, key, default=None):
        return self[key]


def mkdir_p(path):
    """
    Borrowed from: http://stackoverflow.com/a/600612/386925
    """
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno != errno.EEXIST or not os.path.isdir(path):
            raise


class ConfigurationManager(object):
    """
    ConfigurationManager accepts the already-parsed configuration
    (probably from ``NeckbeardLoader``) and creates an expanded configuration
    for each individual `CloudResource` in an environment. It:
        * Validates the configuration for required options and internal
          consistency
        * Allows `resource_addons` and `environment_addons` to validate their
          specific slice of configuration
        * Generates and properly scopes the template context required for
          expanding individual configuration options. eg.
            * environment.constants
            * environment.secrets
            * seed_environment.constants
            * seed_environment.secrets
            * node
            * seed_node
        * Determines the current scaling state for each `scaling_group` to
          determine how many nodes should exist
          (using the `ScalingCoordinator`).
        * The end results combines the scaling state, configuration, and
          context to generate an individual expanded configuration for each
          `CloudResource` that should exist.
    """
    def __init__(
        self,
        scaling_backend,
        environments,
        constants=None,
        neckbeard_configuration=None,
        secrets=None,
        secrets_tpl=None,
        node_templates=None,
    ):
        self.scaling_backend = scaling_backend
        self.environments = environments

        self.constants = constants or {}
        self.neckbeard_configuration = neckbeard_configuration or {}
        self.secrets = secrets or {}
        self.secrets_tpl = secrets_tpl or {}
        self.node_templates = node_templates or {}

        self._expanded_configuration = {}

    @classmethod
    def from_loader(cls, loader):
        """
        Create a new `ConfigurationManager` from an existing
        ``NeckbeardLoader``.
        """
        raw_config = loader.raw_configuration
        configuration = cls(
            environments=raw_config['environments'],
            scaling_backend=MinScalingBackend(),
            constants=raw_config.get('constants', {}),
            neckbeard_configuration=raw_config.get('neckbeard', {}),
            secrets=raw_config.get('secrets', {}),
            secrets_tpl=raw_config.get('secrets_tpl', {}),
            node_templates=raw_config.get('node_templates', {}),
        )

        return configuration

    def is_valid(self):
        # TODO: Actually do some like, you know, validation
        return True

    def print_validation_errors(self):
        pass

    def _get_environment_constants(self, environment_name):
        environments = self.constants.get('environments', {})
        return environments.get(environment_name, {})

    def _get_environment_secrets(self, environment_name):
        environments = self.secrets.get('environments', {})
        return environments.get(environment_name, {})

    def _get_seed_environment_constants(self, environment_name):
        seed_environment_name = self._get_seed_environment_name(
            environment_name,
        )

{'fraction_non_alphanumeric': 0.07877910483773676, 'fraction_numerical': 0.003981179876945349, 'mean_word_length': 2.6169284467713787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '784380', 'n_tokens_mistral': 2160, 'n_tokens_neox': 2131, 'n_words': 580}