<commit_before># Copyright (c) 2016, <NAME>
# Copyright (c) 2016, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Translates a ZApp description into Docker containers."""

import logging

from zoe_master.workspace.filesystem import ZoeFSWorkspace
from zoe_master.exceptions import ZoeStartExecutionRetryException, ZoeStartExecutionFatalException, ZoeException

from zoe_lib.config import get_conf
from zoe_lib.exceptions import ZoeLibException
from zoe_lib.sql_manager import Execution, Service
from zoe_lib.swarm_client import DockerContainerOptions, SwarmClient

log = logging.getLogger(__name__)


def execution_to_containers(execution: Execution) -> None:
    """Translate an execution object into containers.

    If an error occurs some containers may have been created and needs to be cleaned-up.
    In case of error exceptions are raised.
    """
    ordered_service_list = sorted(execution.services, key=lambda x: x.description['startup_order'])

    env_subst_dict = {
        'execution_id': execution.id,
        "execution_name": execution.name,
        'user_name': execution.user_id,
        'deployment_name': get_conf().deployment_name,
    }

    for service in ordered_service_list:
        env_subst_dict['dns_name#' + service.name] = service.dns_name

    for service in ordered_service_list:
        env_subst_dict['dns_name#self'] = service.dns_name
        service.set_starting()
        _spawn_service(execution, service, env_subst_dict)


def _gen_environment(service, env_subst_dict, copts):
    """ Generate a dictionary containing the current cluster status (before the new container is spawned)

    This information is used to substitute template strings in the environment variables."""
    for env_name, env_value in service.description['environment']:
        try:
            env_value = env_value.format(**env_subst_dict)
        except KeyError:
            raise ZoeStartExecutionFatalException("unknown variable in expression '{}', known variables are: {}".format(env_value, list(env_subst_dict.keys())))
        copts.add_env_variable(env_name, env_value)


def _spawn_service(execution: Execution, service: Service, env_subst_dict: dict):
    copts = DockerContainerOptions()
    copts.gelf_log_address = get_conf().gelf_address
    copts.name = service.dns_name
    copts.set_memory_limit(service.description['required_resources']['memory'])
    copts.network_name = get_conf().overlay_network_name
    copts.labels = {
        'zoe.execution.name': execution.name,
        'zoe.execution.id': str(execution.id),
        'zoe.service.name': service.name,
        'zoe.service.id': str(service.id),
        'zoe.owner': execution.user_id,
        'zoe.deployment_name': get_conf().deployment_name,
        'zoe.type': 'app_service'
    }
    if service.description['monitor']:
        copts.labels['zoe.monitor'] = 'true'
    else:
        copts.labels['zoe.monitor'] = 'false'
    copts.restart = not service.description['monitor']  # Monitor containers should not restart

    _gen_environment(service, env_subst_dict, copts)

    for p in service.description['ports']:
        if 'expose' in p and p['expose']:
            copts.ports.append(p['port_number'])  # FIXME UDP ports?

    if 'volumes' in service.description:
        for path, mount_point, readonly in service.description['volumes']:
            copts.add_volume_bind(path, mount_point, readonly)

    if 'constraints' in service.description:
        for constraint in service.description['constraints']:
            copts.add_constraint(constraint)

    fswk = ZoeFSWorkspace()
    if fswk.can_be_attached():
        copts.add_volume_bind(fswk.get_path(execution.user_id), fswk.get_mountpoint(), False)

    # The same dictionary is used for templates in the command
    if 'command' in service.description:
        copts.set_command(service.description['command'].format(**env_subst_dict))

    try:
        swarm = SwarmClient(get_conf())
    except Exception as e:
        raise ZoeStartExecutionFatalException(str(e))

    try:
        cont_info = swarm.spawn_container(service.description['docker_image'], copts)
    except ZoeException as e:
        raise ZoeStartExecutionRetryException(str(e))
    except ZoeLibException as e:
        raise ZoeStartExecutionRetryException(str(e))

    service.set_active(cont_info["docker_id"])

    if 'networks' in service.description:
        for net in service.description['networks']:
            try:
                swarm.connect_to_network(service.docker_id, net)
            except ZoeException as e:
                raise ZoeStartExecutionFatalException(str(e))
<commit_msg>Add an environment variable to all containers with the workspace path location<commit_after># Copyright (c) 2016, <NAME>
# Copyright (c) 2016, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Translates a ZApp description into Docker containers."""

import logging

from zoe_master.workspace.filesystem import ZoeFSWorkspace
from zoe_master.exceptions import ZoeStartExecutionRetryException, ZoeStartExecutionFatalException, ZoeException

from zoe_lib.config import get_conf
from zoe_lib.exceptions import ZoeLibException
from zoe_lib.sql_manager import Execution, Service
from zoe_lib.swarm_client import DockerContainerOptions, SwarmClient

log = logging.getLogger(__name__)


def execution_to_containers(execution: Execution) -> None:
    """Translate an execution object into containers.

    If an error occurs some containers may have been created and needs to be cleaned-up.
    In case of error exceptions are raised.
    """
    ordered_service_list = sorted(execution.services, key=lambda x: x.description['startup_order'])

    env_subst_dict = {
        'execution_id': execution.id,
        "execution_name": execution.name,
        'user_name': execution.user_id,
        'deployment_name': get_conf().deployment_name,
    }

    for service in ordered_service_list:
        env_subst_dict['dns_name#' + service.name] = service.dns_name

    for service in ordered_service_list:
        env_subst_dict['dns_name#self'] = service.dns_name
        service.set_starting()
        _spawn_service(execution, service, env_subst_dict)


def _gen_environment(service, env_subst_dict, copts):
    """ Generate a dictionary containing the current cluster status (before the new container is spawned)

    This information is used to substitute template strings in the environment variables."""
    for env_name, env_value in service.description['environment']:
        try:
            env_value = env_value.format(**env_subst_dict)
        except KeyError:
            raise ZoeStartExecutionFatalException("unknown variable in expression '{}', known variables are: {}".format(env_value, list(env_subst_dict.keys())))
        copts.add_env_variable(env_name, env_value)


def _spawn_service(execution: Execution, service: Service, env_subst_dict: dict):
    copts = DockerContainerOptions()
    copts.gelf_log_address = get_conf().gelf_address
    copts.name = service.dns_name
    copts.set_memory_limit(service.description['required_resources']['memory'])
    copts.network_name = get_conf().overlay_network_name
    copts.labels = {
        'zoe.execution.name': execution.name,
        'zoe.execution.id': str(execution.id),
        'zoe.service.name': service.name,
        'zoe.service.id': str(service.id),
        'zoe.owner': execution.user_id,
        'zoe.deployment_name': get_conf().deployment_name,
        'zoe.type': 'app_service'
    }
    if service.description['monitor']:
        copts.labels['zoe.monitor'] = 'true'
    else:
        copts.labels['zoe.monitor'] = 'false'
    copts.restart = not service.description['monitor']  # Monitor containers should not restart

    _gen_environment(service, env_subst_dict, copts)

    for p in service.description['ports']:
        if 'expose' in p and p['expose']:
            copts.ports.append(p['port_number'])  # FIXME UDP ports?

    if 'volumes' in service.description:
        for path, mount_point, readonly in service.description['volumes']:
            copts.add_volume_bind(path, mount_point, readonly)

    if 'constraints' in service.description:
        for constraint in service.description['constraints']:
            copts.add_constraint(constraint)

    fswk = ZoeFSWorkspace()
    if fswk.can_be_attached():
        copts.add_volume_bind(fswk.get_path(execution.user_id), fswk.get_mountpoint(), False)
        copts.add_env_variable('ZOE_WORKSPACE', fswk.get_mountpoint())

    # The same dictionary is used for templates in the command
    if 'command' in service.description:
        copts.set_command(service.description['command'].format(**env_subst_dict))

    try:
        swarm = SwarmClient(get_conf())
    except Exception as e:
        raise ZoeStartExecutionFatalException(str(e))

    try:
        cont_info = swarm.spawn_container(service.description['docker_image'], copts)
    except ZoeException as e:
        raise ZoeStartExecutionRetryException(str(e))
    except ZoeLibException as e:
        raise ZoeStartExecutionRetryException(str(e))

    service.set_active(cont_info["docker_id"])

    if 'networks' in service.description:
        for net in service.description['networks']:
            try:
                swarm.connect_to_network(service.docker_id, net)
            except ZoeException as e:
                raise ZoeStartExecutionFatalException(str(e))

{'fraction_non_alphanumeric': 0.0938998657203146, 'fraction_numerical': 0.002301937464032227, 'mean_word_length': 3.7874196510560147, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '96433', 'n_tokens_mistral': 3089, 'n_tokens_neox': 2990, 'n_words': 795}