<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2014-2015 Université Catholique de Louvain.
#
# This file is part of INGInious.
#
# INGInious is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# INGInious is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with INGInious.  If not, see <http://www.gnu.org/licenses/>.
""" A Job Manager that automatically launch Agents on distant Docker daemons """

import docker
import docker.utils

from backend.job_managers.remote_manual_agent import RemoteManualAgentJobManager

AGENT_CONTAINER_VERSION = "0.2"


class RemoteDockerJobManager(RemoteManualAgentJobManager):
    """ A Job Manager that automatically launch Agents on distant Docker daemons """

    @classmethod
    def is_agent_valid_and_started(cls, docker_connection):
        try:
            container_data = docker_connection.inspect_container("inginious-agent")
            if container_data["Config"]["Labels"]["agent-version"] != AGENT_CONTAINER_VERSION:
                # kill the container
                docker_connection.kill("inginious-agent")
                docker_connection.remove_container("inginious-agent", force=True)
            elif container_data["State"]["Running"] is False:
                # remove the container and restart it
                docker_connection.remove_container("inginious-agent", force=True)
            else:
                return True
        except:
            pass
        return False

    @classmethod
    def is_agent_image_update_needed(cls, docker_connection):
        try:
            image_data = docker_connection.inspect_image("ingi/inginious-agent")
            if image_data["Config"]["Labels"]["agent-version"] != AGENT_CONTAINER_VERSION:
                return True
        except:
            return True
        return False

    def __init__(self, docker_daemons, image_aliases, hook_manager=None, is_testing=False):
        """
            Starts the job manager.

            :param docker_daemons:
                a list of dict representing docker daemons.

                { "remote_host": "192.168.59.103", ## host of the docker daemon *from the frontend*
                  "remote_docker_port": 2375, ## port of the distant docker daemon *from the frontend*
                  "remote_agent_port": 63456 ## a mandatory port used by the backend and the agent that will be automatically started. Needs to be
                                             ## available on the remote host, and to be open in the firewall.
                  ##does the docker daemon requires tls? Defaults to false
                  #use_tls: false
                  ##link to the docker daemon *from the host that runs the docker daemon*. Defaults to:
                  #"local_location": "unix:///var/run/docker.sock"
                  ##path to the cgroups "mount" *from the host that runs the docker daemon*. Defaults to:
                  #"cgroups_location": "/sys/fs/cgroup"
                }
            :param image_aliases: a dict of image aliases, like {"default": "ingi/inginious-c-default"}.
            :param hook_manager: An instance of HookManager. If no instance is given(None), a new one will be created.
        """
        agents = []

        for daemon in docker_daemons:
            docker_connection = docker.Client(base_url="tcp://" + daemon['remote_host'] + ":" + str(int(daemon["remote_docker_port"])),
                                              tls=daemon.get(
                                                  "use_tls", False))

            # Verify if the container is available and at the right version
            if not self.is_agent_valid_and_started(docker_connection):

                # Verify that the image ingi/inginious-agent exists and is up-to-date
                if self.is_agent_image_update_needed(docker_connection):
                    docker_connection.pull("ingi/inginious-agent")

                    # Verify again that the image is ok
                    if self.is_agent_image_update_needed(docker_connection):
                        raise Exception("The downloaded image ingi/inginious-agent is not at the same version as this instance of INGInious. " \
                                        "Please update  INGInious or pull manually a valid version of the container image ingi/inginious-agent.")

                docker_local_location = daemon.get("local_location", "unix:///var/run/docker.sock")
                environment = {"AGENT_CONTAINER_NAME": "inginious-agent", "AGENT_PORT": daemon.get("remote_agent_port", 63456)}
                volumes = {'/sys/fs/cgroup/': {}}
                binds = {daemon.get('cgroups_location', "/sys/fs/cgroup"): {'ro': False, 'bind': "/sys/fs/cgroup"}}

                if docker_local_location.startswith("unix://"):<commit_msg>Be more verbose when pulling the agent container<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2014-2015 Université Catholique de Louvain.
#
# This file is part of INGInious.
#
# INGInious is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# INGInious is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with INGInious.  If not, see <http://www.gnu.org/licenses/>.
""" A Job Manager that automatically launch Agents on distant Docker daemons """

import docker
import docker.utils

from backend.job_managers.remote_manual_agent import RemoteManualAgentJobManager

AGENT_CONTAINER_VERSION = "0.2"


class RemoteDockerJobManager(RemoteManualAgentJobManager):
    """ A Job Manager that automatically launch Agents on distant Docker daemons """

    @classmethod
    def is_agent_valid_and_started(cls, docker_connection):
        try:
            container_data = docker_connection.inspect_container("inginious-agent")
            if container_data["Config"]["Labels"]["agent-version"] != AGENT_CONTAINER_VERSION:
                # kill the container
                docker_connection.kill("inginious-agent")
                docker_connection.remove_container("inginious-agent", force=True)
            elif container_data["State"]["Running"] is False:
                # remove the container and restart it
                docker_connection.remove_container("inginious-agent", force=True)
            else:
                return True
        except:
            pass
        return False

    @classmethod
    def is_agent_image_update_needed(cls, docker_connection):
        try:
            image_data = docker_connection.inspect_image("ingi/inginious-agent")
            if image_data["Config"]["Labels"]["agent-version"] != AGENT_CONTAINER_VERSION:
                return True
        except:
            return True
        return False

    def __init__(self, docker_daemons, image_aliases, hook_manager=None, is_testing=False):
        """
            Starts the job manager.

            :param docker_daemons:
                a list of dict representing docker daemons.

                { "remote_host": "192.168.59.103", ## host of the docker daemon *from the frontend*
                  "remote_docker_port": 2375, ## port of the distant docker daemon *from the frontend*
                  "remote_agent_port": 63456 ## a mandatory port used by the backend and the agent that will be automatically started. Needs to be
                                             ## available on the remote host, and to be open in the firewall.
                  ##does the docker daemon requires tls? Defaults to false
                  #use_tls: false
                  ##link to the docker daemon *from the host that runs the docker daemon*. Defaults to:
                  #"local_location": "unix:///var/run/docker.sock"
                  ##path to the cgroups "mount" *from the host that runs the docker daemon*. Defaults to:
                  #"cgroups_location": "/sys/fs/cgroup"
                }
            :param image_aliases: a dict of image aliases, like {"default": "ingi/inginious-c-default"}.
            :param hook_manager: An instance of HookManager. If no instance is given(None), a new one will be created.
        """
        agents = []

        for daemon in docker_daemons:
            docker_connection = docker.Client(base_url="tcp://" + daemon['remote_host'] + ":" + str(int(daemon["remote_docker_port"])),
                                              tls=daemon.get(
                                                  "use_tls", False))

            # Verify if the container is available and at the right version
            if not self.is_agent_valid_and_started(docker_connection):

                # Verify that the image ingi/inginious-agent exists and is up-to-date
                if self.is_agent_image_update_needed(docker_connection):
                    print "Pulling the image ingi/inginious-agent. Please wait, this can take some time..."
                    for line in docker_connection.pull("ingi/inginious-agent", stream=True):
                        print line

                    # Verify again that the image is ok
                    if self.is_agent_image_update_needed(docker_connection):
                        raise Exception("The downloaded image ingi/inginious-agent is not at the same version as this instance of INGInious. " \
                                        "Please update  INGInious or pull manually a valid version of the container image ingi/inginious-agent.")

                docker_local_location = daemon.get("local_location", "unix:///var/run/docker.sock")
                environment = {"AGENT_CONTAINER_NAME": "inginious-agent", "AGENT_PORT": daemon.get("remote_agent_port", 63456)}
                volumes = {'/sys/fs/cgroup/': {}}
                binds = {daemon.get('cgroups_location', "/sys/fs/cgroup"): {'ro': False, 'bind': "/sys/fs/cgroup"}}

                if docker_local_location.startswith("unix://"):
{'fraction_non_alphanumeric': 0.08554351886440045, 'fraction_numerical': 0.006910720956294359, 'mean_word_length': 2.2991373998767712, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1196796', 'n_tokens_mistral': 2943, 'n_tokens_neox': 2740, 'n_words': 908}