<commit_before>"""Defines the class for a Docker pull task"""
from __future__ import unicode_literals

import datetime

from job.tasks.base_task import AtomicCounter, Task
from node.resources.node_resources import NodeResources
from node.resources.resource import Cpus, Mem


PULL_TASK_ID_PREFIX = 'scale_pull'
COUNTER = AtomicCounter()


class PullTask(Task):
    """Represents a task that pulls Docker images from the registry. This class is thread-safe.
    """

    def __init__(self, framework_id, agent_id, image_name=None):
        """Constructor

        :param framework_id: The framework ID
        :type framework_id: string
        :param agent_id: The agent ID
        :type agent_id: string
        :param image_name: The name of the Docker image to pull. If None, pulls the Scale Docker image
        :type image_name: string
        """

        task_id = '%s_%s_%d' % (PULL_TASK_ID_PREFIX, framework_id, COUNTER.get_next())
        super(PullTask, self).__init__(task_id, 'Scale Docker Pull', agent_id)

        if image_name:
            self.image_name = image_name
        else:
            self.image_name = self._create_scale_image_name()

        self._uses_docker = False
        self._docker_image = None
        self._force_docker_pull = False
        self._docker_params = []
        self._is_docker_privileged = False
        self._running_timeout_threshold = datetime.timedelta(minutes=15)
        self._staging_timeout_threshold = datetime.timedelta(minutes=2)

        # Create command that pulls image and deletes any dangling images
        pull_cmd = 'docker pull %s' % self.image_name
        delete_cmd = 'for img in `docker images -q -f dangling=true`; do docker rmi $img; done'
        self._command = '%s && %s' % (pull_cmd, delete_cmd)

    def get_resources(self):
        """See :meth:`job.tasks.base_task.Task.get_resources`
        """

        return NodeResources([Cpus(0.1), Mem(32.0)])
<commit_msg>Updated pull task to set DOCKER_CONFIG when needed<commit_after>"""Defines the class for a Docker pull task"""
from __future__ import unicode_literals

import datetime

from django.conf import settings

from job.tasks.base_task import AtomicCounter, Task
from node.resources.node_resources import NodeResources
from node.resources.resource import Cpus, Mem


PULL_TASK_ID_PREFIX = 'scale_pull'
COUNTER = AtomicCounter()


class PullTask(Task):
    """Represents a task that pulls Docker images from the registry. This class is thread-safe.
    """

    def __init__(self, framework_id, agent_id, image_name=None):
        """Constructor

        :param framework_id: The framework ID
        :type framework_id: string
        :param agent_id: The agent ID
        :type agent_id: string
        :param image_name: The name of the Docker image to pull. If None, pulls the Scale Docker image
        :type image_name: string
        """

        task_id = '%s_%s_%d' % (PULL_TASK_ID_PREFIX, framework_id, COUNTER.get_next())
        super(PullTask, self).__init__(task_id, 'Scale Docker Pull', agent_id)

        if image_name:
            self.image_name = image_name
        else:
            self.image_name = self._create_scale_image_name()

        self._uses_docker = False
        self._docker_image = None
        self._force_docker_pull = False
        self._docker_params = []
        self._is_docker_privileged = False
        self._running_timeout_threshold = datetime.timedelta(minutes=15)
        self._staging_timeout_threshold = datetime.timedelta(minutes=2)

        # Create command that pulls image and deletes any dangling images
        pull_cmd = 'docker pull %s' % self.image_name
        delete_cmd = 'for img in `docker images -q -f dangling=true`; do docker rmi $img; done'
        self._command = '%s && %s' % (pull_cmd, delete_cmd)

        # Setting DOCKER_CONFIG env var is needed if CONFIG_URI is set for custom Docker configuration
        if settings.CONFIG_URI:
            export_cmd = 'export DOCKER_CONFIG=`pwd`/.docker'
            self._command = '%s && %s' % (export_cmd, self._command)

    def get_resources(self):
        """See :meth:`job.tasks.base_task.Task.get_resources`
        """

        return NodeResources([Cpus(0.1), Mem(32.0)])

{'fraction_non_alphanumeric': 0.08959810874704492, 'fraction_numerical': 0.0037825059101654845, 'mean_word_length': 3.0180436847103516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '215713', 'n_tokens_mistral': 1308, 'n_tokens_neox': 1233, 'n_words': 359}