<commit_before>import io
import json
import os
import tarfile
import uuid
from contextlib import contextmanager
from json import JSONDecodeError
from pathlib import Path
from random import randint
from time import sleep
from typing import Tuple

import docker
from django.conf import settings
from django.core.files import File
from docker.api.container import ContainerApiMixin
from docker.errors import ContainerError, APIError
from docker.tls import TLSConfig
from requests import HTTPError


class Executor(object):
    def __init__(
        self,
        *,
        job_id: uuid.UUID,
        input_files: Tuple[File, ...],
        exec_image: File,
        exec_image_sha256: str,
        results_file: Path,
    ):
        super().__init__()
        self._job_id = str(job_id)
        self._input_files = input_files
        self._exec_image = exec_image
        self._exec_image_sha256 = exec_image_sha256
        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE
        self._results_file = results_file

        client_kwargs = {"base_url": settings.CONTAINER_EXEC_DOCKER_BASE_URL}

        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:
            tlsconfig = TLSConfig(
                verify=True,
                client_cert=(
                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,
                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,
                ),
                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,
            )
            client_kwargs.update({"tls": tlsconfig})

        self._client = docker.DockerClient(**client_kwargs)

        self._input_volume = f"{self._job_id}-input"
        self._output_volume = f"{self._job_id}-output"

        self._run_kwargs = {
            "labels": {"job_id": self._job_id},
            "network_disabled": True,
            "mem_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            "cpu_period": settings.CONTAINER_EXEC_CPU_PERIOD,
            "cpu_quota": settings.CONTAINER_EXEC_CPU_QUOTA,
            "runtime": settings.CONTAINER_EXEC_DOCKER_RUNTIME,
        }

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):<commit_msg>Use the JSON logger<commit_after>import io
import json
import os
import tarfile
import uuid
from contextlib import contextmanager
from json import JSONDecodeError
from pathlib import Path
from random import randint
from time import sleep
from typing import Tuple

import docker
from django.conf import settings
from django.core.files import File
from docker.api.container import ContainerApiMixin
from docker.errors import ContainerError, APIError
from docker.tls import TLSConfig
from docker.types import LogConfig
from requests import HTTPError


class Executor(object):
    def __init__(
        self,
        *,
        job_id: uuid.UUID,
        input_files: Tuple[File, ...],
        exec_image: File,
        exec_image_sha256: str,
        results_file: Path,
    ):
        super().__init__()
        self._job_id = str(job_id)
        self._input_files = input_files
        self._exec_image = exec_image
        self._exec_image_sha256 = exec_image_sha256
        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE
        self._results_file = results_file

        client_kwargs = {"base_url": settings.CONTAINER_EXEC_DOCKER_BASE_URL}

        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:
            tlsconfig = TLSConfig(
                verify=True,
                client_cert=(
                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,
                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,
                ),
                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,
            )
            client_kwargs.update({"tls": tlsconfig})

        self._client = docker.DockerClient(**client_kwargs)

        self._input_volume = f"{self._job_id}-input"
        self._output_volume = f"{self._job_id}-output"

        self._run_kwargs = {
            "labels": {"job_id": self._job_id},
            "network_disabled": True,
            "mem_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            "cpu_period": settings.CONTAINER_EXEC_CPU_PERIOD,
            "cpu_quota": settings.CONTAINER_EXEC_CPU_QUOTA,
            "runtime": settings.CONTAINER_EXEC_DOCKER_RUNTIME,
            "log_config": LogConfig(
                type=LogConfig.types.JSON, config={"max-size": "1g"}
            ),
        }

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
{'fraction_non_alphanumeric': 0.07569011576135352, 'fraction_numerical': 0.004229741763134461, 'mean_word_length': 2.554588607594937, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '974340', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1322, 'n_words': 275}