<commit_before>from __future__ import absolute_import, division, print_function
import os
import posixpath
import threading

import docker

from ..utils import logger


def _create_volumes_dict(volume_mapping):
    """Create dictionary to pass to `volumes` kwarg of `docker.api.build()`.

    >>> volume_mapping = [("local/dir", "/path/on/docker", "rw")]
    >>> create_volumes_dict(volume_mapping)
    {'/abspath/to/local/dir': {'bind': '/path/on/docker',
                               'mode': 'rw'}}

    Parameters
    ----------
    volume_mapping : list
        List of tuples. Items in each tuple specify the directory on the local
        machine, the directory in the docker container to which the local
        directory should be mapped, and the read/write permissions of that
        directory. Read-write permissions can be either "rw" for read/write or
        "ro" for read-only. Permissions will be read-write if not specified.

    Returns
    -------
    volumes : dict
        Dictionary of volumes to be passed to `volumes` kwargs of
        `docker.api.build()`.
    """
    volumes = {}
    for this_volume in volume_mapping:
        try:
            local_path, docker_path, permissions = this_volume
        except ValueError:
            local_path, docker_path = this_volume
            permissions = "rw"

        # There is a chance symbolic links could cause issues with Docker.
        local_path = os.path.realpath(local_path)
        if not posixpath.isabs(docker_path):
            raise ValueError("Path '{}' is not absolute".format(docker_path))
        if permissions not in ['rw', 'ro']:
            raise ValueError("Unrecognized permissions: '{}'"
                             "".format(permissions))

        volumes[local_path] = {"bind": docker_dir, "mode": permissions}

    return volumes


class EventLogger(threading.Thread):
    """Log Docker events in separate thread.

    >>> events = client.events(decode=True)
    >>> event_logger = EventLogger(events, name="Event-Logger", daemon=True)
    >>> event_logger.start()

    Parameters
    ----------
    events : generator
        Generator of Docker logs. Returned by docker.APIClient.events().
    """
    def __init__(self, events, **kwargs):
        self.events = events
        super(EventLogger, self).__init__(**kwargs)

    def run(self):
        for line in self.events:
            try:
                for key in line:
                    logger.debug("{}: {}".format(key, line[key]))
            except:
                logger.debug(line)


class DockerImage(object):
    """Class to build and run Docker image.

    Parameters
    ----------

    """
    client = docker.from_env()
    # Test communication with the Docker daemon. Raises docker.errors.APIError
    # if communication fails.
    client.ping()

    # Log Docker events in separate thread.
    events = client.events(decode=True)
    event_logger = EventLogger(events, name="Event-Logger")
    event_logger.daemon = True
    event_logger.start()

    def __init__(self, path=None, fileobj=None, **kwargs):
        self.path = path
        self.fileobj = fileobj
        # kwargs?

        # Build image. This method uses the higher-level Docker API. To get the
        # raw output (like you would see if using Docker on the command-line),
        # we would have to use the lower-level Docker API. The lower-level
        # build method returns a blocking generator of the raw output, so we
        # would have to figure out how to get the image ID after it is built.
        self.image = DockerImage.client.images.build(path=path,
                                                     fileobj=fileobj,
                                                     rm=True,
                                                     **kwargs)

    def _create_volumes_dict(self, volume_mapping):
        """Create mapping of volumes. See the documentation for the global
        function _create_volumes_dict()."""
        self.volumes = _create_volumes_dict(volume_mapping)
        return self.volumes

    def run(self, command, **kwargs):
        """Run command within Docker container. Blocks until command finishes
        running. Console output of the command can be accessed with the method
        `self.container.logs()`.

        Parameters
        ----------
        command : str
            Command to run within container.
        """
        # The below method will return a container object only if `detach`
        # is true.
        self.container = DockerImage.client.containers.run(self.image,
                                                           command=command,
                                                           detach=True)

    def _get_console_output(self):
        """Return the console output of the `run` method."""
        try:
            # `container.logs()` returns bytes.
            return self.container.logs().decode(encoding='utf-8').strip()
        except NameError:
            raise Exception("Container must be run for there to be output.")<commit_msg>enh: improve image build commands; get raw output; check that Docker is running; create DockerContainer class to execute commands in container<commit_after>from __future__ import absolute_import, division, print_function
import os
import posixpath
import threading
import warnings

import docker
import requests

from ..utils import logger


def docker_is_running(client):
    """Return true if Docker server is responsive.

    Parameters
    ----------
    client : docker.client.DockerClient
        The Docker client. E.g., `client = docker.from_env()`.

    Returns
    -------
    running : bool
        True if Docker server is responsive.
    """
    try:
        client.ping()
        return True
    except (requests.exceptions.ConnectionError, docker.errors.APIError):
        return False


# Check whether Docker server is running.
client = docker.from_env()
DOCKER_RUNNING = docker_is_running(client)


def require_docker(func):
    """Raise Exception if Docker server is unresponsive (Docker might not be
    installed or not running). Decorate any function that requires the Docker
    application with this wrapper.

    Parameters
    ----------
    func : callable
        Function that requires Docker to run.

    Returns
    -------
    wrapper : callable
        Wrapped function.
    """
    def wrapper(*args, **kwargs):
        if not DOCKER_RUNNING:
            raise Exception("The Docker server is unresponsive. Is Docker "
                            "installed and running?")
        return func(*args, **kwargs)
    return wrapper



class RawOutputLogger(threading.Thread):
    """Log raw output of Docker commands in separate thread.

    The low-level docker-py API must be used to get raw build logs. Those logs
    are very useful, but they must be combed manually for build errors and
    the ID of the built image.

    For example:
    >>> output = client.api.build(path='path/to/context', rm=True)
    >>> build_logger = RawOutputLogger(output, name="Build-Logger")
    >>> build_logger.daemon = True
    >>> build_logger.start()
    >>>

    Parameters
    ----------
    logs : generator
        Generator of Docker logs. Returned by docker.APIClient.events().
    console : bool
        If true, log to console.
    filepath : str
        Log to file `filepath`.
    """
    def __init__(self, logs, console=True, filepath=None, **kwargs):
        self.logs = logs
        self.logger = self.create_logger(console, filepath)
        self.list_of_logs = []
        super(RawOutputLogger, self).__init__(**kwargs)

    @staticmethod
    def create_logger(console=True, filepath=None):
        """`console` is bool. If `filepath` is specified, saves logs to
        file.
        """
        import logging
        logger = logging.getLogger("build")
        logger.setLevel(logging.DEBUG)
        if console:
            ch = logging.StreamHandler()
            ch.setLevel(logging.DEBUG)
            logger.addHandler(ch)
        if filepath is not None:
            fh = logging.FileHandler(filepath)
            fh.setLevel(logging.DEBUG)
            logger.addHandler(fh)
        return logger

    def run(self):
        for line in self.logs:
            line = line.decode('utf-8')
            self.logger.debug(line)
            self.list_of_logs.append(line)
        self.id = self.get_id()

    def get_id(self):
        """Get ID of built image or container."""
        import re
        last_line = self.list_of_logs[-1]
        if re.search("error", last_line, re.IGNORECASE):
            return None
        elif re.search("successfully built", last_line, re.IGNORECASE):
            return re.search('[0-9a-f]{12}', last_line).group(0)

    def show_logs(self, first=None, last=None):
        """Options `first` and `last` can be integers."""
        if first is not None and last is not None:
            raise ValueError("Options `first` and `last` cannot both be used.")
        elif first is not None:
            return "\n".join(self.list_of_logs[:first])
        elif last is not None:
            return "\n".join(self.list_of_logs[-last:])
        else:
            return "\n".join(self.list_of_logs)



class DockerImage(object):
    """Build Docker image."""
    def __init__(self, path, fileobj=None, tag=None):
        self.path = os.path.abspath(path)
        self.fileobj = None
        self.tag = tag

    @require_docker
    def build(self, **kwargs):
        """Return image object."""
        return client.images.build(path=self.path, fileobj=self.fileobj,
                                   tag=self.tags, **kwargs)

    @require_docker
    def build_raw(self, console=True, filepath=None, **kwargs):
        """Return generator of raw console output."""
        logs = client.api.build(path=self.path, fileobj=self.fileobj,
                                tag=self.tag, **kwargs)
        # QUESTION: how do we prevent multiple threads?
        build_logger = RawOutputLogger(logs, console, filepath,
                                       name="BuildLogger")
        build_logger.daemon = True
        build_logger.start()

        # QUESTION: Is it OK to block here? How else could we do this?
        # Wait for build to finish.
        while build_logger.is_alive():
            pass

        try:
            return client.images.get(build_logger.id)
        except docker.errors.NullResource:
            error_logs = build_logger.show_logs(last=2)
            raise docker.errors.BuildError(error_logs)



class DockerContainer(object):
    def __init__(self, image):
        self.image = image
        self.container = None

    @require_docker
    def start(self, **kwargs):
        """Start the container, and optionally mount volumes. `kwargs` are
        arguments for `client.containers.run()`.

        Equivalent to `docker run -it -d IMAGE`.
        """
        self.container = client.containers.run(self.image, command=None,
                                               detach=True, tty=True,
                                               stdin_open=True, **kwargs)

    @require_docker
    def exec_run(self, cmd, **kwargs):
        """Execute command inside the container. `kwargs` are arguments for
        `container.exec_run()`.

        Equivalent to `docker exec CONTAINER CMD`.

        Parameters
        ----------
        cmd : str or list
            Command to execute.

        Returns
        -------
        output : str or generator
            If `stream` is false, return output of the command as one string.
            If `stream` is true, return generator of command output.
        """
        output = self.container.exec_run(cmd, **kwargs)
        return output.decode('utf-8')

    def cleanup(self, remove=True, **kwargs):
        """Remove the container. Use `force=True` to remove running container.
        """
        # self.container.stop() --> self.container.remove() would be ideal,
        # but .stop() complains about timing out (even though it stopped the
        # container).
        if remove:
            self.container.remove(**kwargs)
        else:
            self.container.kill(**kwargs)

    def save_archive(self, path_in_container, filepath):
        """Save a tarball of a file or folder within the Docker container to
        the local machine.

        Parameters
        ----------
        path_in_container : str
            Absolute path to file or folder in the container.
        filepath : str
            Path and name of saved tarball.
        """
        if not posixpath.isabs(path_in_container):
            raise ValueError("`path_in_container` must be absolute.")

        archive = self.container.get_archive(path_in_container)
        response, stat = archive

        with open(filepath, 'wb') as fp:
            fp.write(response.data)
{'fraction_non_alphanumeric': 0.1007300403852128, 'fraction_numerical': 0.0007766387076731904, 'mean_word_length': 2.0543168880455407, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '634199', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3300, 'n_words': 1077}