<commit_before>#!/usr/bin/env python3

"""
Set up Docker Compose environment, run a command, and then clean up said environment.

Usage:

    ./dev/run.py common bash

or:

    ./dev/run.py common -- py.test -v -s /path/to/test.py

This script can print the commands that are going to be run instead of running them itself:

    ./dev/run.py -p common bash

"""

import os
import re
import subprocess
from pathlib import Path
from typing import List

# Given that docker-compose is present, we assume that PyYAML is installed

from utils import DockerArgumentParser, DockerArguments

try:
    from yaml import safe_load as load_yaml
except ModuleNotFoundError:
    raise ImportError("Please install PyYAML.")

DOCKER_COMPOSE_FILENAME = 'docker-compose.tests.yml'
"""Filename of 'docker-compose.yml' used for development and running tests."""


class InvalidDockerComposeYMLException(Exception):
    """Exception that gets thrown on docker-compose.yml errors."""


def _validate_docker_compose_yml(docker_compose_path: str, container_name: str) -> None:
    """
    Validate docker-compose.yml, throw exception on errors
    :param docker_compose_path: Path to docker-compose.yml
    """

    with open(docker_compose_path, mode='r', encoding='utf-8') as f:

        try:
            yaml_root = load_yaml(f)
        except Exception as ex:
            raise InvalidDockerComposeYMLException("Unable to load YAML file: {}".format(ex))

        if 'services' not in yaml_root:
            raise InvalidDockerComposeYMLException("No 'services' key under root.")

        if container_name not in yaml_root['services']:
            raise InvalidDockerComposeYMLException("No '{} key under 'services'.".format(container_name))


def _project_name(container_name: str, command: List[str]) -> str:
    """
    Return docker-compose "project name" for a specific command.

    :param container_name: Main container's name.
    :param command: Command to run in said container.
    :return: docker-compose project name.
    """
    command = ' '.join(command)
    sanitized_command = re.sub(r'\W+', '_', command, flags=re.ASCII).lower()
    sanitized_command = sanitized_command.strip('_')

    project_name = 'mc-{}-{}'.format(container_name, sanitized_command)

    return project_name


def docker_run_commands(all_apps_dir: str, app_dirname: str, command: List[str], map_ports: bool) -> List[List[str]]:
    """
    Return a list commands to execute in order to run a command in the main container within Compose environment.

    :param all_apps_dir: Directory with container subdirectories.
    :param app_dirname: Main container's directory name.
    :param command: Command to run in the main container.
    :param map_ports: True if containers' ports should be mapped to host machine (when configured in "ports:").
    :return: List of commands (as lists) to execute in order to run a command.
    """
    if not os.path.isdir(all_apps_dir):
        raise ValueError("Apps directory '{}' does not exist.".format(all_apps_dir))

    all_apps_dir = os.path.abspath(all_apps_dir)

    main_container_dir = os.path.join(all_apps_dir, app_dirname)

    docker_compose_path = os.path.join(main_container_dir, DOCKER_COMPOSE_FILENAME)
    if not os.path.isfile(docker_compose_path):
        raise ValueError("docker-compose configuration was not found at '{}'.".format(docker_compose_path))

    app_dirname = Path(main_container_dir).parts[-1]
    container_name = app_dirname

    try:
        _validate_docker_compose_yml(docker_compose_path=docker_compose_path, container_name=container_name)
    except InvalidDockerComposeYMLException as ex:
        raise ValueError("docker-compose configuration in '{}' is invalid: {}".format(docker_compose_path, ex))

    project_name = _project_name(container_name=container_name, command=command)

    commands = list()

    map_ports_args = []
    if map_ports:
        map_ports_args = ['--service-ports']

    commands.append(
        [
            'docker-compose',
            '--project-name', project_name,
            '--file', docker_compose_path,
            'run',
            '--rm',
        ] + map_ports_args + [
            container_name,
        ] + command
    )

    commands.append([
        'docker-compose',
        '--project-name', project_name,
        '--file', docker_compose_path,
        'down',
        '--volumes',
    ])

    return commands
<commit_msg>Run docker-compose with --compatibility<commit_after>#!/usr/bin/env python3

"""
Set up Docker Compose environment, run a command, and then clean up said environment.

Usage:

    ./dev/run.py common bash

or:

    ./dev/run.py common -- py.test -v -s /path/to/test.py

This script can print the commands that are going to be run instead of running them itself:

    ./dev/run.py -p common bash

"""

import os
import re
import subprocess
from pathlib import Path
from typing import List

# Given that docker-compose is present, we assume that PyYAML is installed

from utils import DockerArgumentParser, DockerArguments

try:
    from yaml import safe_load as load_yaml
except ModuleNotFoundError:
    raise ImportError("Please install PyYAML.")

DOCKER_COMPOSE_FILENAME = 'docker-compose.tests.yml'
"""Filename of 'docker-compose.yml' used for development and running tests."""


class InvalidDockerComposeYMLException(Exception):
    """Exception that gets thrown on docker-compose.yml errors."""


def _validate_docker_compose_yml(docker_compose_path: str, container_name: str) -> None:
    """
    Validate docker-compose.yml, throw exception on errors
    :param docker_compose_path: Path to docker-compose.yml
    """

    with open(docker_compose_path, mode='r', encoding='utf-8') as f:

        try:
            yaml_root = load_yaml(f)
        except Exception as ex:
            raise InvalidDockerComposeYMLException("Unable to load YAML file: {}".format(ex))

        if 'services' not in yaml_root:
            raise InvalidDockerComposeYMLException("No 'services' key under root.")

        if container_name not in yaml_root['services']:
            raise InvalidDockerComposeYMLException("No '{} key under 'services'.".format(container_name))


def _project_name(container_name: str, command: List[str]) -> str:
    """
    Return docker-compose "project name" for a specific command.

    :param container_name: Main container's name.
    :param command: Command to run in said container.
    :return: docker-compose project name.
    """
    command = ' '.join(command)
    sanitized_command = re.sub(r'\W+', '_', command, flags=re.ASCII).lower()
    sanitized_command = sanitized_command.strip('_')

    project_name = 'mc-{}-{}'.format(container_name, sanitized_command)

    return project_name


def docker_run_commands(all_apps_dir: str, app_dirname: str, command: List[str], map_ports: bool) -> List[List[str]]:
    """
    Return a list commands to execute in order to run a command in the main container within Compose environment.

    :param all_apps_dir: Directory with container subdirectories.
    :param app_dirname: Main container's directory name.
    :param command: Command to run in the main container.
    :param map_ports: True if containers' ports should be mapped to host machine (when configured in "ports:").
    :return: List of commands (as lists) to execute in order to run a command.
    """
    if not os.path.isdir(all_apps_dir):
        raise ValueError("Apps directory '{}' does not exist.".format(all_apps_dir))

    all_apps_dir = os.path.abspath(all_apps_dir)

    main_container_dir = os.path.join(all_apps_dir, app_dirname)

    docker_compose_path = os.path.join(main_container_dir, DOCKER_COMPOSE_FILENAME)
    if not os.path.isfile(docker_compose_path):
        raise ValueError("docker-compose configuration was not found at '{}'.".format(docker_compose_path))

    app_dirname = Path(main_container_dir).parts[-1]
    container_name = app_dirname

    try:
        _validate_docker_compose_yml(docker_compose_path=docker_compose_path, container_name=container_name)
    except InvalidDockerComposeYMLException as ex:
        raise ValueError("docker-compose configuration in '{}' is invalid: {}".format(docker_compose_path, ex))

    project_name = _project_name(container_name=container_name, command=command)

    commands = list()

    map_ports_args = []
    if map_ports:
        map_ports_args = ['--service-ports']

    docker_compose = [
        'docker-compose',
        '--project-name', project_name,
        '--file', docker_compose_path,
        # Enable support for "deploy:" in non-swarm mode
        '--compatibility',
    ]

    commands.append(docker_compose + ['run', '--rm'] + map_ports_args + [container_name] + command)
    commands.append(docker_compose + ['down', '--volumes'])

    return commands

{'fraction_non_alphanumeric': 0.10216823703030992, 'fraction_numerical': 0.0006811215802020661, 'mean_word_length': 3.550619834710744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1238371', 'n_tokens_mistral': 2754, 'n_tokens_neox': 2592, 'n_words': 749}