<commit_before># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------
import os
import tempfile
import datetime
from subprocess import check_output, CalledProcessError
from concurrent.futures import ThreadPoolExecutor, as_completed

try:
    import xmlrpclib
except ImportError:
    import xmlrpc.client as xmlrpclib  # pylint: disable=import-error

from ..utilities.display import print_heading
from ..utilities.path import get_repo_root
from ..utilities.pypi import is_available_on_pypi

# TODO Add 'msi' once we support it
BUILD_TYPES = ['debian', 'docker', 'rpm', 'pypi', 'homebrew']

def build_all_debian(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    # To only build a certain debian package, comment out the ones you don't want from the list below
    dists = [
        ('wheezy', 'debian:wheezy'),
        ('jessie', 'debian:jessie'),
        ('stretch', 'debian:stretch'),
        ('artful', 'ubuntu:artful'),
        ('xenial', 'ubuntu:xenial'),
        ('trusty', 'ubuntu:trusty'),
    ]
    with ThreadPoolExecutor(max_workers=len(dists)) as executor:
            tasks = {executor.submit(build_debian, dist_info, git_url, git_branch, cli_version, artifact_dir, arg_ns=arg_ns) for dist_info
                    in dists}
            for t in as_completed(tasks):
                t.result()
    print('Finished debian builds for {}. Check each build message above for completion status.'.format(', '.join([d[0] for d in dists])))

def build_debian(dist_info, git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    # If you need to release a revision to the package, change this number then reset back to 1 for the next release
    revision = 1
    dist_codename = dist_info[0]
    docker_image = dist_info[1]
    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION=' + cli_version,
           '-e', 'CLI_VERSION_REVISION={}~{}'.format(revision, dist_codename), '-e', 'BUILD_ARTIFACT_DIR=/artifacts',
           '-v', artifact_dir + ':/artifacts', docker_image, '/bin/bash', '-cx',
           'apt-get update && apt-get install -y git wget sudo && git clone --progress --verbose {} --branch {} /repo_clone '
           '&& cd /repo_clone && build_scripts/debian/build.sh /repo_clone'.format(git_url, git_branch)]
    container_id = check_output(cmd, universal_newlines=True).strip()
    print('Debian {} build running. Use `docker logs -f {}`'.format(dist_info[0], container_id))
    exit_code = check_output(['docker', 'wait', container_id], universal_newlines=True).strip()
    print('FINISHED Debian {} build. Exit code {} (0 for success)'.format(dist_info[0], exit_code))


def build_docker(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    cmd = ['docker', 'build', '--no-cache', '--quiet', '--build-arg', 'BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`"',
           '--build-arg', 'CLI_VERSION=' + cli_version, get_repo_root()]
    print('Docker build started.')
    image_id = check_output(cmd, universal_newlines=True).strip()
    image_id = image_id.split(':')[1]
    image_file_location = os.path.join(artifact_dir, 'docker-azure-cli-{}.tar'.format(cli_version))
    cmd = ['docker', 'save', '-o', image_file_location, image_id]
    image_id = check_output(cmd, universal_newlines=True).strip()
    print('COMPLETED Docker build. image id: {}, saved to {}'.format(image_id, image_file_location))


def build_rpm(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION=' + cli_version, '-e', 'REPO_PATH=/repo_clone',
           '-v', artifact_dir + ':/artifacts', 'centos:7', '/bin/bash', '-cx',
           'yum check-update; yum install -y gcc git rpm-build rpm-devel rpmlint make bash coreutils diffutils patch '
           'rpmdevtools python libffi-devel python-devel openssl-devel wget && git clone --progress --verbose {} '
           '--branch {} /repo_clone && cd /repo_clone && rpmbuild -v -bb --clean build_scripts/rpm/azure-cli.spec && '
           'cp /root/rpmbuild/RPMS/x86_64/* /artifacts/'.format(git_url, git_branch)]
    container_id = check_output(cmd, universal_newlines=True).strip()
    print('RPM build running. Use `docker logs -f {}` to view logs'.format(container_id))
    exit_code = check_output(['docker', 'wait', container_id], universal_newlines=True).strip()
    print('FINISHED RPM build. Exit code {} (0 for success)'.format(exit_code))

<commit_msg>Add note to build script for docker (#5571)<commit_after># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------
import os
import tempfile
import datetime
from subprocess import check_output, CalledProcessError
from concurrent.futures import ThreadPoolExecutor, as_completed

try:
    import xmlrpclib
except ImportError:
    import xmlrpc.client as xmlrpclib  # pylint: disable=import-error

from ..utilities.display import print_heading
from ..utilities.path import get_repo_root
from ..utilities.pypi import is_available_on_pypi

# TODO Add 'msi' once we support it
BUILD_TYPES = ['debian', 'docker', 'rpm', 'pypi', 'homebrew']

def build_all_debian(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    # To only build a certain debian package, comment out the ones you don't want from the list below
    dists = [
        ('wheezy', 'debian:wheezy'),
        ('jessie', 'debian:jessie'),
        ('stretch', 'debian:stretch'),
        ('artful', 'ubuntu:artful'),
        ('xenial', 'ubuntu:xenial'),
        ('trusty', 'ubuntu:trusty'),
    ]
    with ThreadPoolExecutor(max_workers=len(dists)) as executor:
            tasks = {executor.submit(build_debian, dist_info, git_url, git_branch, cli_version, artifact_dir, arg_ns=arg_ns) for dist_info
                    in dists}
            for t in as_completed(tasks):
                t.result()
    print('Finished debian builds for {}. Check each build message above for completion status.'.format(', '.join([d[0] for d in dists])))

def build_debian(dist_info, git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    # If you need to release a revision to the package, change this number then reset back to 1 for the next release
    revision = 1
    dist_codename = dist_info[0]
    docker_image = dist_info[1]
    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION=' + cli_version,
           '-e', 'CLI_VERSION_REVISION={}~{}'.format(revision, dist_codename), '-e', 'BUILD_ARTIFACT_DIR=/artifacts',
           '-v', artifact_dir + ':/artifacts', docker_image, '/bin/bash', '-cx',
           'apt-get update && apt-get install -y git wget sudo && git clone --progress --verbose {} --branch {} /repo_clone '
           '&& cd /repo_clone && build_scripts/debian/build.sh /repo_clone'.format(git_url, git_branch)]
    container_id = check_output(cmd, universal_newlines=True).strip()
    print('Debian {} build running. Use `docker logs -f {}`'.format(dist_info[0], container_id))
    exit_code = check_output(['docker', 'wait', container_id], universal_newlines=True).strip()
    print('FINISHED Debian {} build. Exit code {} (0 for success)'.format(dist_info[0], exit_code))


def build_docker(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    cmd = ['docker', 'build', '--no-cache', '--quiet', '--build-arg', 'BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`"',
           '--build-arg', 'CLI_VERSION=' + cli_version, get_repo_root()]
    print('Docker build started. The git url and branch parameters are ignored. We use the current repository.')
    image_id = check_output(cmd, universal_newlines=True).strip()
    image_id = image_id.split(':')[1]
    image_file_location = os.path.join(artifact_dir, 'docker-azure-cli-{}.tar'.format(cli_version))
    cmd = ['docker', 'save', '-o', image_file_location, image_id]
    image_id = check_output(cmd, universal_newlines=True).strip()
    print('COMPLETED Docker build. image id: {}, saved to {}'.format(image_id, image_file_location))


def build_rpm(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION=' + cli_version, '-e', 'REPO_PATH=/repo_clone',
           '-v', artifact_dir + ':/artifacts', 'centos:7', '/bin/bash', '-cx',
           'yum check-update; yum install -y gcc git rpm-build rpm-devel rpmlint make bash coreutils diffutils patch '
           'rpmdevtools python libffi-devel python-devel openssl-devel wget && git clone --progress --verbose {} '
           '--branch {} /repo_clone && cd /repo_clone && rpmbuild -v -bb --clean build_scripts/rpm/azure-cli.spec && '
           'cp /root/rpmbuild/RPMS/x86_64/* /artifacts/'.format(git_url, git_branch)]
    container_id = check_output(cmd, universal_newlines=True).strip()
    print('RPM build running. Use `docker logs -f {}` to view logs'.format(container_id))
    exit_code = check_output(['docker', 'wait', container_id], universal_newlines=True).strip()
    print('FINISHED RPM build. Exit code {} (0 for success)'.format(exit_code))


{'fraction_non_alphanumeric': 0.17811249210027386, 'fraction_numerical': 0.003581209184748262, 'mean_word_length': 3.278954484001803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4807985', 'n_tokens_mistral': 3082, 'n_tokens_neox': 2909, 'n_words': 780}