<commit_before>#!/usr/bin/env python
"""Command-line utility to generate Dockerfiles, build Docker images, run
commands within containers, and get command ouptut.

Example:

    neurodocker -b ubuntu:17.04 -p apt \\
    --ants version=2.1.0 \\
    --freesurfer version=6.0.0 license_path="./license.txt" \\
    --fsl version=5.0.10 \\
    --miniconda python_version=3.5.1 \\
                conda_install="traits pandas" \\
                pip_install="nipype" \\
    --mrtrix3 use_binaries=false \\
    --spm version=12 matlab_version=R2017a \\
    --neurodebian os_codename=zesty download_server=usa-nh pkgs="dcm2niix" \\
    --instruction='ENTRYPOINT ["entrypoint.sh"]'
"""
# Author: <NAME> <<EMAIL>>

from __future__ import absolute_import, unicode_literals
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import sys

from neurodocker import (__version__, Dockerfile, SpecsParser,
                         SUPPORTED_SOFTWARE)


def create_parser():
    """Return command-line argument parser."""
    parser = ArgumentParser(description=__doc__,
                            formatter_class=RawDescriptionHelpFormatter)

    # Global requirements.
    reqs = parser.add_argument_group(title="global requirements")
    reqs.add_argument("-b", "--base", required=True,
                      help="Base Docker image. Eg, ubuntu:17.04")
    reqs.add_argument("-p", "--pkg-manager", required=True,
                      help="Linux package manager {apt, yum}")

    _neuro_servers = ", ".join(SUPPORTED_SOFTWARE['neurodebian'].SERVERS.keys())

    # Software package options.
    pkgs_help = {
        "all": ("Install software packages. Each argument takes a list of "
                "key=value pairs. Where applicable, the default installation "
                "behavior is to install by downloading and uncompressing "
                "binaries."),
        "ants": ("Install ANTs. Valid keys are version (required), "
                "use_binaries (default true), and git_hash. If use_binaries="
                "true, installs pre-compiled binaries; if use_binaries=false, "
                "builds ANTs from source. If use_binaries is a URL, download "
                "tarball of ANTs binaries from that URL. If git_hash is "
                "specified, build from source from that commit."),
        "freesurfer": ("Install FreeSurfer. Valid keys are version (required),"
                       "license_path (relative path to license), and "
                       "use_binaries (default true). A FreeSurfer license is "
                       "required to run the software and is not provided by "
                       "Neurodocker."),
        "fsl": ("Install FSL. Valid keys are version (required), use_binaries "
                "(default true) and use_installer."),
        "miniconda": ("Install Miniconda. Valid keys are python_version "
                      "(required), conda_install, pip_install, and "
                      "miniconda_version (defaults to latest). The options "
                      "conda_install and pip_install accept strings of "
                      'packages: conda_install="traits numpy".'),
        "mrtrix3": ("Install MRtrix3. Valid keys are use_binaries (default "
                    "true) and git_hash. If git_hash is specified and "
                    "use_binaries is false, will checkout to that commit "
                    "before building."),
        "neurodebian": ("Add NeuroDebian repository and optionally install "
                        "NeuroDebian packages. Valid keys are os_codename "
                        "(required; e.g., 'zesty'), download_server "
                        "(required), full (if false, default, use libre "
                        "packages), and pkgs (list of packages to install). "
                        "Valid download servers are {}."
                        "".format(_neuro_servers)),
        "spm": ("Install SPM (and its dependency, Matlab Compiler Runtime). "
                "Valid keys are version and matlab_version."),
    }

    pkgs = parser.add_argument_group(title="software package arguments",
                                     description=pkgs_help['all'])

    list_of_kv = lambda kv: kv.split("=")

    for p in SUPPORTED_SOFTWARE.keys():
        flag = "--{}".format(p)

        # MRtrix3 does not need any arguments by default.
        if p == "mrtrix3":
            pkgs.add_argument(flag, dest=p, action="append", nargs="*",
                              metavar="", type=list_of_kv, help=pkgs_help[p])
            continue

        pkgs.add_argument(flag, dest=p, action="append", nargs="+", metavar="",
                          type=list_of_kv, help=pkgs_help[p])


    # Docker-related arguments.
    other = parser.add_argument_group(title="other options")
    other.add_argument('-i', '--instruction', action="append",
                     help=("Arbitrary Dockerfile instruction. Can be used "
                           "multiple times."))
    other.add_argument('--no-print-df', dest='no_print_df', action="store_true",
                     help="Do not print the Dockerfile")
    other.add_argument('-o', '--output', dest="output",
                     help="If specified, save Dockerfile to file with this name.")
    # other.add_argument('--build', dest="build", action="store_true")


    # Other options.
    parser.add_argument("--check-urls", dest="check_urls", action="store_true",
                        help=("Verify communication with URLs used in "
                              "the build."), default=True,)
    parser.add_argument("--no-check-urls", action="store_false", dest="check_urls",
                        help=("Do not verify communication with URLs used in "
                              "the build."))
    parser.add_argument("-v", "--verbose", action="store_true")
    parser.add_argument("-V", "--version", action="version",
                        version=('neurodocker version {version}'
                                 .format(version=__version__)))

    return parser


def parse_args(args):
    """Return namespace of command-line arguments."""
    parser = create_parser()
    return parser.parse_args(args)<commit_msg>ADD: generate and reprozip sub-commands<commit_after>#!/usr/bin/env python
"""Command-line utility to generate Dockerfiles, build Docker images, run
commands within containers, and get command ouptut.

Example:

    neurodocker -b ubuntu:17.04 -p apt \\
    --ants version=2.1.0 \\
    --freesurfer version=6.0.0 license_path="./license.txt" \\
    --fsl version=5.0.10 \\
    --miniconda python_version=3.5.1 \\
                conda_install="traits pandas" \\
                pip_install="nipype" \\
    --mrtrix3 use_binaries=false \\
    --spm version=12 matlab_version=R2017a \\
    --neurodebian os_codename=zesty download_server=usa-nh pkgs="dcm2niix" \\
    --instruction='ENTRYPOINT ["entrypoint.sh"]'
"""
# Author: <NAME> <<EMAIL>>

from __future__ import absolute_import, unicode_literals
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import sys

from neurodocker import (__version__, Dockerfile, SpecsParser,
                         SUPPORTED_SOFTWARE)


def create_parser():
    """Return command-line argument parser."""
    parser = ArgumentParser(description=__doc__,
                            formatter_class=RawDescriptionHelpFormatter)

    subparsers = parser.add_subparsers()
    gen_parser = subparsers.add_parser('generate', description=__doc__,
                                       formatter_class=RawDescriptionHelpFormatter)

    description = """
Trace an arbitrary number of commands in a running container using ReproZip,
and save pack file to host.

Example:
    $> cmd1="echo Hello World"
    $> cmd2="antsRegistration --help"
    $> neurodocker reprozip --dir=~/pack_files f101b18bf98c $cmd1 $cmd2
"""

    reprozip_parser = subparsers.add_parser('reprozip', description=description,
                                            formatter_class=RawDescriptionHelpFormatter)

    # Global requirements.
    reqs = gen_parser.add_argument_group(title="global requirements")
    reqs.add_argument("-b", "--base", required=True,
                      help="Base Docker image. Eg, ubuntu:17.04")
    reqs.add_argument("-p", "--pkg-manager", required=True,
                      help="Linux package manager {apt, yum}")

    _neuro_servers = ", ".join(SUPPORTED_SOFTWARE['neurodebian'].SERVERS.keys())

    # Software package options.
    pkgs_help = {
        "all": ("Install software packages. Each argument takes a list of "
                "key=value pairs. Where applicable, the default installation "
                "behavior is to install by downloading and uncompressing "
                "binaries."),
        "ants": ("Install ANTs. Valid keys are version (required), "
                "use_binaries (default true), and git_hash. If use_binaries="
                "true, installs pre-compiled binaries; if use_binaries=false, "
                "builds ANTs from source. If use_binaries is a URL, download "
                "tarball of ANTs binaries from that URL. If git_hash is "
                "specified, build from source from that commit."),
        "freesurfer": ("Install FreeSurfer. Valid keys are version (required),"
                       "license_path (relative path to license), and "
                       "use_binaries (default true). A FreeSurfer license is "
                       "required to run the software and is not provided by "
                       "Neurodocker."),
        "fsl": ("Install FSL. Valid keys are version (required), use_binaries "
                "(default true) and use_installer."),
        "miniconda": ("Install Miniconda. Valid keys are python_version "
                      "(required), conda_install, pip_install, and "
                      "miniconda_version (defaults to latest). The options "
                      "conda_install and pip_install accept strings of "
                      'packages: conda_install="traits numpy".'),
        "mrtrix3": ("Install MRtrix3. Valid keys are use_binaries (default "
                    "true) and git_hash. If git_hash is specified and "
                    "use_binaries is false, will checkout to that commit "
                    "before building."),
        "neurodebian": ("Add NeuroDebian repository and optionally install "
                        "NeuroDebian packages. Valid keys are os_codename "
                        "(required; e.g., 'zesty'), download_server "
                        "(required), full (if false, default, use libre "
                        "packages), and pkgs (list of packages to install). "
                        "Valid download servers are {}."
                        "".format(_neuro_servers)),
        "spm": ("Install SPM (and its dependency, Matlab Compiler Runtime). "
                "Valid keys are version and matlab_version."),
    }

    pkgs = gen_parser.add_argument_group(title="software package arguments",
                                     description=pkgs_help['all'])

    list_of_kv = lambda kv: kv.split("=")

    for p in SUPPORTED_SOFTWARE.keys():
        flag = "--{}".format(p)

        # MRtrix3 does not need any arguments by default.
        if p == "mrtrix3":
            pkgs.add_argument(flag, dest=p, action="append", nargs="*",
                              metavar="", type=list_of_kv, help=pkgs_help[p])
            continue

        pkgs.add_argument(flag, dest=p, action="append", nargs="+", metavar="",
                          type=list_of_kv, help=pkgs_help[p])


    # Docker-related arguments.
    other = gen_parser.add_argument_group(title="other options")
    other.add_argument('-i', '--instruction', action="append",
                     help=("Arbitrary Dockerfile instruction. Can be used "
                           "multiple times."))
    other.add_argument('--no-print-df', dest='no_print_df', action="store_true",
                     help="Do not print the Dockerfile")
    other.add_argument('-o', '--output', dest="output",
                     help="If specified, save Dockerfile to file with this name.")
    # other.add_argument('--build', dest="build", action="store_true")


    # ReproZip sub-command
    reprozip_parser = subparsers.add_parser('reprozip', description=description,
                                            formatter_class=RawDescriptionHelpFormatter)
    reprozip_parser.add_argument('container',
                                 help=("Running container in which to trace "
                                       "commands."))
    reprozip_parser.add_argument('commands', nargs='+',
                                 help="Command(s) to trace.")
    reprozip_parser.add_argument('--dir', '-d', default=".",
                                 help=("Directory in which to save pack file. "
                                       "Default is current directory."))


    # Other options.
    gen_parser.add_argument("--check-urls", dest="check_urls", action="store_true",
                        help=("Verify communication with URLs used in "
                              "the build."), default=True)
    gen_parser.add_argument("--no-check-urls", action="store_false", dest="check_urls",
                        help=("Do not verify communication with URLs used in "
                              "the build."))
    parser.add_argument("-v", "--verbose", action="store_true")
    parser.add_argument("-V", "--version", action="version",
                        version=('neurodocker version {version}'
                                 .format(version=__version__)))

    return parser


def parse_args(args):
    """Return namespace of command-line arguments."""
    parser = create_parser()
    return parser.parse_args(args)
{'fraction_non_alphanumeric': 0.11238150372675303, 'fraction_numerical': 0.005572038497720529, 'mean_word_length': 2.1105109160477156, 'pattern_counts': {'":': 18, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330357', 'n_tokens_mistral': 3808, 'n_tokens_neox': 3578, 'n_words': 993}