<commit_before>#!/usr/bin/env python
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

from __future__ import print_function
from multiprocessing.dummy import Pool as ThreadPool
import argparse
import datetime
import errno
import multiprocessing
import os
import pipes
import re
import subprocess
import sys
import tempfile
import templates

OSSFUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
BUILD_DIR = os.path.join(OSSFUZZ_DIR, 'build')

BASE_IMAGES = [
    'gcr.io/oss-fuzz-base/base-image',
    'gcr.io/oss-fuzz-base/base-clang',
    'gcr.io/oss-fuzz-base/base-builder',
    'gcr.io/oss-fuzz-base/base-runner',
    'gcr.io/oss-fuzz-base/base-runner-debug',
    'gcr.io/oss-fuzz-base/base-msan-builder',
    'gcr.io/oss-fuzz-base/msan-builder',
]

VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$')
MAX_PROJECT_NAME_LENGTH = 26

if sys.version_info[0] >= 3:
    raw_input = input

CORPUS_URL_FORMAT = (
    'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/'
    '{fuzz_target}/')
CORPUS_BACKUP_URL_FORMAT = (
    'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/'
    'libFuzzer/{fuzz_target}/')


def main():
  os.chdir(OSSFUZZ_DIR)
  if not os.path.exists(BUILD_DIR):
    os.mkdir(BUILD_DIR)

  parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
  subparsers = parser.add_subparsers(dest='command')

  generate_parser = subparsers.add_parser(
      'generate', help='Generate files for new project.')
  generate_parser.add_argument('project_name')

  build_image_parser = subparsers.add_parser(
      'build_image', help='Build an image.')
  build_image_parser.add_argument('project_name')
  build_image_parser.add_argument('--pull', action='store_true',
                                  help='Pull latest base image.')
  build_image_parser.add_argument('--no-pull', action='store_true',
                                  help='Do not pull latest base image.')

  build_fuzzers_parser = subparsers.add_parser(
      'build_fuzzers', help='Build fuzzers for a project.')
  _add_engine_args(build_fuzzers_parser)
  _add_sanitizer_args(build_fuzzers_parser)
  _add_environment_args(build_fuzzers_parser)
  build_fuzzers_parser.add_argument('project_name')
  build_fuzzers_parser.add_argument('source_path', help='path of local source',
                                    nargs='?')
  build_fuzzers_parser.add_argument('--clean', dest='clean',
                                    action='store_true',
                                    help='clean existing artifacts.')
  build_fuzzers_parser.add_argument('--no-clean', dest='clean',
                                    action='store_false',
                                    help='do not clean existing artifacts '
                                    '(default).')
  build_fuzzers_parser.set_defaults(clean=False)

  check_build_parser = subparsers.add_parser(
      'check_build', help='Checks that fuzzers execute without errors.')
  _add_engine_args(check_build_parser)
  _add_sanitizer_args(check_build_parser)
  _add_environment_args(check_build_parser)
  check_build_parser.add_argument('project_name', help='name of the project')
  check_build_parser.add_argument('fuzzer_name', help='name of the fuzzer',
                                  nargs='?')

  run_fuzzer_parser = subparsers.add_parser(
      'run_fuzzer', help='Run a fuzzer.')
  _add_engine_args(run_fuzzer_parser)
  _add_environment_args(run_fuzzer_parser)
  run_fuzzer_parser.add_argument('project_name', help='name of the project')
  run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  run_fuzzer_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                                 nargs=argparse.REMAINDER)

  coverage_parser = subparsers.add_parser(
      'coverage', help='Run a fuzzer for a while and generate coverage.')
  coverage_parser.add_argument('--run_time', default=60,
                      help='time in seconds to run fuzzer')
  coverage_parser.add_argument('project_name', help='name of the project')
  coverage_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  coverage_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                               nargs=argparse.REMAINDER)

  profile_parser = subparsers.add_parser(
      'profile', help='Generate code coverage report for the project.')
  profile_parser.add_argument('project_name', help='name of the project')
  profile_parser.add_argument('--no-corpus-download', action='store_true',
                              help='do not download corpus backup from OSS-Fuzz; '
                              'use corpus located in build/corpus/<project>/<fuzz_target>/')
  profile_parser.add_argument('--port', default='8008', help='specify port for '
                              'a local HTTP server rendering coverage report')
  profile_parser.add_argument('--fuzz-target', help='specify name of a fuzz '
                              'target to be run for generating coverage report')
  profile_parser.add_argument('--corpus-dir', help='specify location of corpus '
                              'to be used (requires --fuzz-target argument)')

  reproduce_parser = subparsers.add_parser(
      'reproduce', help='Reproduce a crash.')
  reproduce_parser.add_argument('--valgrind', action='store_true',
                       help='run with valgrind')
  reproduce_parser.add_argument('project_name', help='name of the project')
  reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  reproduce_parser.add_argument('testcase_path', help='path of local testcase')
  reproduce_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                                nargs=argparse.REMAINDER)
  _add_environment_args(reproduce_parser)

  shell_parser = subparsers.add_parser(
      'shell', help='Run /bin/bash in an image.')
  shell_parser.add_argument('project_name', help='name of the project')
  _add_engine_args(shell_parser)
  _add_sanitizer_args(shell_parser)
  _add_environment_args(shell_parser)

  pull_images_parser = subparsers.add_parser('pull_images',
                                             help='Pull base images.')

  args = parser.parse_args()
  if args.command == 'generate':
    return generate(args)
  elif args.command == 'build_image':
    return build_image(args)
  elif args.command == 'build_fuzzers':
    return build_fuzzers(args)
  elif args.command == 'check_build':
    return check_build(args)
  elif args.command == 'run_fuzzer':
    return run_fuzzer(args)
  elif args.command == 'coverage':
    return coverage(args)
  elif args.command == 'profile':
    return profile(args)
  elif args.command == 'reproduce':
    return reproduce(args)
  elif args.command == 'shell':
    return shell(args)
  elif args.command == 'pull_images':
    return pull_images(args)

  return 0


def _is_base_image(image_name):
  """Checks if the image name is a base image."""
  return os.path.exists(os.path.join('infra', 'base-images', image_name))


def _check_project_exists(project_name):
  """Checks if a project exists."""
  if not os.path.exists(os.path.join(OSSFUZZ_DIR, 'projects', project_name)):
    print(project_name, 'does not exist', file=sys.stderr)
    return False

  return True


def _check_fuzzer_exists(project_name, fuzzer_name):
  """Checks if a fuzzer exists."""
  command = ['docker', 'run', '--rm']
  command.extend(['-v', '%s:/out' % _get_output_dir(project_name)])
  command.append('ubuntu:16.04')

  command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print(fuzzer_name,
          'does not seem to exist. Please run build_fuzzers first.',
          file=sys.stderr)
    return False

  return True


def _get_absolute_path(path):
  """Returns absolute path with user expansion."""
  return os.path.abspath(os.path.expanduser(path))


def _get_command_string(command):
  """Returns a shell escaped command string."""
  return ' '.join(pipes.quote(part) for part in command)


def _get_corpus_dir(project_name=''):
  """Returns path to /corpus directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'corpus', project_name)


def _get_output_dir(project_name=''):
  """Returns path to /out directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'out', project_name)


def _get_work_dir(project_name=''):
  """Returns path to /work directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'work', project_name)


def _add_engine_args(parser):
  """Add common engine args."""
  parser.add_argument('--engine', default='libfuzzer',
                      choices=['libfuzzer', 'afl', 'honggfuzz', 'none'])


def _add_sanitizer_args(parser):
  """Add common sanitizer args."""
  parser.add_argument('--sanitizer', default='address',
                      choices=['address', 'memory', 'undefined', 'coverage', 'profile'])


def _add_environment_args(parser):
  """Add common environment args."""
  parser.add_argument('-e', action='append',
                      help="set environment variable e.g. VAR=value")


def _build_image(image_name, no_cache=False, pull=False):
  """Build image."""

  is_base_image = _is_base_image(image_name)
  if is_base_image:
    image_project = 'oss-fuzz-base'
    dockerfile_dir = os.path.join('infra', 'base-images', image_name)
  else:
    image_project = 'oss-fuzz'
    if not _check_project_exists(image_name):
      return False

    dockerfile_dir = os.path.join('projects', image_name)

  build_args = []
  if no_cache:
    build_args.append('--no-cache')

  build_args += ['-t', 'gcr.io/%s/%s' % (image_project, image_name), dockerfile_dir]

  return docker_build(build_args, pull=pull)


def _env_to_docker_args(env_list):
  """Turn envirnoment variable list into docker arguments."""
  return sum([['-e', v] for v in env_list], [])


def docker_run(run_args, print_output=True):
  """Call `docker run`."""
  command = ['docker', 'run', '--rm', '-i', '--privileged']
  command.extend(run_args)

  print('Running:', _get_command_string(command))
  stdout = None
  if not print_output:
    stdout = open(os.devnull, 'w')

  try:
    subprocess.check_call(command, stdout=stdout, stderr=subprocess.STDOUT)
  except subprocess.CalledProcessError as e:
    return e.returncode

  return 0


def docker_build(build_args, pull=False):
  """Call `docker build`."""
  command = ['docker', 'build']
  if pull:
    command.append('--pull')

  command.extend(build_args)
  print('Running:', _get_command_string(command))

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print('docker build failed.', file=sys.stderr)
    return False

  return True


def docker_pull(image, pull=False):
  """Call `docker pull`."""
  command = ['docker', 'pull', image]
  print('Running:', _get_command_string(command))

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print('docker pull failed.', file=sys.stderr)
    return False

  return True


def build_image(args):
  """Build docker image."""
  if args.pull and args.no_pull:
    print('Incompatible arguments --pull and --no-pull.')
    return 1

  if args.pull:
    pull = True
  elif args.no_pull:
    pull = False
  else:
    y_or_n = raw_input('Pull latest base images (compiler/runtime)? (y/N): ')
    pull = y_or_n.lower() == 'y'

  if pull:
    print('Pulling latest base images...')
  else:
    print('Using cached base images...')

  # If build_image is called explicitly, don't use cache.
  if _build_image(args.project_name, no_cache=True, pull=pull):
    return 0

  return 1


def build_fuzzers(args):
  """Build fuzzers."""
  project_name = args.project_name
  if not _build_image(args.project_name):
    return 1

  project_out_dir = _get_output_dir(project_name)
  if args.clean:
    print('Cleaning existing build artifacts.')

    # Clean old and possibly conflicting artifacts in project's out directory.
    docker_run([
        '-v', '%s:/out' % project_out_dir,
        '-t', 'gcr.io/oss-fuzz/%s' % project_name,
        '/bin/bash', '-c', 'rm -rf /out/*'
    ])
  else:
    print('Keeping existing build artifacts as-is (if any).')

  env = [
      'FUZZING_ENGINE=' + args.engine,
      'SANITIZER=' + args.sanitizer
  ]
  if args.e:
    env += args.e

  project_work_dir = _get_work_dir(project_name)

  # Copy instrumented libraries.
  if args.sanitizer == 'memory':
    docker_run([
        '-v', '%s:/work' % project_work_dir,
        'gcr.io/oss-fuzz-base/msan-builder',
        'bash', '-c', 'cp -r /msan /work'])
    env.append('MSAN_LIBS_PATH=' + '/work/msan')

  command = (
      ['docker', 'run', '--rm', '-i', '--cap-add', 'SYS_PTRACE'] +
      _env_to_docker_args(env))
  if args.source_path:
    command += [
        '-v',
        '%s:/src/%s' % (_get_absolute_path(args.source_path), args.project_name)
    ]
  command += [
      '-v', '%s:/out' % project_out_dir,
      '-v', '%s:/work' % project_work_dir,
      '-t', 'gcr.io/oss-fuzz/%s' % project_name
  ]

  print('Running:', _get_command_string(command))

  try:<commit_msg>helper: Better support for building from local source (#1589). (#1595)<commit_after>#!/usr/bin/env python
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

from __future__ import print_function
from multiprocessing.dummy import Pool as ThreadPool
import argparse
import datetime
import errno
import multiprocessing
import os
import pipes
import re
import subprocess
import sys
import tempfile
import templates

OSSFUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
BUILD_DIR = os.path.join(OSSFUZZ_DIR, 'build')

BASE_IMAGES = [
    'gcr.io/oss-fuzz-base/base-image',
    'gcr.io/oss-fuzz-base/base-clang',
    'gcr.io/oss-fuzz-base/base-builder',
    'gcr.io/oss-fuzz-base/base-runner',
    'gcr.io/oss-fuzz-base/base-runner-debug',
    'gcr.io/oss-fuzz-base/base-msan-builder',
    'gcr.io/oss-fuzz-base/msan-builder',
]

VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$')
MAX_PROJECT_NAME_LENGTH = 26

if sys.version_info[0] >= 3:
    raw_input = input

CORPUS_URL_FORMAT = (
    'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/'
    '{fuzz_target}/')
CORPUS_BACKUP_URL_FORMAT = (
    'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/'
    'libFuzzer/{fuzz_target}/')


def main():
  os.chdir(OSSFUZZ_DIR)
  if not os.path.exists(BUILD_DIR):
    os.mkdir(BUILD_DIR)

  parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
  subparsers = parser.add_subparsers(dest='command')

  generate_parser = subparsers.add_parser(
      'generate', help='Generate files for new project.')
  generate_parser.add_argument('project_name')

  build_image_parser = subparsers.add_parser(
      'build_image', help='Build an image.')
  build_image_parser.add_argument('project_name')
  build_image_parser.add_argument('--pull', action='store_true',
                                  help='Pull latest base image.')
  build_image_parser.add_argument('--no-pull', action='store_true',
                                  help='Do not pull latest base image.')

  build_fuzzers_parser = subparsers.add_parser(
      'build_fuzzers', help='Build fuzzers for a project.')
  _add_engine_args(build_fuzzers_parser)
  _add_sanitizer_args(build_fuzzers_parser)
  _add_environment_args(build_fuzzers_parser)
  build_fuzzers_parser.add_argument('project_name')
  build_fuzzers_parser.add_argument('source_path', help='path of local source',
                                    nargs='?')
  build_fuzzers_parser.add_argument('--clean', dest='clean',
                                    action='store_true',
                                    help='clean existing artifacts.')
  build_fuzzers_parser.add_argument('--no-clean', dest='clean',
                                    action='store_false',
                                    help='do not clean existing artifacts '
                                    '(default).')
  build_fuzzers_parser.set_defaults(clean=False)

  check_build_parser = subparsers.add_parser(
      'check_build', help='Checks that fuzzers execute without errors.')
  _add_engine_args(check_build_parser)
  _add_sanitizer_args(check_build_parser)
  _add_environment_args(check_build_parser)
  check_build_parser.add_argument('project_name', help='name of the project')
  check_build_parser.add_argument('fuzzer_name', help='name of the fuzzer',
                                  nargs='?')

  run_fuzzer_parser = subparsers.add_parser(
      'run_fuzzer', help='Run a fuzzer.')
  _add_engine_args(run_fuzzer_parser)
  _add_environment_args(run_fuzzer_parser)
  run_fuzzer_parser.add_argument('project_name', help='name of the project')
  run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  run_fuzzer_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                                 nargs=argparse.REMAINDER)

  coverage_parser = subparsers.add_parser(
      'coverage', help='Run a fuzzer for a while and generate coverage.')
  coverage_parser.add_argument('--run_time', default=60,
                      help='time in seconds to run fuzzer')
  coverage_parser.add_argument('project_name', help='name of the project')
  coverage_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  coverage_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                               nargs=argparse.REMAINDER)

  profile_parser = subparsers.add_parser(
      'profile', help='Generate code coverage report for the project.')
  profile_parser.add_argument('project_name', help='name of the project')
  profile_parser.add_argument('--no-corpus-download', action='store_true',
                              help='do not download corpus backup from OSS-Fuzz; '
                              'use corpus located in build/corpus/<project>/<fuzz_target>/')
  profile_parser.add_argument('--port', default='8008', help='specify port for '
                              'a local HTTP server rendering coverage report')
  profile_parser.add_argument('--fuzz-target', help='specify name of a fuzz '
                              'target to be run for generating coverage report')
  profile_parser.add_argument('--corpus-dir', help='specify location of corpus '
                              'to be used (requires --fuzz-target argument)')

  reproduce_parser = subparsers.add_parser(
      'reproduce', help='Reproduce a crash.')
  reproduce_parser.add_argument('--valgrind', action='store_true',
                       help='run with valgrind')
  reproduce_parser.add_argument('project_name', help='name of the project')
  reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  reproduce_parser.add_argument('testcase_path', help='path of local testcase')
  reproduce_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                                nargs=argparse.REMAINDER)
  _add_environment_args(reproduce_parser)

  shell_parser = subparsers.add_parser(
      'shell', help='Run /bin/bash in an image.')
  shell_parser.add_argument('project_name', help='name of the project')
  _add_engine_args(shell_parser)
  _add_sanitizer_args(shell_parser)
  _add_environment_args(shell_parser)

  pull_images_parser = subparsers.add_parser('pull_images',
                                             help='Pull base images.')

  args = parser.parse_args()
  if args.command == 'generate':
    return generate(args)
  elif args.command == 'build_image':
    return build_image(args)
  elif args.command == 'build_fuzzers':
    return build_fuzzers(args)
  elif args.command == 'check_build':
    return check_build(args)
  elif args.command == 'run_fuzzer':
    return run_fuzzer(args)
  elif args.command == 'coverage':
    return coverage(args)
  elif args.command == 'profile':
    return profile(args)
  elif args.command == 'reproduce':
    return reproduce(args)
  elif args.command == 'shell':
    return shell(args)
  elif args.command == 'pull_images':
    return pull_images(args)

  return 0


def _is_base_image(image_name):
  """Checks if the image name is a base image."""
  return os.path.exists(os.path.join('infra', 'base-images', image_name))


def _check_project_exists(project_name):
  """Checks if a project exists."""
  if not os.path.exists(_get_project_dir(project_name)):
    print(project_name, 'does not exist', file=sys.stderr)
    return False

  return True


def _check_fuzzer_exists(project_name, fuzzer_name):
  """Checks if a fuzzer exists."""
  command = ['docker', 'run', '--rm']
  command.extend(['-v', '%s:/out' % _get_output_dir(project_name)])
  command.append('ubuntu:16.04')

  command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print(fuzzer_name,
          'does not seem to exist. Please run build_fuzzers first.',
          file=sys.stderr)
    return False

  return True


def _get_absolute_path(path):
  """Returns absolute path with user expansion."""
  return os.path.abspath(os.path.expanduser(path))


def _get_command_string(command):
  """Returns a shell escaped command string."""
  return ' '.join(pipes.quote(part) for part in command)


def _get_project_dir(project_name):
  """Returns path to the project."""
  return os.path.join(OSSFUZZ_DIR, 'projects', project_name)


def _get_dockerfile_path(project_name):
  """Returns path to the project Dockerfile."""
  return os.path.join(_get_project_dir(project_name), 'Dockerfile')


def _get_corpus_dir(project_name=''):
  """Returns path to /corpus directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'corpus', project_name)


def _get_output_dir(project_name=''):
  """Returns path to /out directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'out', project_name)


def _get_work_dir(project_name=''):
  """Returns path to /work directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'work', project_name)


def _add_engine_args(parser):
  """Add common engine args."""
  parser.add_argument('--engine', default='libfuzzer',
                      choices=['libfuzzer', 'afl', 'honggfuzz', 'none'])


def _add_sanitizer_args(parser):
  """Add common sanitizer args."""
  parser.add_argument('--sanitizer', default='address',
                      choices=['address', 'memory', 'undefined', 'coverage', 'profile'])


def _add_environment_args(parser):
  """Add common environment args."""
  parser.add_argument('-e', action='append',
                      help="set environment variable e.g. VAR=value")


def _build_image(image_name, no_cache=False, pull=False):
  """Build image."""

  is_base_image = _is_base_image(image_name)
  if is_base_image:
    image_project = 'oss-fuzz-base'
    dockerfile_dir = os.path.join('infra', 'base-images', image_name)
  else:
    image_project = 'oss-fuzz'
    if not _check_project_exists(image_name):
      return False

    dockerfile_dir = os.path.join('projects', image_name)

  build_args = []
  if no_cache:
    build_args.append('--no-cache')

  build_args += ['-t', 'gcr.io/%s/%s' % (image_project, image_name), dockerfile_dir]

  return docker_build(build_args, pull=pull)


def _env_to_docker_args(env_list):
  """Turn envirnoment variable list into docker arguments."""
  return sum([['-e', v] for v in env_list], [])


def _workdir_from_dockerfile(dockerfile_path):
  """Parse WORKDIR from the Dockerfile."""
  WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')

  with open(dockerfile_path) as f:
    lines = f.readlines()

  for line in reversed(lines):  # reversed to get last WORKDIR.
    match = re.match(WORKDIR_REGEX, line)
    if match:
      workdir = match.group(1)
      workdir = workdir.replace('$SRC', '/src')

      if not os.path.isabs(workdir):
        workdir = os.path.join('/src', workdir)

      return workdir

  return os.path.join('/src', project_name)


def docker_run(run_args, print_output=True):
  """Call `docker run`."""
  command = ['docker', 'run', '--rm', '-i', '--privileged']
  command.extend(run_args)

  print('Running:', _get_command_string(command))
  stdout = None
  if not print_output:
    stdout = open(os.devnull, 'w')

  try:
    subprocess.check_call(command, stdout=stdout, stderr=subprocess.STDOUT)
  except subprocess.CalledProcessError as e:
    return e.returncode

  return 0


def docker_build(build_args, pull=False):
  """Call `docker build`."""
  command = ['docker', 'build']
  if pull:
    command.append('--pull')

  command.extend(build_args)
  print('Running:', _get_command_string(command))

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print('docker build failed.', file=sys.stderr)
    return False

  return True


def docker_pull(image, pull=False):
  """Call `docker pull`."""
  command = ['docker', 'pull', image]
  print('Running:', _get_command_string(command))

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print('docker pull failed.', file=sys.stderr)
    return False

  return True


def build_image(args):
  """Build docker image."""
  if args.pull and args.no_pull:
    print('Incompatible arguments --pull and --no-pull.')
    return 1

  if args.pull:
    pull = True
  elif args.no_pull:
    pull = False
  else:
    y_or_n = raw_input('Pull latest base images (compiler/runtime)? (y/N): ')
    pull = y_or_n.lower() == 'y'

  if pull:
    print('Pulling latest base images...')
  else:
    print('Using cached base images...')

  # If build_image is called explicitly, don't use cache.
  if _build_image(args.project_name, no_cache=True, pull=pull):
    return 0

  return 1


def build_fuzzers(args):
  """Build fuzzers."""
  project_name = args.project_name
  if not _build_image(args.project_name):
    return 1

  project_out_dir = _get_output_dir(project_name)
  if args.clean:
    print('Cleaning existing build artifacts.')

    # Clean old and possibly conflicting artifacts in project's out directory.
    docker_run([
        '-v', '%s:/out' % project_out_dir,
        '-t', 'gcr.io/oss-fuzz/%s' % project_name,
        '/bin/bash', '-c', 'rm -rf /out/*'
    ])
  else:
    print('Keeping existing build artifacts as-is (if any).')

  env = [
      'FUZZING_ENGINE=' + args.engine,
      'SANITIZER=' + args.sanitizer
  ]
  if args.e:
    env += args.e

  project_work_dir = _get_work_dir(project_name)

  # Copy instrumented libraries.
  if args.sanitizer == 'memory':
    docker_run([
        '-v', '%s:/work' % project_work_dir,
        'gcr.io/oss-fuzz-base/msan-builder',
        'bash', '-c', 'cp -r /msan /work'])
    env.append('MSAN_LIBS_PATH=' + '/work/msan')

  command = (
      ['docker', 'run', '--rm', '-i', '--cap-add', 'SYS_PTRACE'] +
      _env_to_docker_args(env))
  if args.source_path:
    command += [
        '-v',
        '%s:%s' % (_get_absolute_path(args.source_path),
                   _workdir_from_dockerfile(
                       _get_dockerfile_path(args.project_name))),
    ]
  command += [
      '-v', '%s:/out' % project_out_dir,
      '-v', '%s:/work' % project_work_dir,
      '-t', 'gcr.io/oss-fuzz/%s' % project_name
  ]

  print('Running:', _get_command_string(command))

  try:
{'fraction_non_alphanumeric': 0.1341314864441583, 'fraction_numerical': 0.0024014199700692586, 'mean_word_length': 3.524326877657062, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7100435', 'n_tokens_mistral': 9534, 'n_tokens_neox': 9214, 'n_words': 2044}