<commit_before>"""
Command-line entry point for the Craftr build system.
"""

import argparse
import contextlib
import functools
import json
import os
import platform
import posixpath
import shlex
import shutil
import subprocess
import sys
import toml

import craftr from 'craftr'
import {concat} from 'craftr/utils/it'
import {plural, reindent, ReindentHelpFormatter} from 'craftr/utils/text'

error = functools.partial(print, file=sys.stderr)


parser = argparse.ArgumentParser(
  formatter_class=ReindentHelpFormatter,
  prog='craftr',
  description='''
    Craftr is a modular language-indepenent build system that is written in
    Python. It's core features are cross-platform compatibility, easy
    extensibility and Python as powerful build scripting language.
  ''',
)

parser.add_argument(
  '--quickstart',
  metavar='EXAMPLE',
  nargs='?',
  help='Copy one of the examples to the current working directory. Use '
       '--list-quickstart to show all available quickstart examples.'
)

parser.add_argument(
  '--list-quickstart',
  action='store_true',
  help='Show a list of all available quickstart examples and exit.'
)

parser.add_argument(
  '--list-tools',
  action='store_true',
  help='List tools available with the --tool option and exit.'
)

parser.add_argument(
  '-t', '--tool',
  metavar='NAME [ARGS ...]',
  nargs='...',
  help='Run a script from the craftr/tools directory.'
)

parser.add_argument(
  '--show-config-tags',
  action='store_true',
  help='Show the tags associated with the configuration and exit. They are '
       'the properties that you can use in the Craftr configuration file '
       'for conditional options.'
)

parser.add_argument(
  '--show-build-directory',
  action='store_true',
  help='Show the final build directory (including release and backend name) '
       'and exit.'
)

parser.add_argument(
  '--show-config',
  action='store_true',
  help='Print the configuration values in TOML format.'
)

parser.add_argument(
  '--recursive',
  action='store_true',
  help='Clean targets recursively in the --clean step.'
)

parser.add_argument(
  '--dry',
  action='store_true',
  help='In the --configure step, do not save any build configuration.'
)

parser.add_argument(
  '--release',
  action='store_true',
  help='Configure a release build. This option will set the `craftr.release` '
       'configuration value and also the `is_release` member of the Craftr '
       'core module.'
)

parser.add_argument(
  '--build-root',
  metavar='DIRECTORY',
  help='The build root directory. If not specified, defaults to the value '
       'of the `build.directory` option. If that is not specified, it '
       'defaults to build/ on --configure, otherwise it will be determined '
       'from the directories in the current working directory.'
)

parser.add_argument(
  '--build-directory',
  metavar='DIRECTORY',
  help='The build directory. If specified, overwrites what would be generated '
       'with --build-root. Defaults to <--build-root>/<debug/release>.'
)

parser.add_argument(
  '--cwd',
  metavar='DIRECTORY',
  help='Switch to the specified directory before executing. Note that any '
       'paths specified on the command-line will be relative to that '
       'directory.'
)

parser.add_argument(
  '--backend',
  help='The backend to use for building. The last (explicitly) used backend '
       'is remembered when using the --prepare-build or --build options. If '
       'this option is not defined, it is read from the `build.backend` '
       'configuration value. Defaults to `ninja`.'
)

parser.add_argument(
  '--save-cache',
  action='store_true',
  help='Save the cache (which includes the specified --options) to the build '
       'cache file. This option is implied by the --configure step.'
)

parser.add_argument(
  '--config',
  help='Specify a TOML configuration file to load. Note that any values '
       'specified with --options take precedence over the configuration '
       'file. Defaults to BUILD.cr.toml'
)

parser.add_argument(
  '--options',
  metavar='KEY=VALUE',
  nargs='+',
  default=[],
  action='append',
  help='Define one or more options that override the Craftr configuration '
       'file. This command-line option can be used multiple times.'
)

parser.add_argument(
  '--flush',
  action='store_true',
  help='Flush existing build configuration files. Use this option if you '
       'want to run the --configure step again without taking into options '
       'saved from a previous run.'
)

parser.add_argument(
  '-c', '--configure',
  metavar='BUILDSCRIPT',
  nargs='?',
  default=NotImplemented,
  help='Execute the build script and generate a JSON database file that '
       'contains all the build information.'
)

parser.add_argument(
  '--prepare-build',
  action='store_true',
  help='Prepare the build process by generating all files for the selected '
       'build backend, but not actually execute the build. Use this option if '
       'you want to invoke the build backend manually instead of via the '
       '--build step.'
)

parser.add_argument(
  '-b', '--build',
  metavar='TARGET',
  nargs='*',
  default=NotImplemented,
  help='Execute the build for all or the specified TARGETs using the build '
       'backend configured with --backend or in the Craftr configuration '
       'file. This step implies the --prepare-build step. Additional '
       'arguments can be passed to a target using the TARGET="ARGS ..." '
       'syntax (useful for run targets).'
)

parser.add_argument(
  '--run-node',
  metavar='NAME[^HASH]',
  help='Execute the command for the specified build-node. Build nodes names '
       'are formatted as `//<cell>:<target>#<node>`. This can be used '
       'internally by build backends to implement deduplication. This option '
       'requires the --build-directory to be set.'
)

parser.add_argument(
  '--show-node',
  metavar='NAME',
  help='Show the information for the specified build node.'
)

parser.add_argument(
  '--list-nodes',
  action='store_true',
  help='List all build nodes and exit.'
)

parser.add_argument(
  '--clean',
  metavar='TARGET',
  nargs='*',
  default=NotImplemented,
  help='Clean all or the specified TARGETs. Use the --recursive option if '
       'you want the specified targets to be cleaned recursively.'
)

parser.add_argument(
  '--dotviz',
  metavar='FILE',
  nargs='?',
  default=NotImplemented,
  help='Render a .dot graph file to the specified FILE or stdout and exit.'
)

parser.add_argument(
  '--build-args',
  action='append',
  default=[],
  help='Additional arguments for the --build step that are passed on to the '
       'build backend. The argument(s) to this option must be a string that '
       'will itself be treated as a list of arguments.'
)

parser.add_argument(
  '--clean-args',
  action='append',
  default=[],
  help='Additional arguments for the --clean step that are passed on to the '
       'build backend. The argument(s) to this option must be a string that '
       'will itself be treated as a list of arguments.'
)


def get_platform_tags():
  name = sys.platform.lower()
  system = platform.system().lower()
  tags = set()
  if name.startswith('win32'):
    if 'cygwin' in system:
      tags.add('cygwin')
    tags.add('win32')
  elif name.startswith('darwin'):
    tags.add('posix')
    tags.add('darwin')
  elif name.startswith('linux'):
    tags.add('posix')
    tags.add('linux')
    with open('/proc/version') as fp:
      if 'microsoft' in fp.read().lower():
        tags.add('wsl')
        tags.remove('linux')
  return tags


def mark_build_root(build_root):
  os.makedirs(build_root, exist_ok=True)
  open(os.path.join(build_root, '.craftr_build_root'), 'w').close()


def find_build_root():
  results = []
  for name in os.listdir('.'):
    if os.path.isfile(os.path.join(name, '.craftr_build_root')):
      results.append(name)
  if len(results) > 1:
    error('fatal: multiple candidates found for --build-root')
    error('       candidates are')
    for name in results:
      print('  --build-root', name)  # TODO shell quote
    sys.exit(1)
  if not results:
    return None
  return results[0]


def set_options(options):
  options = list(options)
  no_such_options = set()
  invalid_options = set()
  for option in options:
    key, sep, value = option.partition('=')
    if not sep:
      value = True
    elif not value:
      try:
        craftr.options.pop(key)
      except KeyError:
        no_such_options.add(key)
      continue
    else:
      if value.lower() in ('', 'true', '1', 'on', 'yes'):
        value = True
      elif value.lower() in ('false', '0', 'off', 'no'):
        value = False
    try:
      craftr.options[key] = value
    except KeyError:
      invalid_options.add(key)
  return no_such_options, invalid_options


def merge_options(*opts):
  result = []
  for option in concat(opts):
    key, sep, value = option.partition('=')
    for other in result:
      if other == key or other.startswith(key + '='):
        break
    else:
      result.append(option)
  return result


def get_additional_args_for(node_name):
  """
  In the --build step, a `craftr_additional_args.json` file may be present
  which specifies additional arguments passed to a target via the command-line.
  This extracts the additional arguments for the specified *node_name*, falling
  back to the node's target if such an entry exists.
  """

  additional_args_file = os.path.join(craftr.build_directory, 'craftr_additional_args.json')
  if not os.path.isfile(additional_args_file):
    return []
  with open(additional_args_file, 'r') as fp:
    additional_args = json.load(fp)
  try:
    return shlex.split(additional_args[node_name])
  except KeyError:
    pass
  target_name = node_name.partition('#')[0]
  try:
    return shlex.split(additional_args[target_name])
  except KeyError:
    pass
  return []


def run_build_node(graph, node_name):
  if '^' in node_name:
    node_name, node_hash = node_name.split('^', 1)
  else:
    node_hash = None

  try:
    node = graph[node_name]
  except KeyError:
    error('fatal: build node "{}" does not exist'.format(node_name))
    return 1

  if node_hash is not None and node_hash != graph.hash(node):
    error('fatal: build node hash inconsistency, maybe try --prepare-build')
    return 1

  # TODO: The additional args feature should be explicitly supported by the
  #       build node, allowing it to specify a position where the additional
  #       args will be rendered.
  #       Usually, the option only makes sense for targets that run a single
  #       command such as cxx.run(), java.run(), etc.
  additional_args = get_additional_args_for(node_name)

  # Ensure that the output directories exist.
  created_dirs = set()
  for directory in (os.path.dirname(x) for x in node.output_files):
    if directory not in created_dirs:
      os.makedirs(directory, exist_ok=True)
      created_dirs.add(directory)

  # Update the environment and working directory.
  old_env = os.environ.copy()
  os.environ.update(node.environ or {})
  if node.cwd:
    os.chdir(node.cwd)

  # Used to print the command-list on failure.
  def print_command_list(current=-1):
    error('Command list:'.format(node.name))
    for i, cmd in enumerate(node.commands):
      error('>' if current == i else ' ', '$', ' '.join(map(shlex.quote , cmd)))

  # Execute the subcommands/
  for i, cmd in enumerate(node.commands):
    # Add the additional_args to the last command in the chain.
    if i == len(node.commands) - 1:
      cmd = cmd + additional_args
    try:
      code = subprocess.call(cmd)
    except FileNotFoundError as e:
      error(e)
      code = 127
    if code != 0:
      error('\n' + '-'*60)
      error('fatal: "{}" exited with code {}.'.format(node.name, code))
      print_command_list(i)
      error('-'*60 + '\n')
      return code

  # Check if all output files have been produced by the commands.
  missing_files = [x for x in node.output_files if not os.path.exists(x)]
  if missing_files:
    error('\n' + '-'*60)
    error('fatal: "{}" produced only {} of {} listed output files.'.format(node.name,
        len(node.output_files) - len(missing_files), len(node.output_files)))
    error('The missing files are:')
    for x in missing_files:
      print('  -', x)
    print_command_list()
    error('-'*60 + '\n')
    return 1

  return 0


def show_build_node(graph, node_name):
  try:
    node = graph[node_name]
  except KeyError:
    error('fatal: build node "{}" does not exist'.format(node_name))
    return 1
  data = node._asdict()
  data['hash'] = graph.hash(node)
  json.dump(data, sys.stdout, indent=2)
  print()
  return 0


def list_build_nodes(graph):
  for node in sorted(graph.nodes(), key=lambda x: x.name):
    print(node.name)
  return 0<commit_msg>--show-node supports relative names, for convenience<commit_after>"""
Command-line entry point for the Craftr build system.
"""

import argparse
import contextlib
import functools
import json
import os
import platform
import posixpath
import shlex
import shutil
import subprocess
import sys
import toml

import craftr from 'craftr'
import {concat} from 'craftr/utils/it'
import {plural, reindent, ReindentHelpFormatter} from 'craftr/utils/text'

error = functools.partial(print, file=sys.stderr)


parser = argparse.ArgumentParser(
  formatter_class=ReindentHelpFormatter,
  prog='craftr',
  description='''
    Craftr is a modular language-indepenent build system that is written in
    Python. It's core features are cross-platform compatibility, easy
    extensibility and Python as powerful build scripting language.
  ''',
)

parser.add_argument(
  '--quickstart',
  metavar='EXAMPLE',
  nargs='?',
  help='Copy one of the examples to the current working directory. Use '
       '--list-quickstart to show all available quickstart examples.'
)

parser.add_argument(
  '--list-quickstart',
  action='store_true',
  help='Show a list of all available quickstart examples and exit.'
)

parser.add_argument(
  '--list-tools',
  action='store_true',
  help='List tools available with the --tool option and exit.'
)

parser.add_argument(
  '-t', '--tool',
  metavar='NAME [ARGS ...]',
  nargs='...',
  help='Run a script from the craftr/tools directory.'
)

parser.add_argument(
  '--show-config-tags',
  action='store_true',
  help='Show the tags associated with the configuration and exit. They are '
       'the properties that you can use in the Craftr configuration file '
       'for conditional options.'
)

parser.add_argument(
  '--show-build-directory',
  action='store_true',
  help='Show the final build directory (including release and backend name) '
       'and exit.'
)

parser.add_argument(
  '--show-config',
  action='store_true',
  help='Print the configuration values in TOML format.'
)

parser.add_argument(
  '--recursive',
  action='store_true',
  help='Clean targets recursively in the --clean step.'
)

parser.add_argument(
  '--dry',
  action='store_true',
  help='In the --configure step, do not save any build configuration.'
)

parser.add_argument(
  '--release',
  action='store_true',
  help='Configure a release build. This option will set the `craftr.release` '
       'configuration value and also the `is_release` member of the Craftr '
       'core module.'
)

parser.add_argument(
  '--build-root',
  metavar='DIRECTORY',
  help='The build root directory. If not specified, defaults to the value '
       'of the `build.directory` option. If that is not specified, it '
       'defaults to build/ on --configure, otherwise it will be determined '
       'from the directories in the current working directory.'
)

parser.add_argument(
  '--build-directory',
  metavar='DIRECTORY',
  help='The build directory. If specified, overwrites what would be generated '
       'with --build-root. Defaults to <--build-root>/<debug/release>.'
)

parser.add_argument(
  '--cwd',
  metavar='DIRECTORY',
  help='Switch to the specified directory before executing. Note that any '
       'paths specified on the command-line will be relative to that '
       'directory.'
)

parser.add_argument(
  '--backend',
  help='The backend to use for building. The last (explicitly) used backend '
       'is remembered when using the --prepare-build or --build options. If '
       'this option is not defined, it is read from the `build.backend` '
       'configuration value. Defaults to `ninja`.'
)

parser.add_argument(
  '--save-cache',
  action='store_true',
  help='Save the cache (which includes the specified --options) to the build '
       'cache file. This option is implied by the --configure step.'
)

parser.add_argument(
  '--config',
  help='Specify a TOML configuration file to load. Note that any values '
       'specified with --options take precedence over the configuration '
       'file. Defaults to BUILD.cr.toml'
)

parser.add_argument(
  '--options',
  metavar='KEY=VALUE',
  nargs='+',
  default=[],
  action='append',
  help='Define one or more options that override the Craftr configuration '
       'file. This command-line option can be used multiple times.'
)

parser.add_argument(
  '--flush',
  action='store_true',
  help='Flush existing build configuration files. Use this option if you '
       'want to run the --configure step again without taking into options '
       'saved from a previous run.'
)

parser.add_argument(
  '-c', '--configure',
  metavar='BUILDSCRIPT',
  nargs='?',
  default=NotImplemented,
  help='Execute the build script and generate a JSON database file that '
       'contains all the build information.'
)

parser.add_argument(
  '--prepare-build',
  action='store_true',
  help='Prepare the build process by generating all files for the selected '
       'build backend, but not actually execute the build. Use this option if '
       'you want to invoke the build backend manually instead of via the '
       '--build step.'
)

parser.add_argument(
  '-b', '--build',
  metavar='TARGET',
  nargs='*',
  default=NotImplemented,
  help='Execute the build for all or the specified TARGETs using the build '
       'backend configured with --backend or in the Craftr configuration '
       'file. This step implies the --prepare-build step. Additional '
       'arguments can be passed to a target using the TARGET="ARGS ..." '
       'syntax (useful for run targets).'
)

parser.add_argument(
  '--run-node',
  metavar='NAME[^HASH]',
  help='Execute the command for the specified build-node. Build nodes names '
       'are formatted as `//<cell>:<target>#<node>`. This can be used '
       'internally by build backends to implement deduplication. This option '
       'requires the --build-directory to be set.'
)

parser.add_argument(
  '--show-node',
  metavar='NAME',
  help='Show the information for the specified build node.'
)

parser.add_argument(
  '--list-nodes',
  action='store_true',
  help='List all build nodes and exit.'
)

parser.add_argument(
  '--clean',
  metavar='TARGET',
  nargs='*',
  default=NotImplemented,
  help='Clean all or the specified TARGETs. Use the --recursive option if '
       'you want the specified targets to be cleaned recursively.'
)

parser.add_argument(
  '--dotviz',
  metavar='FILE',
  nargs='?',
  default=NotImplemented,
  help='Render a .dot graph file to the specified FILE or stdout and exit.'
)

parser.add_argument(
  '--build-args',
  action='append',
  default=[],
  help='Additional arguments for the --build step that are passed on to the '
       'build backend. The argument(s) to this option must be a string that '
       'will itself be treated as a list of arguments.'
)

parser.add_argument(
  '--clean-args',
  action='append',
  default=[],
  help='Additional arguments for the --clean step that are passed on to the '
       'build backend. The argument(s) to this option must be a string that '
       'will itself be treated as a list of arguments.'
)


def get_platform_tags():
  name = sys.platform.lower()
  system = platform.system().lower()
  tags = set()
  if name.startswith('win32'):
    if 'cygwin' in system:
      tags.add('cygwin')
    tags.add('win32')
  elif name.startswith('darwin'):
    tags.add('posix')
    tags.add('darwin')
  elif name.startswith('linux'):
    tags.add('posix')
    tags.add('linux')
    with open('/proc/version') as fp:
      if 'microsoft' in fp.read().lower():
        tags.add('wsl')
        tags.remove('linux')
  return tags


def mark_build_root(build_root):
  os.makedirs(build_root, exist_ok=True)
  open(os.path.join(build_root, '.craftr_build_root'), 'w').close()


def find_build_root():
  results = []
  for name in os.listdir('.'):
    if os.path.isfile(os.path.join(name, '.craftr_build_root')):
      results.append(name)
  if len(results) > 1:
    error('fatal: multiple candidates found for --build-root')
    error('       candidates are')
    for name in results:
      print('  --build-root', name)  # TODO shell quote
    sys.exit(1)
  if not results:
    return None
  return results[0]


def set_options(options):
  options = list(options)
  no_such_options = set()
  invalid_options = set()
  for option in options:
    key, sep, value = option.partition('=')
    if not sep:
      value = True
    elif not value:
      try:
        craftr.options.pop(key)
      except KeyError:
        no_such_options.add(key)
      continue
    else:
      if value.lower() in ('', 'true', '1', 'on', 'yes'):
        value = True
      elif value.lower() in ('false', '0', 'off', 'no'):
        value = False
    try:
      craftr.options[key] = value
    except KeyError:
      invalid_options.add(key)
  return no_such_options, invalid_options


def merge_options(*opts):
  result = []
  for option in concat(opts):
    key, sep, value = option.partition('=')
    for other in result:
      if other == key or other.startswith(key + '='):
        break
    else:
      result.append(option)
  return result


def get_additional_args_for(node_name):
  """
  In the --build step, a `craftr_additional_args.json` file may be present
  which specifies additional arguments passed to a target via the command-line.
  This extracts the additional arguments for the specified *node_name*, falling
  back to the node's target if such an entry exists.
  """

  additional_args_file = os.path.join(craftr.build_directory, 'craftr_additional_args.json')
  if not os.path.isfile(additional_args_file):
    return []
  with open(additional_args_file, 'r') as fp:
    additional_args = json.load(fp)
  try:
    return shlex.split(additional_args[node_name])
  except KeyError:
    pass
  target_name = node_name.partition('#')[0]
  try:
    return shlex.split(additional_args[target_name])
  except KeyError:
    pass
  return []


def run_build_node(graph, node_name):
  if '^' in node_name:
    node_name, node_hash = node_name.split('^', 1)
  else:
    node_hash = None
  if node_name.startswith(':'):
    node_name = '//' + craftr.cache['main_build_cell'] + node_name

  try:
    node = graph[node_name]
  except KeyError:
    error('fatal: build node "{}" does not exist'.format(node_name))
    return 1

  if node_hash is not None and node_hash != graph.hash(node):
    error('fatal: build node hash inconsistency, maybe try --prepare-build')
    return 1

  # TODO: The additional args feature should be explicitly supported by the
  #       build node, allowing it to specify a position where the additional
  #       args will be rendered.
  #       Usually, the option only makes sense for targets that run a single
  #       command such as cxx.run(), java.run(), etc.
  additional_args = get_additional_args_for(node_name)

  # Ensure that the output directories exist.
  created_dirs = set()
  for directory in (os.path.dirname(x) for x in node.output_files):
    if directory not in created_dirs:
      os.makedirs(directory, exist_ok=True)
      created_dirs.add(directory)

  # Update the environment and working directory.
  old_env = os.environ.copy()
  os.environ.update(node.environ or {})
  if node.cwd:
    os.chdir(node.cwd)

  # Used to print the command-list on failure.
  def print_command_list(current=-1):
    error('Command list:'.format(node.name))
    for i, cmd in enumerate(node.commands):
      error('>' if current == i else ' ', '$', ' '.join(map(shlex.quote , cmd)))

  # Execute the subcommands/
  for i, cmd in enumerate(node.commands):
    # Add the additional_args to the last command in the chain.
    if i == len(node.commands) - 1:
      cmd = cmd + additional_args
    try:
      code = subprocess.call(cmd)
    except FileNotFoundError as e:
      error(e)
      code = 127
    if code != 0:
      error('\n' + '-'*60)
      error('fatal: "{}" exited with code {}.'.format(node.name, code))
      print_command_list(i)
      error('-'*60 + '\n')
      return code

  # Check if all output files have been produced by the commands.
  missing_files = [x for x in node.output_files if not os.path.exists(x)]
  if missing_files:
    error('\n' + '-'*60)
    error('fatal: "{}" produced only {} of {} listed output files.'.format(node.name,
        len(node.output_files) - len(missing_files), len(node.output_files)))
    error('The missing files are:')
    for x in missing_files:
      print('  -', x)
    print_command_list()
    error('-'*60 + '\n')
    return 1

  return 0


def show_build_node(graph, node_name):
  if node_name.startswith(':'):
    node_name = '//' + craftr.cache['main_build_cell'] + node_name
  try:
    node = graph[node_name]
  except KeyError:
    error('fatal: build node "{}" does not exist'.format(node_name))
    return 1
  data = node._asdict()
  data['hash'] = graph.hash(node)
  json.dump(data, sys.stdout, indent=2)
  print()
  return 0


def list_build_nodes(graph):
  for node in sorted(graph.nodes(), key=lambda x: x.name):
    print(node.name)
  return 0
{'fraction_non_alphanumeric': 0.12059951202509585, 'fraction_numerical': 0.0025560590217265015, 'mean_word_length': 3.3245687489532743, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '703991', 'n_tokens_mistral': 8132, 'n_tokens_neox': 7945, 'n_words': 2580}