<commit_before>#!/usr/bin/python
# Copyright 2016 The Tulsi Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bridge between Xcode and Bazel for the "build" action.

NOTE: This script must be executed in the same directory as the Xcode project's
main group in order to generate correct debug symbols.
"""

import collections
import json
import os
import re
import shutil
import stat
import StringIO
import subprocess
import sys
import tempfile
import textwrap
import time
import zipfile
import tulsi_logging


def _PrintXcodeWarning(msg):
  sys.stdout.write(':: warning: %s\n' % msg)
  sys.stdout.flush()


def _PrintXcodeError(msg):
  sys.stderr.write(':: error: %s\n' % msg)
  sys.stderr.flush()


class Timer(object):
  """Simple profiler."""

  def __init__(self, action_name, action_id):
    """Creates a new Timer object.

    Args:
      action_name: A human-readable action name, shown in the build log.
      action_id: A machine-readable action identifier, can be used for metrics.

    Returns:
      A Timer instance.
    """
    self.action_name = action_name
    self.action_id = action_id
    self._start = None

  def Start(self):
    self._start = time.time()
    return self

  def End(self):
    end = time.time()
    seconds = end - self._start
    tulsi_logging.Logger().log_action(self.action_name, self.action_id, seconds)


class CodesignBundleAttributes(object):
  """Wrapper class for codesigning attributes of a signed bundle."""

  # List of codesigning attributes that this script requires.
  _ATTRIBUTES = ['Authority', 'Identifier', 'TeamIdentifier']

  def __init__(self, codesign_output):
    self.attributes = {}

    pending_attributes = list(self._ATTRIBUTES)
    for line in codesign_output.split('\n'):
      if not pending_attributes:
        break

      for attribute in pending_attributes:
        if line.startswith(attribute):
          value = line[len(attribute) + 1:]
          self.attributes[attribute] = value
          pending_attributes.remove(attribute)
          break

    for attribute in self._ATTRIBUTES:
      if attribute not in self.attributes:
        _PrintXcodeError(
            'Failed to extract %s from %s.\n' % (attribute, codesign_output))

  def Get(self, attribute):
    """Returns the value for the given attribute, or None if it wasn't found."""
    value = self.attributes.get(attribute)
    if attribute not in self._ATTRIBUTES:
      _PrintXcodeError(
          'Attribute %s not declared to be parsed. ' % attribute +
          'Available attributes are %s.\n' % self._ATTRIBUTES)
    return value


class _OptionsParser(object):
  """Handles parsing script options."""

  # Key for options that should be applied to all build configurations.
  ALL_CONFIGS = '__all__'

  # The build configurations handled by this parser.
  KNOWN_CONFIGS = ['Debug', 'Release', 'Fastbuild']

  def __init__(self, sdk_version, platform_name, arch, main_group_path):
    self.targets = []
    self.startup_options = collections.defaultdict(list)
    self.build_options = collections.defaultdict(
        list,
        {
            _OptionsParser.ALL_CONFIGS: [
                '--experimental_enable_objc_cc_deps',
                '--verbose_failures',
                '--announce_rc',
            ],

            'Debug': [
                '--compilation_mode=dbg',
            ],

            'Release': [
                '--compilation_mode=opt',
                '--strip=always',
            ],

            'Fastbuild': [
                '--compilation_mode=fastbuild',
            ],
        })

    # Options specific to debugger integration in Xcode.
    xcode_version_major = int(os.environ['XCODE_VERSION_MAJOR'])
    if xcode_version_major < 800:
      xcode_lldb_options = [
          '--copt=-Xclang', '--copt=-fdebug-compilation-dir',
          '--copt=-Xclang', '--copt=%s' % main_group_path,
          '--objccopt=-Xclang', '--objccopt=-fdebug-compilation-dir',
          '--objccopt=-Xclang', '--objccopt=%s' % main_group_path,
      ]
      self.build_options['Debug'].extend(xcode_lldb_options)
      self.build_options['Release'].extend(xcode_lldb_options)

    self.sdk_version = sdk_version
    self.platform_name = platform_name

    if self.platform_name.startswith('watch'):
      config_platform = 'watchos'
    elif self.platform_name.startswith('iphone'):
      config_platform = 'ios'
    elif self.platform_name.startswith('appletv'):
      config_platform = 'tvos'
    else:
      self._WarnUnknownPlatform()
      config_platform = 'ios'
    self.build_options[_OptionsParser.ALL_CONFIGS].append(
        '--config=%s_%s' % (config_platform, arch))

    self.verbose = 0
    self.install_generated_artifacts = False
    self.bazel_bin_path = 'bazel-bin'
    self.bazel_package_path = None
    self.bazel_executable = None

  @staticmethod
  def _UsageMessage():
    """Returns a usage message string."""
    usage = textwrap.dedent("""\
      Usage: %s <target> [<target2> ...] --bazel <bazel_binary_path> [options]

      Where options are:
        --verbose [-v]
            Increments the verbosity of the script by one level. This argument
            may be provided multiple times to enable additional output levels.

        --unpack_generated_ipa
            Unzips the contents of the IPA artifact generated by this build.

        --bazel_startup_options <option1> [<option2> ...] --
            Provides one or more Bazel startup options.

        --bazel_options <option1> [<option2> ...] --
            Provides one or more Bazel build options.

        --bazel_bin_path <path>
            Path at which Bazel-generated artifacts may be retrieved.

        --bazel_package_path <path>
            The value of the package_path variable in Bazel workspace.
      """ % sys.argv[0])

    usage += '\n' + textwrap.fill(
        'Note that the --bazel_startup_options and --bazel_options options may '
        'include an optional configuration specifier in brackets to limit '
        'their contents to a given build configuration. Options provided with '
        'no configuration filter will apply to all configurations in addition '
        'to any configuration-specific options.', 120)

    usage += '\n' + textwrap.fill(
        'E.g., --bazel_options common --  --bazel_options[Release] release -- '
        'would result in "bazel build common release" in the "Release" '
        'configuration and "bazel build common" in all other configurations.',
        120)

    return usage

  def ParseOptions(self, args):
    """Parses arguments, returning (message, exit_code)."""

    bazel_executable_index = args.index('--bazel')

    self.targets = args[:bazel_executable_index]
    if not self.targets or len(args) < bazel_executable_index + 2:
      return (self._UsageMessage(), 10)
    self.bazel_executable = args[bazel_executable_index + 1]

    return self._ParseVariableOptions(args[bazel_executable_index + 2:])

  def GetStartupOptions(self, config):
    """Returns the full set of startup options for the given config."""
    return self._GetOptions(self.startup_options, config)

  def GetBuildOptions(self, config):
    """Returns the full set of build options for the given config."""
    options = self._GetOptions(self.build_options, config)

    version_string = self._GetXcodeVersionString()
    if version_string:
      self._AddDefaultOption(options, '--xcode_version', version_string)

    if self.sdk_version:
      if self.platform_name.startswith('watch'):
        self._AddDefaultOption(options,
                               '--watchos_sdk_version',
                               self.sdk_version)
      elif self.platform_name.startswith('iphone'):
        self._AddDefaultOption(options, '--ios_sdk_version', self.sdk_version)
      elif self.platform_name.startswith('appletv'):
        self._AddDefaultOption(options, '--tvos_sdk_version', self.sdk_version)
      else:
        self._WarnUnknownPlatform()
        self._AddDefaultOption(options, '--ios_sdk_version', self.sdk_version)
    return options

  @staticmethod
  def _AddDefaultOption(option_list, option, default_value):
    matching_options = [opt for opt in option_list if opt.startswith(option)]
    if matching_options:
      return option_list

    option_list.append('%s=%s' % (option, default_value))
    return option_list

  @staticmethod
  def _GetOptions(option_set, config):
    """Returns a flattened list from options_set for the given config."""
    options = list(option_set[_OptionsParser.ALL_CONFIGS])
    if config != _OptionsParser.ALL_CONFIGS:
      options.extend(option_set[config])
    return options

  def _WarnUnknownPlatform(self):
    sys.stdout.write('Warning: unknown platform "%s" will be treated as '
                     'iOS\n' % self.platform_name)
    sys.stdout.flush()

  def _ParseVariableOptions(self, args):
    """Parses flag-based args, returning (message, exit_code)."""

    verbose_re = re.compile('-(v+)$')

    while args:
      arg = args[0]
      args = args[1:]

      if arg == '--install_generated_artifacts':
        self.install_generated_artifacts = True

      elif arg.startswith('--bazel_startup_options'):
        config = self._ParseConfigFilter(arg)
        args, items, terminated = self._ParseDoubleDashDelimitedItems(args)
        if not terminated:
          return ('Missing "--" terminator while parsing %s' % arg, 2)
        duplicates = self._FindDuplicateOptions(self.startup_options,
                                                config,
                                                items)
        if duplicates:
          return (
              '%s items conflict with common options: %s' % (
                  arg, ','.join(duplicates)),
              2)
        self.startup_options[config].extend(items)

      elif arg.startswith('--bazel_options'):
        config = self._ParseConfigFilter(arg)
        args, items, terminated = self._ParseDoubleDashDelimitedItems(args)
        if not terminated:
          return ('Missing "--" terminator while parsing %s' % arg, 2)
        duplicates = self._FindDuplicateOptions(self.build_options,
                                                config,
                                                items)
        if duplicates:
          return (
              '%s items conflict with common options: %s' % (
                  arg, ','.join(duplicates)),
              2)
        self.build_options[config].extend(items)

      elif arg == '--bazel_bin_path':
        if not args:
          return ('Missing required parameter for %s' % arg, 2)
        self.bazel_bin_path = args[0]
        args = args[1:]

      elif arg == '--bazel_package_path':
        if not args:
          return ('Missing required parameter for %s' % arg, 2)
        self.bazel_package_path = args[0]
        args = args[1:]

      elif arg == '--verbose':
        self.verbose += 1

      else:
        match = verbose_re.match(arg)
        if match:
          self.verbose += len(match.group(1))
        else:
          return ('Unknown option "%s"\n%s' % (arg, self._UsageMessage()), 1)

    return (None, 0)

  @staticmethod
  def _ParseConfigFilter(arg):
    match = re.search(r'\[([^\]]+)\]', arg)
    if not match:
      return _OptionsParser.ALL_CONFIGS
    return match.group(1)

  @staticmethod
  def _ConsumeArgumentForParam(param, args):
    if not args:
      return (None, 'Missing required parameter for "%s" option' % param)
    val = args[0]
    return (args[1:], val)

  @staticmethod
  def _ParseDoubleDashDelimitedItems(args):
    """Consumes options until -- is found."""
    options = []
    terminator_found = False

    opts = args
    while opts:
      opt = opts[0]
      opts = opts[1:]
      if opt == '--':
        terminator_found = True
        break
      options.append(opt)

    return opts, options, terminator_found

  @staticmethod
  def _FindDuplicateOptions(options_dict, config, new_options):
    """Returns a list of options appearing in both given option lists."""

    allowed_duplicates = [
        '--copt',
        '--config',
        '--define',
        '--objccopt',
    ]

    def ExtractOptionNames(opts):
      names = set()
      for opt in opts:
        split_opt = opt.split('=', 1)
        if split_opt[0] not in allowed_duplicates:
          names.add(split_opt[0])
      return names

    current_set = ExtractOptionNames(options_dict[config])
    new_set = ExtractOptionNames(new_options)
    conflicts = current_set.intersection(new_set)

    if config != _OptionsParser.ALL_CONFIGS:
      current_set = ExtractOptionNames(options_dict[_OptionsParser.ALL_CONFIGS])
      conflicts = conflicts.union(current_set.intersection(new_set))
    return conflicts

  @staticmethod
  def _GetXcodeVersionString():
    """Returns Xcode version info from the environment as a string."""
    reported_version = os.environ['XCODE_VERSION_ACTUAL']
    match = re.match(r'(\d{2})(\d)(\d)$', reported_version)
    if not match:
      sys.stdout.write('Warning: Failed to extract Xcode version from %s\n' % (
          reported_version))
      sys.stdout.flush()
      return None
    major_version = int(match.group(1))
    minor_version = int(match.group(2))
    fix_version = int(match.group(3))
    fix_version_string = ''
    if fix_version:
      fix_version_string = '.%d' % fix_version
    return '%d.%d%s' % (major_version, minor_version, fix_version_string)


class BazelBuildBridge(object):
  """Handles invoking Bazel and unpacking generated binaries."""

  def __init__(self):
    self.verbose = 0
    self.build_path = None
    self.bazel_bin_path = None
    # The actual path to the Bazel output directory (not a symlink)
    self.real_bazel_bin_path = None
    self.bazel_package_path = None
    # The path to the Bazel's sandbox source root.
    self.bazel_build_workspace_root = None
    self.bazel_genfiles_path = None
    self.bazel_symlink_prefix = None
    self.codesign_attributes = {}

    # Certain potentially expensive patchups need to be made for non-Xcode IDE
    # integrations. There isn't a fool-proof way of determining if the script is
    # being used with Xcode or not, but searching the CODESIGNING_FOLDER_PATH
    # env var for "/Xcode/" should catch the majority of use-cases.
    self.codesigning_folder_path = os.environ['CODESIGNING_FOLDER_PATH']
    self.likely_xcode = self.codesigning_folder_path.find('/Xcode/') != -1

    self.xcode_action = os.environ['ACTION']  # The Xcode build action.
    # When invoked as an external build system script, Xcode will set ACTION to
    # an empty string.
    if not self.xcode_action:
      self.xcode_action = 'build'

    self.generate_dsym = os.environ.get('TULSI_USE_DSYM', 'NO') == 'YES'
    self.use_dynamic_outputs = (
        os.environ.get('TULSI_USE_DYNAMIC_OUTPUTS', 'NO') == 'YES')

    # Target architecture.  Must be defined for correct setting of
    # the --config flag
    self.arch = os.environ.get('CURRENT_ARCH')
    if not self.arch:
      _PrintXcodeError('Tulsi requires env variable CURRENT_ARCH to be '
                       'set.  Please file a bug against Tulsi.')
      sys.exit(1)

    # Declared outputs of the target.
    self.bazel_outputs = os.environ.get('BAZEL_OUTPUTS', [])
    if self.bazel_outputs:
      self.bazel_outputs = self.bazel_outputs.split('\n')
    # Bazel's notion of the type of artifact being generated.
    self.bazel_target_type = os.environ.get('BAZEL_TARGET_TYPE')
    # Path into which generated artifacts should be copied.
    self.built_products_dir = os.environ['BUILT_PRODUCTS_DIR']
    # Whether or not code coverage information should be generated.
    self.code_coverage_enabled = (
        os.environ.get('CLANG_COVERAGE_MAPPING') == 'YES')
    # Path where Xcode expects generated sources to be placed.
    self.derived_sources_folder_path = os.environ.get('DERIVED_SOURCES_DIR')
    # Full name of the target artifact (e.g., "MyApp.app" or "Test.xctest").
    self.full_product_name = os.environ['FULL_PRODUCT_NAME']
    # Target SDK version.
    self.sdk_version = os.environ.get('SDK_VERSION')
    # TEST_HOST for unit tests.
    self.test_host_binary = os.environ.get('TEST_HOST')
    # Whether this target is a test or not.
    self.is_test = os.environ.get('WRAPPER_EXTENSION') == 'xctest'
    # UTI type of the target.
    self.package_type = os.environ.get('PACKAGE_TYPE')
    # Target platform.
    self.platform_name = os.environ['PLATFORM_NAME']
    # Type of the target artifact.
    self.product_type = os.environ['PRODUCT_TYPE']
    # Path to the parent of the xcodeproj bundle.
    self.project_dir = os.environ['PROJECT_DIR']
    # Path to the xcodeproj bundle.
    self.project_file_path = os.environ['PROJECT_FILE_PATH']
    # Path to the parent of the Xcode project's mainGroup.
    self.source_root = os.environ['SOURCE_ROOT']
    # Path to the directory containing the WORKSPACE file.
    self.workspace_root = os.path.abspath(os.environ['TULSI_WR'])
    # Set to the name of the generated bundle for bundle-type targets, None for
    # single file targets (like static libraries).
    self.wrapper_name = os.environ.get('WRAPPER_NAME')
    self.wrapper_suffix = os.environ.get('WRAPPER_SUFFIX', '')
    self.xcode_version_major = int(os.environ['XCODE_VERSION_MAJOR'])
    self.xcode_version_minor = int(os.environ['XCODE_VERSION_MINOR'])

    # Path where Xcode expects the artifacts to be written to. This is not the
    # codesigning_path as device vs simulator builds have different signing
    # requirements, so Xcode expects different things to be signed. This is
    # mostly apparent on XCUITests where simulator builds set the codesigning
    # path to be the .xctest bundle, but for device builds it is actually the
    # UI runner app (since it needs to be codesigned to run on the device.) The
    # contents folder path is a stable path on where to put the expected
    # artifacts. For static libraries (objc_library, swift_library),
    # CONTENTS_FOLDER_PATH does not exist, but the location where Xcode expects
    # the archive coincides with the TARGET_BUILD_DIR, so using an empty
    # default for CONTENTS_FOLDER_PATH supports both bundle and single artifact
    # outputs.
    # TODO(b/35811023): Check these paths are still valid.
    self.content_folder_path = os.path.join(
        os.environ['TARGET_BUILD_DIR'],
        os.environ.get('CONTENTS_FOLDER_PATH', ''))

    # Path to where Xcode expects the binary to be placed.
    self.binary_path = os.path.join(
        os.environ['TARGET_BUILD_DIR'], os.environ['EXECUTABLE_PATH'])

    self.is_simulator = self.platform_name.endswith('simulator')
    # Check to see if code signing actions should be skipped or not.
    if self.is_simulator:
      self.codesigning_allowed = False
    else:
      self.codesigning_allowed = os.environ.get('CODE_SIGNING_ALLOWED') == 'YES'

    self.post_processor_binary = os.path.join(self.project_file_path,
                                              '.tulsi',
                                              'Utils',
                                              'post_processor')
    if self.codesigning_allowed:
      self.runner_entitlements_template = os.path.join(self.project_file_path,
                                                       '.tulsi',
                                                       'Resources',
                                                       'XCTRunner.entitlements')

    self.main_group_path = os.getcwd()

  def Run(self, args):
    """Executes a Bazel build based on the environment and given arguments."""
    if self.xcode_action != 'build':
      sys.stderr.write('Xcode action is %s, ignoring.' % self.xcode_action)
      return 0

    parser = _OptionsParser(self.sdk_version,
                            self.platform_name,
                            self.arch,
                            self.main_group_path)
    timer = Timer('Parsing options', 'parsing_options').Start()
    message, exit_code = parser.ParseOptions(args[1:])
    timer.End()
    if exit_code:
      _PrintXcodeError('Option parsing failed: %s' % message)
      return exit_code

    self.verbose = parser.verbose
    self.bazel_bin_path = os.path.abspath(parser.bazel_bin_path)
    # bazel_bin_path is assumed to always end in "-bin".
    self.bazel_symlink_prefix = self.bazel_bin_path[:-3]
    self.bazel_genfiles_path = self.bazel_symlink_prefix + 'genfiles'

    self.build_path = os.path.join(self.bazel_bin_path,
                                   os.environ.get('TULSI_BUILD_PATH', ''))
    self.bazel_package_path = parser.bazel_package_path
    (command, retval) = self._BuildBazelCommand(parser)
    if retval:
      return retval

    timer = Timer('Running Bazel', 'running_bazel').Start()
    exit_code, outputs = self._RunBazelAndPatchOutput(command)
    timer.End()
    if exit_code:
      _PrintXcodeError('Bazel build failed.')
      return exit_code

    exit_code = self._EnsureBazelBinIsValid()
    if exit_code:
      _PrintXcodeError('Failed to ensure existence of bazel-bin directory.')
      return exit_code

    if parser.install_generated_artifacts:
      timer = Timer('Installing artifacts', 'installing_artifacts').Start()
      exit_code = self._InstallArtifact(outputs)
      timer.End()
      if exit_code:
        return exit_code

      timer = Timer('Installing generated headers',
                    'installing_generated_headers').Start()
      exit_code = self._InstallGeneratedHeaders(outputs)
      timer.End()
      if exit_code:
        return exit_code

      if self.generate_dsym:
        timer = Timer('Installing DSYM bundles', 'installing_dsym').Start()
        exit_code, dsym_path = self._InstallDSYMBundles(self.built_products_dir)
        timer.End()
        if exit_code:
          return exit_code
        if dsym_path:
          timer = Timer('Patching DSYM source file paths',
                        'patching_dsym').Start()
          exit_code = self._PatchdSYMPaths(dsym_path)
          timer.End()
          if exit_code:
            return exit_code

      # Starting with Xcode 7.3, XCTests inject several supporting frameworks
      # into the test host that need to be signed with the same identity as
      # the host itself.
      if (self.is_test and self.xcode_version_minor >= 730 and
          self.codesigning_allowed):
        exit_code = self._ResignTestArtifacts()
        if exit_code:
          return exit_code

    # Starting with Xcode 8, .lldbinit files are honored during Xcode debugging
    # sessions. This allows use of the target.source-map field to remap the
    # debug symbol paths encoded in the binary to the paths expected by Xcode.
    # In cases where a dSYM bundle was produced, the post_processor will have
    # already corrected the paths and use of target.source-map is redundant (and
    # appears to trigger actual problems in Xcode 8.1 betas).
    if self.xcode_version_major >= 800:
      timer = Timer('Updating .lldbinit', 'updating_lldbinit').Start()
      exit_code = self._UpdateLLDBInit(self.generate_dsym)
      timer.End()
      if exit_code:
        _PrintXcodeWarning('Updating .lldbinit action failed with code %d' %
                           exit_code)

    if self.code_coverage_enabled:
      timer = Timer('Patching LLVM covmap', 'patching_llvm_covmap').Start()
      exit_code = self._PatchLLVMCovmapPaths()
      timer.End()
      if exit_code:
        _PrintXcodeWarning('Patch LLVM covmap action failed with code %d' %
                           exit_code)
    return 0

  def _BuildBazelCommand(self, options):
    """Builds up a commandline string suitable for running Bazel."""
    bazel_command = [options.bazel_executable]

    configuration = os.environ['CONFIGURATION']
    # Treat the special testrunner build config as a Debug compile.
    test_runner_config_prefix = '__TulsiTestRunner_'
    if configuration.startswith(test_runner_config_prefix):
      configuration = configuration[len(test_runner_config_prefix):]
    elif os.environ.get('TULSI_TEST_RUNNER_ONLY') == 'YES':
      _PrintXcodeError('Building test targets with configuration "%s" is not '
                       'allowed. Please use the "Test" action or "Build for" > '
                       '"Testing" instead.' % configuration)
      return (None, 1)

    if configuration not in _OptionsParser.KNOWN_CONFIGS:
      _PrintXcodeError('Unknown build configuration "%s"' % configuration)
      return (None, 1)

    bazel_command.extend(options.GetStartupOptions(configuration))
    bazel_command.append('build')
    bazel_command.extend(options.GetBuildOptions(configuration))

    if self.use_dynamic_outputs:
      self._PrintVerbose('Using dynamic outputs')
      # Do not follow symlinks on __file__ in case this script is linked during
      # development.
      tulsi_package_dir = os.path.abspath(
          os.path.join(os.path.dirname(__file__), '..', 'Bazel'))
      package_path = '%s:%s' % (self.bazel_package_path, tulsi_package_dir)

      bazel_command.extend([
          '--experimental_show_artifacts',
          '--output_groups=tulsi-outputs,default',
          '--aspects', '/tulsi/tulsi_aspects.bzl%tulsi_outputs_aspect',
          '--package_path=%s' % package_path])

    if self.code_coverage_enabled:
      self._PrintVerbose('Enabling code coverage information.')
      bazel_command.extend([
          '--collect_code_coverage',
          '--experimental_use_llvm_covmap'])

    if self.generate_dsym:
      bazel_command.append('--apple_generate_dsym')

    bazel_command.extend(options.targets)

    return (bazel_command, 0)

  def _RunBazelAndPatchOutput(self, command):
    """Runs subprocess command, patching output as it's received."""
    self._PrintVerbose('Running "%s", patching output for main group path at '
                       '"%s" with project path at "%s".' %
                       (' '.join(command),
                        self.main_group_path,
                        self.project_dir))
    # Xcode translates anything that looks like ""<path>:<line>:" that is not
    # followed by the word "warning" into an error. Bazel warnings do not fit
    # this scheme and must be patched here.
    bazel_warning_line_regex = re.compile(
        r'WARNING: ([^:]+:\d+:(?:\d+:)?)\s+(.+)')

    def PatchBazelWarningStatements(output_line):
      match = bazel_warning_line_regex.match(output_line)
      if match:
        output_line = '%s warning: %s' % (match.group(1), match.group(2))
      return output_line

    def ExtractOutputs(output_line):
      if output_line.startswith('>>>') and output_line.endswith('.tulsiouts'):
        return output_line[3:]

    patch_xcode_parsable_line = PatchBazelWarningStatements
    if self.main_group_path != self.project_dir:
      # Match (likely) filename:line_number: lines.
      xcode_parsable_line_regex = re.compile(r'([^/][^:]+):\d+:')

      def PatchOutputLine(output_line):
        output_line = PatchBazelWarningStatements(output_line)
        if xcode_parsable_line_regex.match(output_line):
          output_line = '%s/%s' % (self.main_group_path, output_line)
        return output_line
      patch_xcode_parsable_line = PatchOutputLine

    process = subprocess.Popen(command,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
                               bufsize=1)
    output_locations = []
    linebuf = ''
    while process.returncode is None:
      for line in process.stdout.readline():
        # Occasionally Popen's line-buffering appears to break down. Not
        # entirely certain why this happens, but we use an accumulator to
        # try to deal with it.
        if not line.endswith('\n'):
          linebuf += line
          continue

        complete_line = linebuf + line
        # >>> marks the start of an aspect output location.
        # .tulsiouts files contin build output locations
        # TODO(b/35322727): Use BEP instead of stderr output.
        output_path = ExtractOutputs(complete_line.strip())
        if output_path:
          output_locations.append(output_path)
        else:
          line = patch_xcode_parsable_line(complete_line)
        linebuf = ''
        sys.stdout.write(line)
        sys.stdout.flush()
      process.poll()

    output, _ = process.communicate()
    output = linebuf + output

    for line in output.split('\n'):
      output_path = ExtractOutputs(line)
      if output_path:
        output_locations.append(output_path)
      else:
        line = patch_xcode_parsable_line(line)
      print line

    return process.returncode, output_locations

  def _EnsureBazelBinIsValid(self):
    """Ensures that the Bazel output path points at a real directory."""

    if not os.path.isdir(self.bazel_bin_path):
      _PrintXcodeWarning('Bazel "-bin" path at "%s" non-existent' %
                         (self.bazel_bin_path))
      return 0

    self.real_bazel_bin_path = (
        os.path.abspath(os.path.realpath(self.bazel_bin_path)))
    if not os.path.isdir(self.real_bazel_bin_path):
      try:
        os.makedirs(self.real_bazel_bin_path)
      except OSError as e:
        _PrintXcodeError('Failed to create Bazel binary dir at "%s". %s' %
                         (self.real_bazel_bin_path, e))
        return 20

    # The Bazel bin path will be of the form:
    #   <sandbox>/execroot/<workspace_path>/bazel-out/<arch>/bin
    # As the workspace root is user-configurable and could be set to
    # "bazel-out," the workspace path is obtained by slicing off the last
    # three components.
    path_components = self.real_bazel_bin_path.split(os.sep)
    if len(path_components) < 5:
      _PrintXcodeWarning('Failed to derive Bazel build root path from %r' %
                         self.real_bazel_bin_path)
    else:
      self.bazel_build_workspace_root = (
          os.sep + os.path.join(*path_components[:-3]))
    return 0

  def _InstallArtifact(self, outputs):
    """Installs Bazel-generated artifacts into the Xcode output directory."""
    xcode_artifact_path = self.content_folder_path

    if os.path.isdir(xcode_artifact_path):
      try:
        shutil.rmtree(xcode_artifact_path)
      except OSError as e:
        _PrintXcodeError('Failed to remove stale output directory ""%s". '
                         '%s' % (xcode_artifact_path, e))
        return 600
    elif os.path.isfile(xcode_artifact_path):
      try:
        os.remove(xcode_artifact_path)
      except OSError as e:
        _PrintXcodeError('Failed to remove stale output file ""%s". '
                         '%s' % (xcode_artifact_path, e))
        return 600

    if self.use_dynamic_outputs:
      try:
        output_data = json.load(open(outputs[0]))
      except (ValueError, IOError) as e:
        _PrintXcodeError('Failed to load output map ""%s". '
                         '%s' % (outputs[0], e))
        return 600

      if 'artifacts' not in output_data:
        _PrintXcodeError(
            'Failed to find an output artifact for target %s in output map %r' %
            (xcode_artifact_path, output_data))
        return 601

      primary_artifact = output_data['artifacts'][0]
    else:
      if not self.bazel_outputs:
        _PrintXcodeError(
            'Failed to find an output artifact for target %s in candidates %r' %
            (xcode_artifact_path, self.bazel_outputs))
        return 601

      primary_artifact = self.bazel_outputs[0]

    # The PRODUCT_NAME used by the Xcode project is not trustable as it may be
    # modified by the user and, more importantly, may have been modified by
    # Tulsi to disambiguate multiple targets with the same name.
    # To work around this, the product name is determined by dropping any
    # extension from the primary artifact.
    # TODO(abaire): Consider passing this value to the script explicitly.
    self.bazel_product_name = os.path.splitext(
        os.path.basename(primary_artifact))[0]

    if primary_artifact.endswith('.ipa') or primary_artifact.endswith('.zip'):
      exit_code = self._UnpackTarget(primary_artifact, xcode_artifact_path)
      if exit_code:
        return exit_code

      exit_code = self._RewriteInfoPlistIfNecessary(xcode_artifact_path)
      if exit_code:
        return exit_code
    elif os.path.isfile(primary_artifact):
      exit_code = self._CopyFile(os.path.basename(primary_artifact),
                                 primary_artifact,
                                 xcode_artifact_path)
      if exit_code:
        return exit_code
    else:
      self._CopyBundle(os.path.basename(primary_artifact),
                       primary_artifact,
                       xcode_artifact_path)

    return 0

  def _InstallGeneratedHeaders(self, output_files):
    """Installs Bazel-generated headers into tulsi-includes directory."""
    tulsi_root = os.path.join(self.workspace_root, 'tulsi-includes')

    # Older versions of this script created tulsi-includes as a concrete
    # directory instead of symlink, in which case we need to cleanup.
    if os.path.exists(tulsi_root) and not os.path.islink(tulsi_root):
      shutil.rmtree(tulsi_root)

    if not os.path.exists(tulsi_root):
      tmp_dir = tempfile.mkdtemp(prefix='tulsi')
      os.symlink(tmp_dir, tulsi_root)

    for f in output_files:
      data = json.load(open(f))
      if 'generated_sources' not in data:
        continue

      for gs in data['generated_sources']:
        real_path, link_path = gs
        src = os.path.join(self.bazel_build_workspace_root, real_path)

        # Bazel outputs are not guaranteed to be created if nothing references
        # them. This check skips the processing if an output was declared
        # but not created.
        if not os.path.exists(src):
          continue

        # The /x/x/ part is here to match the number of directory components
        # between tulsi root and bazel root. See tulsi_aspects.bzl for futher
        # explanation.
        dst = os.path.join(tulsi_root, 'x/x/', link_path)
        self._PrintVerbose('Symlinking %s to %s' % (src, dst), 2)

        dst_dir = os.path.split(dst)[0]
        if not os.path.exists(dst_dir):
          os.makedirs(dst_dir)

        # It's important to use lexists() here in case dst is a broken symlink
        # (in which case exists() would return False). For example, older<commit_msg>Remove tulsi-includes symlink if it’s broken<commit_after>#!/usr/bin/python
# Copyright 2016 The Tulsi Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bridge between Xcode and Bazel for the "build" action.

NOTE: This script must be executed in the same directory as the Xcode project's
main group in order to generate correct debug symbols.
"""

import collections
import json
import os
import re
import shutil
import stat
import StringIO
import subprocess
import sys
import tempfile
import textwrap
import time
import zipfile
import tulsi_logging


def _PrintXcodeWarning(msg):
  sys.stdout.write(':: warning: %s\n' % msg)
  sys.stdout.flush()


def _PrintXcodeError(msg):
  sys.stderr.write(':: error: %s\n' % msg)
  sys.stderr.flush()


class Timer(object):
  """Simple profiler."""

  def __init__(self, action_name, action_id):
    """Creates a new Timer object.

    Args:
      action_name: A human-readable action name, shown in the build log.
      action_id: A machine-readable action identifier, can be used for metrics.

    Returns:
      A Timer instance.
    """
    self.action_name = action_name
    self.action_id = action_id
    self._start = None

  def Start(self):
    self._start = time.time()
    return self

  def End(self):
    end = time.time()
    seconds = end - self._start
    tulsi_logging.Logger().log_action(self.action_name, self.action_id, seconds)


class CodesignBundleAttributes(object):
  """Wrapper class for codesigning attributes of a signed bundle."""

  # List of codesigning attributes that this script requires.
  _ATTRIBUTES = ['Authority', 'Identifier', 'TeamIdentifier']

  def __init__(self, codesign_output):
    self.attributes = {}

    pending_attributes = list(self._ATTRIBUTES)
    for line in codesign_output.split('\n'):
      if not pending_attributes:
        break

      for attribute in pending_attributes:
        if line.startswith(attribute):
          value = line[len(attribute) + 1:]
          self.attributes[attribute] = value
          pending_attributes.remove(attribute)
          break

    for attribute in self._ATTRIBUTES:
      if attribute not in self.attributes:
        _PrintXcodeError(
            'Failed to extract %s from %s.\n' % (attribute, codesign_output))

  def Get(self, attribute):
    """Returns the value for the given attribute, or None if it wasn't found."""
    value = self.attributes.get(attribute)
    if attribute not in self._ATTRIBUTES:
      _PrintXcodeError(
          'Attribute %s not declared to be parsed. ' % attribute +
          'Available attributes are %s.\n' % self._ATTRIBUTES)
    return value


class _OptionsParser(object):
  """Handles parsing script options."""

  # Key for options that should be applied to all build configurations.
  ALL_CONFIGS = '__all__'

  # The build configurations handled by this parser.
  KNOWN_CONFIGS = ['Debug', 'Release', 'Fastbuild']

  def __init__(self, sdk_version, platform_name, arch, main_group_path):
    self.targets = []
    self.startup_options = collections.defaultdict(list)
    self.build_options = collections.defaultdict(
        list,
        {
            _OptionsParser.ALL_CONFIGS: [
                '--experimental_enable_objc_cc_deps',
                '--verbose_failures',
                '--announce_rc',
            ],

            'Debug': [
                '--compilation_mode=dbg',
            ],

            'Release': [
                '--compilation_mode=opt',
                '--strip=always',
            ],

            'Fastbuild': [
                '--compilation_mode=fastbuild',
            ],
        })

    # Options specific to debugger integration in Xcode.
    xcode_version_major = int(os.environ['XCODE_VERSION_MAJOR'])
    if xcode_version_major < 800:
      xcode_lldb_options = [
          '--copt=-Xclang', '--copt=-fdebug-compilation-dir',
          '--copt=-Xclang', '--copt=%s' % main_group_path,
          '--objccopt=-Xclang', '--objccopt=-fdebug-compilation-dir',
          '--objccopt=-Xclang', '--objccopt=%s' % main_group_path,
      ]
      self.build_options['Debug'].extend(xcode_lldb_options)
      self.build_options['Release'].extend(xcode_lldb_options)

    self.sdk_version = sdk_version
    self.platform_name = platform_name

    if self.platform_name.startswith('watch'):
      config_platform = 'watchos'
    elif self.platform_name.startswith('iphone'):
      config_platform = 'ios'
    elif self.platform_name.startswith('appletv'):
      config_platform = 'tvos'
    else:
      self._WarnUnknownPlatform()
      config_platform = 'ios'
    self.build_options[_OptionsParser.ALL_CONFIGS].append(
        '--config=%s_%s' % (config_platform, arch))

    self.verbose = 0
    self.install_generated_artifacts = False
    self.bazel_bin_path = 'bazel-bin'
    self.bazel_package_path = None
    self.bazel_executable = None

  @staticmethod
  def _UsageMessage():
    """Returns a usage message string."""
    usage = textwrap.dedent("""\
      Usage: %s <target> [<target2> ...] --bazel <bazel_binary_path> [options]

      Where options are:
        --verbose [-v]
            Increments the verbosity of the script by one level. This argument
            may be provided multiple times to enable additional output levels.

        --unpack_generated_ipa
            Unzips the contents of the IPA artifact generated by this build.

        --bazel_startup_options <option1> [<option2> ...] --
            Provides one or more Bazel startup options.

        --bazel_options <option1> [<option2> ...] --
            Provides one or more Bazel build options.

        --bazel_bin_path <path>
            Path at which Bazel-generated artifacts may be retrieved.

        --bazel_package_path <path>
            The value of the package_path variable in Bazel workspace.
      """ % sys.argv[0])

    usage += '\n' + textwrap.fill(
        'Note that the --bazel_startup_options and --bazel_options options may '
        'include an optional configuration specifier in brackets to limit '
        'their contents to a given build configuration. Options provided with '
        'no configuration filter will apply to all configurations in addition '
        'to any configuration-specific options.', 120)

    usage += '\n' + textwrap.fill(
        'E.g., --bazel_options common --  --bazel_options[Release] release -- '
        'would result in "bazel build common release" in the "Release" '
        'configuration and "bazel build common" in all other configurations.',
        120)

    return usage

  def ParseOptions(self, args):
    """Parses arguments, returning (message, exit_code)."""

    bazel_executable_index = args.index('--bazel')

    self.targets = args[:bazel_executable_index]
    if not self.targets or len(args) < bazel_executable_index + 2:
      return (self._UsageMessage(), 10)
    self.bazel_executable = args[bazel_executable_index + 1]

    return self._ParseVariableOptions(args[bazel_executable_index + 2:])

  def GetStartupOptions(self, config):
    """Returns the full set of startup options for the given config."""
    return self._GetOptions(self.startup_options, config)

  def GetBuildOptions(self, config):
    """Returns the full set of build options for the given config."""
    options = self._GetOptions(self.build_options, config)

    version_string = self._GetXcodeVersionString()
    if version_string:
      self._AddDefaultOption(options, '--xcode_version', version_string)

    if self.sdk_version:
      if self.platform_name.startswith('watch'):
        self._AddDefaultOption(options,
                               '--watchos_sdk_version',
                               self.sdk_version)
      elif self.platform_name.startswith('iphone'):
        self._AddDefaultOption(options, '--ios_sdk_version', self.sdk_version)
      elif self.platform_name.startswith('appletv'):
        self._AddDefaultOption(options, '--tvos_sdk_version', self.sdk_version)
      else:
        self._WarnUnknownPlatform()
        self._AddDefaultOption(options, '--ios_sdk_version', self.sdk_version)
    return options

  @staticmethod
  def _AddDefaultOption(option_list, option, default_value):
    matching_options = [opt for opt in option_list if opt.startswith(option)]
    if matching_options:
      return option_list

    option_list.append('%s=%s' % (option, default_value))
    return option_list

  @staticmethod
  def _GetOptions(option_set, config):
    """Returns a flattened list from options_set for the given config."""
    options = list(option_set[_OptionsParser.ALL_CONFIGS])
    if config != _OptionsParser.ALL_CONFIGS:
      options.extend(option_set[config])
    return options

  def _WarnUnknownPlatform(self):
    sys.stdout.write('Warning: unknown platform "%s" will be treated as '
                     'iOS\n' % self.platform_name)
    sys.stdout.flush()

  def _ParseVariableOptions(self, args):
    """Parses flag-based args, returning (message, exit_code)."""

    verbose_re = re.compile('-(v+)$')

    while args:
      arg = args[0]
      args = args[1:]

      if arg == '--install_generated_artifacts':
        self.install_generated_artifacts = True

      elif arg.startswith('--bazel_startup_options'):
        config = self._ParseConfigFilter(arg)
        args, items, terminated = self._ParseDoubleDashDelimitedItems(args)
        if not terminated:
          return ('Missing "--" terminator while parsing %s' % arg, 2)
        duplicates = self._FindDuplicateOptions(self.startup_options,
                                                config,
                                                items)
        if duplicates:
          return (
              '%s items conflict with common options: %s' % (
                  arg, ','.join(duplicates)),
              2)
        self.startup_options[config].extend(items)

      elif arg.startswith('--bazel_options'):
        config = self._ParseConfigFilter(arg)
        args, items, terminated = self._ParseDoubleDashDelimitedItems(args)
        if not terminated:
          return ('Missing "--" terminator while parsing %s' % arg, 2)
        duplicates = self._FindDuplicateOptions(self.build_options,
                                                config,
                                                items)
        if duplicates:
          return (
              '%s items conflict with common options: %s' % (
                  arg, ','.join(duplicates)),
              2)
        self.build_options[config].extend(items)

      elif arg == '--bazel_bin_path':
        if not args:
          return ('Missing required parameter for %s' % arg, 2)
        self.bazel_bin_path = args[0]
        args = args[1:]

      elif arg == '--bazel_package_path':
        if not args:
          return ('Missing required parameter for %s' % arg, 2)
        self.bazel_package_path = args[0]
        args = args[1:]

      elif arg == '--verbose':
        self.verbose += 1

      else:
        match = verbose_re.match(arg)
        if match:
          self.verbose += len(match.group(1))
        else:
          return ('Unknown option "%s"\n%s' % (arg, self._UsageMessage()), 1)

    return (None, 0)

  @staticmethod
  def _ParseConfigFilter(arg):
    match = re.search(r'\[([^\]]+)\]', arg)
    if not match:
      return _OptionsParser.ALL_CONFIGS
    return match.group(1)

  @staticmethod
  def _ConsumeArgumentForParam(param, args):
    if not args:
      return (None, 'Missing required parameter for "%s" option' % param)
    val = args[0]
    return (args[1:], val)

  @staticmethod
  def _ParseDoubleDashDelimitedItems(args):
    """Consumes options until -- is found."""
    options = []
    terminator_found = False

    opts = args
    while opts:
      opt = opts[0]
      opts = opts[1:]
      if opt == '--':
        terminator_found = True
        break
      options.append(opt)

    return opts, options, terminator_found

  @staticmethod
  def _FindDuplicateOptions(options_dict, config, new_options):
    """Returns a list of options appearing in both given option lists."""

    allowed_duplicates = [
        '--copt',
        '--config',
        '--define',
        '--objccopt',
    ]

    def ExtractOptionNames(opts):
      names = set()
      for opt in opts:
        split_opt = opt.split('=', 1)
        if split_opt[0] not in allowed_duplicates:
          names.add(split_opt[0])
      return names

    current_set = ExtractOptionNames(options_dict[config])
    new_set = ExtractOptionNames(new_options)
    conflicts = current_set.intersection(new_set)

    if config != _OptionsParser.ALL_CONFIGS:
      current_set = ExtractOptionNames(options_dict[_OptionsParser.ALL_CONFIGS])
      conflicts = conflicts.union(current_set.intersection(new_set))
    return conflicts

  @staticmethod
  def _GetXcodeVersionString():
    """Returns Xcode version info from the environment as a string."""
    reported_version = os.environ['XCODE_VERSION_ACTUAL']
    match = re.match(r'(\d{2})(\d)(\d)$', reported_version)
    if not match:
      sys.stdout.write('Warning: Failed to extract Xcode version from %s\n' % (
          reported_version))
      sys.stdout.flush()
      return None
    major_version = int(match.group(1))
    minor_version = int(match.group(2))
    fix_version = int(match.group(3))
    fix_version_string = ''
    if fix_version:
      fix_version_string = '.%d' % fix_version
    return '%d.%d%s' % (major_version, minor_version, fix_version_string)


class BazelBuildBridge(object):
  """Handles invoking Bazel and unpacking generated binaries."""

  def __init__(self):
    self.verbose = 0
    self.build_path = None
    self.bazel_bin_path = None
    # The actual path to the Bazel output directory (not a symlink)
    self.real_bazel_bin_path = None
    self.bazel_package_path = None
    # The path to the Bazel's sandbox source root.
    self.bazel_build_workspace_root = None
    self.bazel_genfiles_path = None
    self.bazel_symlink_prefix = None
    self.codesign_attributes = {}

    # Certain potentially expensive patchups need to be made for non-Xcode IDE
    # integrations. There isn't a fool-proof way of determining if the script is
    # being used with Xcode or not, but searching the CODESIGNING_FOLDER_PATH
    # env var for "/Xcode/" should catch the majority of use-cases.
    self.codesigning_folder_path = os.environ['CODESIGNING_FOLDER_PATH']
    self.likely_xcode = self.codesigning_folder_path.find('/Xcode/') != -1

    self.xcode_action = os.environ['ACTION']  # The Xcode build action.
    # When invoked as an external build system script, Xcode will set ACTION to
    # an empty string.
    if not self.xcode_action:
      self.xcode_action = 'build'

    self.generate_dsym = os.environ.get('TULSI_USE_DSYM', 'NO') == 'YES'
    self.use_dynamic_outputs = (
        os.environ.get('TULSI_USE_DYNAMIC_OUTPUTS', 'NO') == 'YES')

    # Target architecture.  Must be defined for correct setting of
    # the --config flag
    self.arch = os.environ.get('CURRENT_ARCH')
    if not self.arch:
      _PrintXcodeError('Tulsi requires env variable CURRENT_ARCH to be '
                       'set.  Please file a bug against Tulsi.')
      sys.exit(1)

    # Declared outputs of the target.
    self.bazel_outputs = os.environ.get('BAZEL_OUTPUTS', [])
    if self.bazel_outputs:
      self.bazel_outputs = self.bazel_outputs.split('\n')
    # Bazel's notion of the type of artifact being generated.
    self.bazel_target_type = os.environ.get('BAZEL_TARGET_TYPE')
    # Path into which generated artifacts should be copied.
    self.built_products_dir = os.environ['BUILT_PRODUCTS_DIR']
    # Whether or not code coverage information should be generated.
    self.code_coverage_enabled = (
        os.environ.get('CLANG_COVERAGE_MAPPING') == 'YES')
    # Path where Xcode expects generated sources to be placed.
    self.derived_sources_folder_path = os.environ.get('DERIVED_SOURCES_DIR')
    # Full name of the target artifact (e.g., "MyApp.app" or "Test.xctest").
    self.full_product_name = os.environ['FULL_PRODUCT_NAME']
    # Target SDK version.
    self.sdk_version = os.environ.get('SDK_VERSION')
    # TEST_HOST for unit tests.
    self.test_host_binary = os.environ.get('TEST_HOST')
    # Whether this target is a test or not.
    self.is_test = os.environ.get('WRAPPER_EXTENSION') == 'xctest'
    # UTI type of the target.
    self.package_type = os.environ.get('PACKAGE_TYPE')
    # Target platform.
    self.platform_name = os.environ['PLATFORM_NAME']
    # Type of the target artifact.
    self.product_type = os.environ['PRODUCT_TYPE']
    # Path to the parent of the xcodeproj bundle.
    self.project_dir = os.environ['PROJECT_DIR']
    # Path to the xcodeproj bundle.
    self.project_file_path = os.environ['PROJECT_FILE_PATH']
    # Path to the parent of the Xcode project's mainGroup.
    self.source_root = os.environ['SOURCE_ROOT']
    # Path to the directory containing the WORKSPACE file.
    self.workspace_root = os.path.abspath(os.environ['TULSI_WR'])
    # Set to the name of the generated bundle for bundle-type targets, None for
    # single file targets (like static libraries).
    self.wrapper_name = os.environ.get('WRAPPER_NAME')
    self.wrapper_suffix = os.environ.get('WRAPPER_SUFFIX', '')
    self.xcode_version_major = int(os.environ['XCODE_VERSION_MAJOR'])
    self.xcode_version_minor = int(os.environ['XCODE_VERSION_MINOR'])

    # Path where Xcode expects the artifacts to be written to. This is not the
    # codesigning_path as device vs simulator builds have different signing
    # requirements, so Xcode expects different things to be signed. This is
    # mostly apparent on XCUITests where simulator builds set the codesigning
    # path to be the .xctest bundle, but for device builds it is actually the
    # UI runner app (since it needs to be codesigned to run on the device.) The
    # contents folder path is a stable path on where to put the expected
    # artifacts. For static libraries (objc_library, swift_library),
    # CONTENTS_FOLDER_PATH does not exist, but the location where Xcode expects
    # the archive coincides with the TARGET_BUILD_DIR, so using an empty
    # default for CONTENTS_FOLDER_PATH supports both bundle and single artifact
    # outputs.
    # TODO(b/35811023): Check these paths are still valid.
    self.content_folder_path = os.path.join(
        os.environ['TARGET_BUILD_DIR'],
        os.environ.get('CONTENTS_FOLDER_PATH', ''))

    # Path to where Xcode expects the binary to be placed.
    self.binary_path = os.path.join(
        os.environ['TARGET_BUILD_DIR'], os.environ['EXECUTABLE_PATH'])

    self.is_simulator = self.platform_name.endswith('simulator')
    # Check to see if code signing actions should be skipped or not.
    if self.is_simulator:
      self.codesigning_allowed = False
    else:
      self.codesigning_allowed = os.environ.get('CODE_SIGNING_ALLOWED') == 'YES'

    self.post_processor_binary = os.path.join(self.project_file_path,
                                              '.tulsi',
                                              'Utils',
                                              'post_processor')
    if self.codesigning_allowed:
      self.runner_entitlements_template = os.path.join(self.project_file_path,
                                                       '.tulsi',
                                                       'Resources',
                                                       'XCTRunner.entitlements')

    self.main_group_path = os.getcwd()

  def Run(self, args):
    """Executes a Bazel build based on the environment and given arguments."""
    if self.xcode_action != 'build':
      sys.stderr.write('Xcode action is %s, ignoring.' % self.xcode_action)
      return 0

    parser = _OptionsParser(self.sdk_version,
                            self.platform_name,
                            self.arch,
                            self.main_group_path)
    timer = Timer('Parsing options', 'parsing_options').Start()
    message, exit_code = parser.ParseOptions(args[1:])
    timer.End()
    if exit_code:
      _PrintXcodeError('Option parsing failed: %s' % message)
      return exit_code

    self.verbose = parser.verbose
    self.bazel_bin_path = os.path.abspath(parser.bazel_bin_path)
    # bazel_bin_path is assumed to always end in "-bin".
    self.bazel_symlink_prefix = self.bazel_bin_path[:-3]
    self.bazel_genfiles_path = self.bazel_symlink_prefix + 'genfiles'

    self.build_path = os.path.join(self.bazel_bin_path,
                                   os.environ.get('TULSI_BUILD_PATH', ''))
    self.bazel_package_path = parser.bazel_package_path
    (command, retval) = self._BuildBazelCommand(parser)
    if retval:
      return retval

    timer = Timer('Running Bazel', 'running_bazel').Start()
    exit_code, outputs = self._RunBazelAndPatchOutput(command)
    timer.End()
    if exit_code:
      _PrintXcodeError('Bazel build failed.')
      return exit_code

    exit_code = self._EnsureBazelBinIsValid()
    if exit_code:
      _PrintXcodeError('Failed to ensure existence of bazel-bin directory.')
      return exit_code

    if parser.install_generated_artifacts:
      timer = Timer('Installing artifacts', 'installing_artifacts').Start()
      exit_code = self._InstallArtifact(outputs)
      timer.End()
      if exit_code:
        return exit_code

      timer = Timer('Installing generated headers',
                    'installing_generated_headers').Start()
      exit_code = self._InstallGeneratedHeaders(outputs)
      timer.End()
      if exit_code:
        return exit_code

      if self.generate_dsym:
        timer = Timer('Installing DSYM bundles', 'installing_dsym').Start()
        exit_code, dsym_path = self._InstallDSYMBundles(self.built_products_dir)
        timer.End()
        if exit_code:
          return exit_code
        if dsym_path:
          timer = Timer('Patching DSYM source file paths',
                        'patching_dsym').Start()
          exit_code = self._PatchdSYMPaths(dsym_path)
          timer.End()
          if exit_code:
            return exit_code

      # Starting with Xcode 7.3, XCTests inject several supporting frameworks
      # into the test host that need to be signed with the same identity as
      # the host itself.
      if (self.is_test and self.xcode_version_minor >= 730 and
          self.codesigning_allowed):
        exit_code = self._ResignTestArtifacts()
        if exit_code:
          return exit_code

    # Starting with Xcode 8, .lldbinit files are honored during Xcode debugging
    # sessions. This allows use of the target.source-map field to remap the
    # debug symbol paths encoded in the binary to the paths expected by Xcode.
    # In cases where a dSYM bundle was produced, the post_processor will have
    # already corrected the paths and use of target.source-map is redundant (and
    # appears to trigger actual problems in Xcode 8.1 betas).
    if self.xcode_version_major >= 800:
      timer = Timer('Updating .lldbinit', 'updating_lldbinit').Start()
      exit_code = self._UpdateLLDBInit(self.generate_dsym)
      timer.End()
      if exit_code:
        _PrintXcodeWarning('Updating .lldbinit action failed with code %d' %
                           exit_code)

    if self.code_coverage_enabled:
      timer = Timer('Patching LLVM covmap', 'patching_llvm_covmap').Start()
      exit_code = self._PatchLLVMCovmapPaths()
      timer.End()
      if exit_code:
        _PrintXcodeWarning('Patch LLVM covmap action failed with code %d' %
                           exit_code)
    return 0

  def _BuildBazelCommand(self, options):
    """Builds up a commandline string suitable for running Bazel."""
    bazel_command = [options.bazel_executable]

    configuration = os.environ['CONFIGURATION']
    # Treat the special testrunner build config as a Debug compile.
    test_runner_config_prefix = '__TulsiTestRunner_'
    if configuration.startswith(test_runner_config_prefix):
      configuration = configuration[len(test_runner_config_prefix):]
    elif os.environ.get('TULSI_TEST_RUNNER_ONLY') == 'YES':
      _PrintXcodeError('Building test targets with configuration "%s" is not '
                       'allowed. Please use the "Test" action or "Build for" > '
                       '"Testing" instead.' % configuration)
      return (None, 1)

    if configuration not in _OptionsParser.KNOWN_CONFIGS:
      _PrintXcodeError('Unknown build configuration "%s"' % configuration)
      return (None, 1)

    bazel_command.extend(options.GetStartupOptions(configuration))
    bazel_command.append('build')
    bazel_command.extend(options.GetBuildOptions(configuration))

    if self.use_dynamic_outputs:
      self._PrintVerbose('Using dynamic outputs')
      # Do not follow symlinks on __file__ in case this script is linked during
      # development.
      tulsi_package_dir = os.path.abspath(
          os.path.join(os.path.dirname(__file__), '..', 'Bazel'))
      package_path = '%s:%s' % (self.bazel_package_path, tulsi_package_dir)

      bazel_command.extend([
          '--experimental_show_artifacts',
          '--output_groups=tulsi-outputs,default',
          '--aspects', '/tulsi/tulsi_aspects.bzl%tulsi_outputs_aspect',
          '--package_path=%s' % package_path])

    if self.code_coverage_enabled:
      self._PrintVerbose('Enabling code coverage information.')
      bazel_command.extend([
          '--collect_code_coverage',
          '--experimental_use_llvm_covmap'])

    if self.generate_dsym:
      bazel_command.append('--apple_generate_dsym')

    bazel_command.extend(options.targets)

    return (bazel_command, 0)

  def _RunBazelAndPatchOutput(self, command):
    """Runs subprocess command, patching output as it's received."""
    self._PrintVerbose('Running "%s", patching output for main group path at '
                       '"%s" with project path at "%s".' %
                       (' '.join(command),
                        self.main_group_path,
                        self.project_dir))
    # Xcode translates anything that looks like ""<path>:<line>:" that is not
    # followed by the word "warning" into an error. Bazel warnings do not fit
    # this scheme and must be patched here.
    bazel_warning_line_regex = re.compile(
        r'WARNING: ([^:]+:\d+:(?:\d+:)?)\s+(.+)')

    def PatchBazelWarningStatements(output_line):
      match = bazel_warning_line_regex.match(output_line)
      if match:
        output_line = '%s warning: %s' % (match.group(1), match.group(2))
      return output_line

    def ExtractOutputs(output_line):
      if output_line.startswith('>>>') and output_line.endswith('.tulsiouts'):
        return output_line[3:]

    patch_xcode_parsable_line = PatchBazelWarningStatements
    if self.main_group_path != self.project_dir:
      # Match (likely) filename:line_number: lines.
      xcode_parsable_line_regex = re.compile(r'([^/][^:]+):\d+:')

      def PatchOutputLine(output_line):
        output_line = PatchBazelWarningStatements(output_line)
        if xcode_parsable_line_regex.match(output_line):
          output_line = '%s/%s' % (self.main_group_path, output_line)
        return output_line
      patch_xcode_parsable_line = PatchOutputLine

    process = subprocess.Popen(command,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
                               bufsize=1)
    output_locations = []
    linebuf = ''
    while process.returncode is None:
      for line in process.stdout.readline():
        # Occasionally Popen's line-buffering appears to break down. Not
        # entirely certain why this happens, but we use an accumulator to
        # try to deal with it.
        if not line.endswith('\n'):
          linebuf += line
          continue

        complete_line = linebuf + line
        # >>> marks the start of an aspect output location.
        # .tulsiouts files contin build output locations
        # TODO(b/35322727): Use BEP instead of stderr output.
        output_path = ExtractOutputs(complete_line.strip())
        if output_path:
          output_locations.append(output_path)
        else:
          line = patch_xcode_parsable_line(complete_line)
        linebuf = ''
        sys.stdout.write(line)
        sys.stdout.flush()
      process.poll()

    output, _ = process.communicate()
    output = linebuf + output

    for line in output.split('\n'):
      output_path = ExtractOutputs(line)
      if output_path:
        output_locations.append(output_path)
      else:
        line = patch_xcode_parsable_line(line)
      print line

    return process.returncode, output_locations

  def _EnsureBazelBinIsValid(self):
    """Ensures that the Bazel output path points at a real directory."""

    if not os.path.isdir(self.bazel_bin_path):
      _PrintXcodeWarning('Bazel "-bin" path at "%s" non-existent' %
                         (self.bazel_bin_path))
      return 0

    self.real_bazel_bin_path = (
        os.path.abspath(os.path.realpath(self.bazel_bin_path)))
    if not os.path.isdir(self.real_bazel_bin_path):
      try:
        os.makedirs(self.real_bazel_bin_path)
      except OSError as e:
        _PrintXcodeError('Failed to create Bazel binary dir at "%s". %s' %
                         (self.real_bazel_bin_path, e))
        return 20

    # The Bazel bin path will be of the form:
    #   <sandbox>/execroot/<workspace_path>/bazel-out/<arch>/bin
    # As the workspace root is user-configurable and could be set to
    # "bazel-out," the workspace path is obtained by slicing off the last
    # three components.
    path_components = self.real_bazel_bin_path.split(os.sep)
    if len(path_components) < 5:
      _PrintXcodeWarning('Failed to derive Bazel build root path from %r' %
                         self.real_bazel_bin_path)
    else:
      self.bazel_build_workspace_root = (
          os.sep + os.path.join(*path_components[:-3]))
    return 0

  def _InstallArtifact(self, outputs):
    """Installs Bazel-generated artifacts into the Xcode output directory."""
    xcode_artifact_path = self.content_folder_path

    if os.path.isdir(xcode_artifact_path):
      try:
        shutil.rmtree(xcode_artifact_path)
      except OSError as e:
        _PrintXcodeError('Failed to remove stale output directory ""%s". '
                         '%s' % (xcode_artifact_path, e))
        return 600
    elif os.path.isfile(xcode_artifact_path):
      try:
        os.remove(xcode_artifact_path)
      except OSError as e:
        _PrintXcodeError('Failed to remove stale output file ""%s". '
                         '%s' % (xcode_artifact_path, e))
        return 600

    if self.use_dynamic_outputs:
      try:
        output_data = json.load(open(outputs[0]))
      except (ValueError, IOError) as e:
        _PrintXcodeError('Failed to load output map ""%s". '
                         '%s' % (outputs[0], e))
        return 600

      if 'artifacts' not in output_data:
        _PrintXcodeError(
            'Failed to find an output artifact for target %s in output map %r' %
            (xcode_artifact_path, output_data))
        return 601

      primary_artifact = output_data['artifacts'][0]
    else:
      if not self.bazel_outputs:
        _PrintXcodeError(
            'Failed to find an output artifact for target %s in candidates %r' %
            (xcode_artifact_path, self.bazel_outputs))
        return 601

      primary_artifact = self.bazel_outputs[0]

    # The PRODUCT_NAME used by the Xcode project is not trustable as it may be
    # modified by the user and, more importantly, may have been modified by
    # Tulsi to disambiguate multiple targets with the same name.
    # To work around this, the product name is determined by dropping any
    # extension from the primary artifact.
    # TODO(abaire): Consider passing this value to the script explicitly.
    self.bazel_product_name = os.path.splitext(
        os.path.basename(primary_artifact))[0]

    if primary_artifact.endswith('.ipa') or primary_artifact.endswith('.zip'):
      exit_code = self._UnpackTarget(primary_artifact, xcode_artifact_path)
      if exit_code:
        return exit_code

      exit_code = self._RewriteInfoPlistIfNecessary(xcode_artifact_path)
      if exit_code:
        return exit_code
    elif os.path.isfile(primary_artifact):
      exit_code = self._CopyFile(os.path.basename(primary_artifact),
                                 primary_artifact,
                                 xcode_artifact_path)
      if exit_code:
        return exit_code
    else:
      self._CopyBundle(os.path.basename(primary_artifact),
                       primary_artifact,
                       xcode_artifact_path)

    return 0

  def _InstallGeneratedHeaders(self, output_files):
    """Installs Bazel-generated headers into tulsi-includes directory."""
    tulsi_root = os.path.join(self.workspace_root, 'tulsi-includes')

    if os.path.islink(tulsi_root):
      # If the symlink is broken, remove it.
      if not os.path.exists(tulsi_root):
        os.unlink(tulsi_root)
    elif os.path.exists(tulsi_root):
      # Older versions of this script created tulsi-includes as a concrete
      # directory instead of symlink, in which case we need to cleanup.
      shutil.rmtree(tulsi_root)

    # If there's no symlink, create a temp directory and link.
    if not os.path.exists(tulsi_root):
      tmp_dir = tempfile.mkdtemp(prefix='tulsi')
      os.symlink(tmp_dir, tulsi_root)

    for f in output_files:
      data = json.load(open(f))
      if 'generated_sources' not in data:
        continue

      for gs in data['generated_sources']:
        real_path, link_path = gs
        src = os.path.join(self.bazel_build_workspace_root, real_path)

        # Bazel outputs are not guaranteed to be created if nothing references
        # them. This check skips the processing if an output was declared
        # but not created.
        if not os.path.exists(src):
          continue

        # The /x/x/ part is here to match the number of directory components
        # between tulsi root and bazel root. See tulsi_aspects.bzl for futher
        # explanation.
        dst = os.path.join(tulsi_root, 'x/x/', link_path)
        self._PrintVerbose('Symlinking %s to %s' % (src, dst), 2)

        dst_dir = os.path.split(dst)[0]
        if not os.path.exists(dst_dir):
          os.makedirs(dst_dir)

        # It's important to use lexists() here in case dst is a broken symlink
        # (in which case exists() would return False). For example, older
{'fraction_non_alphanumeric': 0.09631258747250863, 'fraction_numerical': 0.003655993830510411, 'mean_word_length': 2.6504535502033155, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4217140', 'n_tokens_mistral': 21342, 'n_tokens_neox': 20451, 'n_words': 5486}