<commit_before>{ stdenv, callPackage, lib, fetchurl, fetchpatch, fetchFromGitHub, installShellFiles
, runCommand, runCommandCC, makeWrapper, recurseIntoAttrs
# this package (through the fixpoint glass)
, bazel_self
# needed only for the updater
, bazel_4
, lr, xe, zip, unzip, bash, writeCBin, coreutils
, which, gawk, gnused, gnutar, gnugrep, gzip, findutils
# updater
, python27, python3, writeScript
# Apple dependencies
, cctools, libcxx, CoreFoundation, CoreServices, Foundation
# Allow to independently override the jdks used to build and run respectively
, buildJdk, runJdk
, runtimeShell
# Downstream packages for tests
, bazel-watcher
# Always assume all markers valid (this is needed because we remove markers; they are non-deterministic).
# Also, don't clean up environment variables (so that NIX_ environment variables are passed to compilers).
, enableNixHacks ? false
, gcc-unwrapped
, autoPatchelfHook
, file
, substituteAll
, writeTextFile
}:

let
  version = "5.0.0";
  sourceRoot = ".";

  src = fetchurl {
    url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip";
    sha256 = "By3WLSN9vBHgusAuEY2MLbTQujugnxoOseKkYPuEGds=";
  };

  # Update with `eval $(nix-build -A bazel.updater)`,
  # then add new dependencies from the dict in ./src-deps.json as required.
  srcDeps = lib.attrsets.attrValues srcDepsSet;
  srcDepsSet =
    let
      srcs = lib.importJSON ./src-deps.json;
      toFetchurl = d: lib.attrsets.nameValuePair d.name (fetchurl {
        urls = d.urls;
        sha256 = d.sha256;
        });
        in builtins.listToAttrs (map toFetchurl [
      srcs.desugar_jdk_libs
      srcs.io_bazel_skydoc
      srcs.bazel_skylib
      srcs.bazelci_rules
      srcs.io_bazel_rules_sass
      srcs.platforms
      srcs."remote_java_tools_for_testing"
      srcs."coverage_output_generator-v2.5.zip"
      srcs.build_bazel_rules_nodejs
      srcs."android_tools_pkg-0.23.0.tar.gz"
      srcs.bazel_toolchains
      srcs.com_github_grpc_grpc
      srcs.upb
      srcs.com_google_protobuf
      srcs.rules_pkg
      srcs.rules_cc
      srcs.rules_java
      srcs.rules_proto
      srcs.com_google_absl
      srcs.com_googlesource_code_re2
      srcs.com_github_cares_cares
      ]);

  distDir = runCommand "bazel-deps" {} ''
    mkdir -p $out
    for i in ${builtins.toString srcDeps}; do cp $i $out/$(stripHash $i); done
  '';

  defaultShellUtils =
    # Keep this list conservative. For more exotic tools, prefer to use
    # @rules_nixpkgs to pull in tools from the nix repository. Example:
    #
    # WORKSPACE:
    #
    #     nixpkgs_git_repository(
    #         name = "nixpkgs",
    #         revision = "def5124ec8367efdba95a99523dd06d918cb0ae8",
    #     )
    #
    #     # This defines an external Bazel workspace.
    #     nixpkgs_package(
    #         name = "bison",
    #         repositories = { "nixpkgs": "@nixpkgs//:default.nix" },
    #     )
    #
    # some/BUILD.bazel:
    #
    #     genrule(
    #        ...
    #        cmd = "$(location @bison//:bin/bison) -other -args",
    #        tools = [
    #            ...
    #            "@bison//:bin/bison",
    #        ],
    #     )
    #
    # Some of the scripts explicitly depend on Python 2.7. Otherwise, we
    # default to using python3. Therefore, both python27 and python3 are
    # runtime dependencies.
    [
      bash
      coreutils
      file
      findutils
      gawk
      gnugrep
      gnused
      gnutar
      gzip
      python27
      python3
      unzip
      which
      zip
    ];

  defaultShellPath = lib.makeBinPath defaultShellUtils;

  platforms = lib.platforms.linux ++ lib.platforms.darwin;

  system = if stdenv.hostPlatform.isDarwin then "darwin" else "linux";
  arch = stdenv.hostPlatform.parsed.cpu.name;

  bazelRC = writeTextFile {
    name = "bazel-rc";
    text = ''
      startup --server_javabase=${runJdk}

      # Can't use 'common'; https://github.com/bazelbuild/bazel/issues/3054
      # Most commands inherit from 'build' anyway.
      build --distdir=${distDir}
      fetch --distdir=${distDir}
      query --distdir=${distDir}

      build --extra_toolchains=@bazel_tools//tools/jdk:nonprebuilt_toolchain_definition
      build --tool_java_runtime_version=local_jdk_11
      build --java_runtime_version=local_jdk_11

      # load default location for the system wide configuration
      try-import /etc/bazel.bazelrc
    '';
  };

in
stdenv.mkDerivation rec {
  pname = "bazel";
  inherit version;

  meta = with lib; {
    homepage = "https://github.com/bazelbuild/bazel/";
    description = "Build tool that builds code quickly and reliably";
    license = licenses.asl20;
    maintainers = lib.teams.bazel.members;
    inherit platforms;
  };

  inherit src;
  inherit sourceRoot;
  patches = [
    # On Darwin, the last argument to gcc is coming up as an empty string. i.e: ''
    # This is breaking the build of any C target. This patch removes the last
    # argument if it's found to be an empty string.
    ../trim-last-argument-to-gcc-if-empty.patch

    # On Darwin, using clang 6 to build fails because of a linker error (see #105573),
    # but using clang 7 fails because libarclite_macosx.a cannot be found when linking
    # the xcode_locator tool.
    # This patch removes using the -fobjc-arc compiler option and makes the code
    # compile without automatic reference counting. Caveat: this leaks memory, but
    # we accept this fact because xcode_locator is only a short-lived process used during the build.
    ./no-arc.patch

    # --experimental_strict_action_env (which may one day become the default
    # see bazelbuild/bazel#2574) hardcodes the default
    # action environment to a non hermetic value (e.g. "/usr/local/bin").
    # This is non hermetic on non-nixos systems. On NixOS, bazel cannot find the required binaries.
    # So we are replacing this bazel paths by defaultShellPath,
    # improving hermeticity and making it work in nixos.
    (substituteAll {
      src = ../strict_action_env.patch;
      strictActionEnvPatch = defaultShellPath;
    })

    (substituteAll {
      src = ./actions_path.patch;
      actionsPathPatch = defaultShellPath;
    })

    # bazel reads its system bazelrc in /etc
    # override this path to a builtin one
    (substituteAll {
      src = ../bazel_rc.patch;
      bazelSystemBazelRCPath = bazelRC;
    })
  ] ++ lib.optional enableNixHacks ../nix-hacks.patch;


  # Additional tests that check bazel’s functionality. Execute
  #
  #     nix-build . -A bazel.tests
  #
  # in the nixpkgs checkout root to exercise them locally.
  passthru.tests =
    let
      runLocal = name: attrs: script:
      let
        attrs' = removeAttrs attrs [ "buildInputs" ];
        buildInputs = [ python3 which ] ++ (attrs.buildInputs or []);
      in
      runCommandCC name ({
        inherit buildInputs;
        preferLocalBuild = true;
        meta.platforms = platforms;
      } // attrs') script;

      # bazel wants to extract itself into $install_dir/install every time it runs,
      # so let’s do that only once.
      extracted = bazelPkg:
        let install_dir =
          # `install_base` field printed by `bazel info`, minus the hash.
          # yes, this path is kinda magic. Sorry.
          "$HOME/.cache/bazel/_bazel_nixbld";
        in runLocal "bazel-extracted-homedir" { passthru.install_dir = install_dir; } ''
            export HOME=$(mktemp -d)
            touch WORKSPACE # yeah, everything sucks
            install_base="$(${bazelPkg}/bin/bazel info | grep install_base)"
            # assert it’s actually below install_dir
            [[ "$install_base" =~ ${install_dir} ]] \
              || (echo "oh no! $install_base but we are \
            trying to copy ${install_dir} to $out instead!"; exit 1)
            cp -R ${install_dir} $out
          '';

      bazelTest = { name, bazelScript, workspaceDir, bazelPkg, buildInputs ? [] }:
        let
          be = extracted bazelPkg;
        in runLocal name { inherit buildInputs; } (
          # skip extraction caching on Darwin, because nobody knows how Darwin works
          (lib.optionalString (!stdenv.hostPlatform.isDarwin) ''
            # set up home with pre-unpacked bazel
            export HOME=$(mktemp -d)
            mkdir -p ${be.install_dir}
            cp -R ${be}/install ${be.install_dir}

            # https://stackoverflow.com/questions/47775668/bazel-how-to-skip-corrupt-installation-on-centos6
            # Bazel checks whether the mtime of the install dir files
            # is >9 years in the future, otherwise it extracts itself again.
            # see PosixFileMTime::IsUntampered in src/main/cpp/util
            # What the hell bazel.
            ${lr}/bin/lr -0 -U ${be.install_dir} | ${xe}/bin/xe -N0 -0 touch --date="9 years 6 months" {}
          '')
          +
          ''
            # Note https://github.com/bazelbuild/bazel/issues/5763#issuecomment-456374609
            # about why to create a subdir for the workspace.
            cp -r ${workspaceDir} wd && chmod u+w wd && cd wd

            ${bazelScript}

            touch $out
          '');

      bazelWithNixHacks = bazel_self.override { enableNixHacks = true; };

      bazel-examples = fetchFromGitHub {
        owner = "bazelbuild";
        repo = "examples";
        rev = "4183fc709c26a00366665e2d60d70521dc0b405d";
        sha256 = "1mm4awx6sa0myiz9j4hwp71rpr7yh8vihf3zm15n2ii6xb82r31k";
      };

    in (if !stdenv.hostPlatform.isDarwin then {
      # `extracted` doesn’t work on darwin
      shebang = callPackage ../shebang-test.nix { inherit runLocal extracted bazelTest distDir; };
    } else {}) // {
      bashTools = callPackage ../bash-tools-test.nix { inherit runLocal bazelTest distDir; };
      cpp = callPackage ../cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; };
      java = callPackage ../java-test.nix { inherit runLocal bazelTest bazel-examples distDir; };
      protobuf = callPackage ../protobuf-test.nix { inherit runLocal bazelTest distDir; };
      pythonBinPath = callPackage ../python-bin-path-test.nix { inherit runLocal bazelTest distDir; };

      bashToolsWithNixHacks = callPackage ../bash-tools-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; };

      cppWithNixHacks = callPackage ../cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; bazel = bazelWithNixHacks; };
      javaWithNixHacks = callPackage ../java-test.nix { inherit runLocal bazelTest bazel-examples distDir; bazel = bazelWithNixHacks; };
      protobufWithNixHacks = callPackage ../protobuf-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; };
      pythonBinPathWithNixHacks = callPackage ../python-bin-path-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; };

      # downstream packages using buildBazelPackage
      # fixed-output hashes of the fetch phase need to be spot-checked manually
      downstream = recurseIntoAttrs ({
        inherit bazel-watcher;
      }
          # dm-sonnet is only packaged for linux
      // (lib.optionalAttrs stdenv.isLinux {
          # TODO(timokau) dm-sonnet is broken currently
          # dm-sonnet-linux = python3.pkgs.dm-sonnet;
      }));
    };

  src_for_updater = stdenv.mkDerivation rec {
    name = "updater-sources";
    inherit src;
    nativeBuildInputs = [ unzip ];
    inherit sourceRoot;
    installPhase = ''
      runHook preInstall

      cp -r . "$out"

      runHook postInstall
    '';
  };
  # update the list of workspace dependencies
  passthru.updater = writeScript "update-bazel-deps.sh" ''
    #!${runtimeShell}
    (cd "${src_for_updater}" &&
        BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
        "${bazel_4}"/bin/bazel \
            query 'kind(http_archive, //external:all) + kind(http_file, //external:all) + kind(distdir_tar, //external:all) + kind(git_repository, //external:all)' \
            --loading_phase_threads=1 \
            --output build) \
    | "${python3}"/bin/python3 "${./update-srcDeps.py}" \
      "${builtins.toString ./src-deps.json}"
  '';

  # Necessary for the tests to pass on Darwin with sandbox enabled.
  # Bazel starts a local server and needs to bind a local address.
  __darwinAllowLocalNetworking = true;

  postPatch = let

    darwinPatches = ''
      bazelLinkFlags () {
        eval set -- "$NIX_LDFLAGS"
        local flag
        for flag in "$@"; do
          printf ' -Wl,%s' "$flag"
        done
      }

      # Disable Bazel's Xcode toolchain detection which would configure compilers
      # and linkers from Xcode instead of from PATH
      export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1

      # Explicitly configure gcov since we don't have it on Darwin, so autodetection fails
      export GCOV=${coreutils}/bin/false

      # Framework search paths aren't added by bintools hook
      # https://github.com/NixOS/nixpkgs/pull/41914
      export NIX_LDFLAGS+=" -F${CoreFoundation}/Library/Frameworks -F${CoreServices}/Library/Frameworks -F${Foundation}/Library/Frameworks"

      # libcxx includes aren't added by libcxx hook
      # https://github.com/NixOS/nixpkgs/pull/41589
      export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem ${lib.getDev libcxx}/include/c++/v1"

      # don't use system installed Xcode to run clang, use Nix clang instead
      sed -i -E "s;/usr/bin/xcrun (--sdk macosx )?clang;${stdenv.cc}/bin/clang $NIX_CFLAGS_COMPILE $(bazelLinkFlags) -framework CoreFoundation;g" \
        scripts/bootstrap/compile.sh \
        src/tools/xcode/realpath/BUILD \
        src/tools/xcode/stdredirect/BUILD \
        tools/osx/BUILD

      substituteInPlace scripts/bootstrap/compile.sh --replace ' -mmacosx-version-min=10.9' ""

      # nixpkgs's libSystem cannot use pthread headers directly, must import GCD headers instead
      sed -i -e "/#include <pthread\/spawn.h>/i #include <dispatch/dispatch.h>" src/main/cpp/blaze_util_darwin.cc<commit_msg>bazel_5: remove patching of tools/xcode/realpath/BUILD and tools/xcode/stdredirect/BUILD in darwinPatches as these files do not exist anymore<commit_after>{ stdenv, callPackage, lib, fetchurl, fetchpatch, fetchFromGitHub, installShellFiles
, runCommand, runCommandCC, makeWrapper, recurseIntoAttrs
# this package (through the fixpoint glass)
, bazel_self
# needed only for the updater
, bazel_4
, lr, xe, zip, unzip, bash, writeCBin, coreutils
, which, gawk, gnused, gnutar, gnugrep, gzip, findutils
# updater
, python27, python3, writeScript
# Apple dependencies
, cctools, libcxx, CoreFoundation, CoreServices, Foundation
# Allow to independently override the jdks used to build and run respectively
, buildJdk, runJdk
, runtimeShell
# Downstream packages for tests
, bazel-watcher
# Always assume all markers valid (this is needed because we remove markers; they are non-deterministic).
# Also, don't clean up environment variables (so that NIX_ environment variables are passed to compilers).
, enableNixHacks ? false
, gcc-unwrapped
, autoPatchelfHook
, file
, substituteAll
, writeTextFile
}:

let
  version = "5.0.0";
  sourceRoot = ".";

  src = fetchurl {
    url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip";
    sha256 = "By3WLSN9vBHgusAuEY2MLbTQujugnxoOseKkYPuEGds=";
  };

  # Update with `eval $(nix-build -A bazel.updater)`,
  # then add new dependencies from the dict in ./src-deps.json as required.
  srcDeps = lib.attrsets.attrValues srcDepsSet;
  srcDepsSet =
    let
      srcs = lib.importJSON ./src-deps.json;
      toFetchurl = d: lib.attrsets.nameValuePair d.name (fetchurl {
        urls = d.urls;
        sha256 = d.sha256;
        });
        in builtins.listToAttrs (map toFetchurl [
      srcs.desugar_jdk_libs
      srcs.io_bazel_skydoc
      srcs.bazel_skylib
      srcs.bazelci_rules
      srcs.io_bazel_rules_sass
      srcs.platforms
      srcs."remote_java_tools_for_testing"
      srcs."coverage_output_generator-v2.5.zip"
      srcs.build_bazel_rules_nodejs
      srcs."android_tools_pkg-0.23.0.tar.gz"
      srcs.bazel_toolchains
      srcs.com_github_grpc_grpc
      srcs.upb
      srcs.com_google_protobuf
      srcs.rules_pkg
      srcs.rules_cc
      srcs.rules_java
      srcs.rules_proto
      srcs.com_google_absl
      srcs.com_googlesource_code_re2
      srcs.com_github_cares_cares
      ]);

  distDir = runCommand "bazel-deps" {} ''
    mkdir -p $out
    for i in ${builtins.toString srcDeps}; do cp $i $out/$(stripHash $i); done
  '';

  defaultShellUtils =
    # Keep this list conservative. For more exotic tools, prefer to use
    # @rules_nixpkgs to pull in tools from the nix repository. Example:
    #
    # WORKSPACE:
    #
    #     nixpkgs_git_repository(
    #         name = "nixpkgs",
    #         revision = "def5124ec8367efdba95a99523dd06d918cb0ae8",
    #     )
    #
    #     # This defines an external Bazel workspace.
    #     nixpkgs_package(
    #         name = "bison",
    #         repositories = { "nixpkgs": "@nixpkgs//:default.nix" },
    #     )
    #
    # some/BUILD.bazel:
    #
    #     genrule(
    #        ...
    #        cmd = "$(location @bison//:bin/bison) -other -args",
    #        tools = [
    #            ...
    #            "@bison//:bin/bison",
    #        ],
    #     )
    #
    # Some of the scripts explicitly depend on Python 2.7. Otherwise, we
    # default to using python3. Therefore, both python27 and python3 are
    # runtime dependencies.
    [
      bash
      coreutils
      file
      findutils
      gawk
      gnugrep
      gnused
      gnutar
      gzip
      python27
      python3
      unzip
      which
      zip
    ];

  defaultShellPath = lib.makeBinPath defaultShellUtils;

  platforms = lib.platforms.linux ++ lib.platforms.darwin;

  system = if stdenv.hostPlatform.isDarwin then "darwin" else "linux";
  arch = stdenv.hostPlatform.parsed.cpu.name;

  bazelRC = writeTextFile {
    name = "bazel-rc";
    text = ''
      startup --server_javabase=${runJdk}

      # Can't use 'common'; https://github.com/bazelbuild/bazel/issues/3054
      # Most commands inherit from 'build' anyway.
      build --distdir=${distDir}
      fetch --distdir=${distDir}
      query --distdir=${distDir}

      build --extra_toolchains=@bazel_tools//tools/jdk:nonprebuilt_toolchain_definition
      build --tool_java_runtime_version=local_jdk_11
      build --java_runtime_version=local_jdk_11

      # load default location for the system wide configuration
      try-import /etc/bazel.bazelrc
    '';
  };

in
stdenv.mkDerivation rec {
  pname = "bazel";
  inherit version;

  meta = with lib; {
    homepage = "https://github.com/bazelbuild/bazel/";
    description = "Build tool that builds code quickly and reliably";
    license = licenses.asl20;
    maintainers = lib.teams.bazel.members;
    inherit platforms;
  };

  inherit src;
  inherit sourceRoot;
  patches = [
    # On Darwin, the last argument to gcc is coming up as an empty string. i.e: ''
    # This is breaking the build of any C target. This patch removes the last
    # argument if it's found to be an empty string.
    ../trim-last-argument-to-gcc-if-empty.patch

    # On Darwin, using clang 6 to build fails because of a linker error (see #105573),
    # but using clang 7 fails because libarclite_macosx.a cannot be found when linking
    # the xcode_locator tool.
    # This patch removes using the -fobjc-arc compiler option and makes the code
    # compile without automatic reference counting. Caveat: this leaks memory, but
    # we accept this fact because xcode_locator is only a short-lived process used during the build.
    ./no-arc.patch

    # --experimental_strict_action_env (which may one day become the default
    # see bazelbuild/bazel#2574) hardcodes the default
    # action environment to a non hermetic value (e.g. "/usr/local/bin").
    # This is non hermetic on non-nixos systems. On NixOS, bazel cannot find the required binaries.
    # So we are replacing this bazel paths by defaultShellPath,
    # improving hermeticity and making it work in nixos.
    (substituteAll {
      src = ../strict_action_env.patch;
      strictActionEnvPatch = defaultShellPath;
    })

    (substituteAll {
      src = ./actions_path.patch;
      actionsPathPatch = defaultShellPath;
    })

    # bazel reads its system bazelrc in /etc
    # override this path to a builtin one
    (substituteAll {
      src = ../bazel_rc.patch;
      bazelSystemBazelRCPath = bazelRC;
    })
  ] ++ lib.optional enableNixHacks ../nix-hacks.patch;


  # Additional tests that check bazel’s functionality. Execute
  #
  #     nix-build . -A bazel.tests
  #
  # in the nixpkgs checkout root to exercise them locally.
  passthru.tests =
    let
      runLocal = name: attrs: script:
      let
        attrs' = removeAttrs attrs [ "buildInputs" ];
        buildInputs = [ python3 which ] ++ (attrs.buildInputs or []);
      in
      runCommandCC name ({
        inherit buildInputs;
        preferLocalBuild = true;
        meta.platforms = platforms;
      } // attrs') script;

      # bazel wants to extract itself into $install_dir/install every time it runs,
      # so let’s do that only once.
      extracted = bazelPkg:
        let install_dir =
          # `install_base` field printed by `bazel info`, minus the hash.
          # yes, this path is kinda magic. Sorry.
          "$HOME/.cache/bazel/_bazel_nixbld";
        in runLocal "bazel-extracted-homedir" { passthru.install_dir = install_dir; } ''
            export HOME=$(mktemp -d)
            touch WORKSPACE # yeah, everything sucks
            install_base="$(${bazelPkg}/bin/bazel info | grep install_base)"
            # assert it’s actually below install_dir
            [[ "$install_base" =~ ${install_dir} ]] \
              || (echo "oh no! $install_base but we are \
            trying to copy ${install_dir} to $out instead!"; exit 1)
            cp -R ${install_dir} $out
          '';

      bazelTest = { name, bazelScript, workspaceDir, bazelPkg, buildInputs ? [] }:
        let
          be = extracted bazelPkg;
        in runLocal name { inherit buildInputs; } (
          # skip extraction caching on Darwin, because nobody knows how Darwin works
          (lib.optionalString (!stdenv.hostPlatform.isDarwin) ''
            # set up home with pre-unpacked bazel
            export HOME=$(mktemp -d)
            mkdir -p ${be.install_dir}
            cp -R ${be}/install ${be.install_dir}

            # https://stackoverflow.com/questions/47775668/bazel-how-to-skip-corrupt-installation-on-centos6
            # Bazel checks whether the mtime of the install dir files
            # is >9 years in the future, otherwise it extracts itself again.
            # see PosixFileMTime::IsUntampered in src/main/cpp/util
            # What the hell bazel.
            ${lr}/bin/lr -0 -U ${be.install_dir} | ${xe}/bin/xe -N0 -0 touch --date="9 years 6 months" {}
          '')
          +
          ''
            # Note https://github.com/bazelbuild/bazel/issues/5763#issuecomment-456374609
            # about why to create a subdir for the workspace.
            cp -r ${workspaceDir} wd && chmod u+w wd && cd wd

            ${bazelScript}

            touch $out
          '');

      bazelWithNixHacks = bazel_self.override { enableNixHacks = true; };

      bazel-examples = fetchFromGitHub {
        owner = "bazelbuild";
        repo = "examples";
        rev = "4183fc709c26a00366665e2d60d70521dc0b405d";
        sha256 = "1mm4awx6sa0myiz9j4hwp71rpr7yh8vihf3zm15n2ii6xb82r31k";
      };

    in (if !stdenv.hostPlatform.isDarwin then {
      # `extracted` doesn’t work on darwin
      shebang = callPackage ../shebang-test.nix { inherit runLocal extracted bazelTest distDir; };
    } else {}) // {
      bashTools = callPackage ../bash-tools-test.nix { inherit runLocal bazelTest distDir; };
      cpp = callPackage ../cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; };
      java = callPackage ../java-test.nix { inherit runLocal bazelTest bazel-examples distDir; };
      protobuf = callPackage ../protobuf-test.nix { inherit runLocal bazelTest distDir; };
      pythonBinPath = callPackage ../python-bin-path-test.nix { inherit runLocal bazelTest distDir; };

      bashToolsWithNixHacks = callPackage ../bash-tools-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; };

      cppWithNixHacks = callPackage ../cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; bazel = bazelWithNixHacks; };
      javaWithNixHacks = callPackage ../java-test.nix { inherit runLocal bazelTest bazel-examples distDir; bazel = bazelWithNixHacks; };
      protobufWithNixHacks = callPackage ../protobuf-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; };
      pythonBinPathWithNixHacks = callPackage ../python-bin-path-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; };

      # downstream packages using buildBazelPackage
      # fixed-output hashes of the fetch phase need to be spot-checked manually
      downstream = recurseIntoAttrs ({
        inherit bazel-watcher;
      }
          # dm-sonnet is only packaged for linux
      // (lib.optionalAttrs stdenv.isLinux {
          # TODO(timokau) dm-sonnet is broken currently
          # dm-sonnet-linux = python3.pkgs.dm-sonnet;
      }));
    };

  src_for_updater = stdenv.mkDerivation rec {
    name = "updater-sources";
    inherit src;
    nativeBuildInputs = [ unzip ];
    inherit sourceRoot;
    installPhase = ''
      runHook preInstall

      cp -r . "$out"

      runHook postInstall
    '';
  };
  # update the list of workspace dependencies
  passthru.updater = writeScript "update-bazel-deps.sh" ''
    #!${runtimeShell}
    (cd "${src_for_updater}" &&
        BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
        "${bazel_4}"/bin/bazel \
            query 'kind(http_archive, //external:all) + kind(http_file, //external:all) + kind(distdir_tar, //external:all) + kind(git_repository, //external:all)' \
            --loading_phase_threads=1 \
            --output build) \
    | "${python3}"/bin/python3 "${./update-srcDeps.py}" \
      "${builtins.toString ./src-deps.json}"
  '';

  # Necessary for the tests to pass on Darwin with sandbox enabled.
  # Bazel starts a local server and needs to bind a local address.
  __darwinAllowLocalNetworking = true;

  postPatch = let

    darwinPatches = ''
      bazelLinkFlags () {
        eval set -- "$NIX_LDFLAGS"
        local flag
        for flag in "$@"; do
          printf ' -Wl,%s' "$flag"
        done
      }

      # Disable Bazel's Xcode toolchain detection which would configure compilers
      # and linkers from Xcode instead of from PATH
      export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1

      # Explicitly configure gcov since we don't have it on Darwin, so autodetection fails
      export GCOV=${coreutils}/bin/false

      # Framework search paths aren't added by bintools hook
      # https://github.com/NixOS/nixpkgs/pull/41914
      export NIX_LDFLAGS+=" -F${CoreFoundation}/Library/Frameworks -F${CoreServices}/Library/Frameworks -F${Foundation}/Library/Frameworks"

      # libcxx includes aren't added by libcxx hook
      # https://github.com/NixOS/nixpkgs/pull/41589
      export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem ${lib.getDev libcxx}/include/c++/v1"

      # don't use system installed Xcode to run clang, use Nix clang instead
      sed -i -E "s;/usr/bin/xcrun (--sdk macosx )?clang;${stdenv.cc}/bin/clang $NIX_CFLAGS_COMPILE $(bazelLinkFlags) -framework CoreFoundation;g" \
        scripts/bootstrap/compile.sh \
        tools/osx/BUILD

      substituteInPlace scripts/bootstrap/compile.sh --replace ' -mmacosx-version-min=10.9' ""

      # nixpkgs's libSystem cannot use pthread headers directly, must import GCD headers instead
      sed -i -e "/#include <pthread\/spawn.h>/i #include <dispatch/dispatch.h>" src/main/cpp/blaze_util_darwin.cc
{'fraction_non_alphanumeric': 0.10119835449830084, 'fraction_numerical': 0.012985154712931497, 'mean_word_length': 2.5942401645667266, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3632441', 'n_tokens_mistral': 9447, 'n_tokens_neox': 8937, 'n_words': 2314}