<commit_before>{ lib, config, stdenv }:

rec {
  # `mkDerivation` wraps the builtin `derivation` function to
  # produce derivations that use this stdenv and its shell.
  #
  # See also:
  #
  # * https://nixos.org/nixpkgs/manual/#sec-using-stdenv
  #   Details on how to use this mkDerivation function
  #
  # * https://nixos.org/nix/manual/#ssec-derivation
  #   Explanation about derivations in general
  mkDerivation =
    { name ? ""

    # These types of dependencies are all exhaustively documented in
    # the "Specifying Dependencies" section of the "Standard
    # Environment" chapter of the Nixpkgs manual.

    # TODO(@Ericson2314): Stop using legacy dep attribute names

    #                           host offset -> target offset
    , depsBuildBuild              ? [] # -1 -> -1
    , depsBuildBuildPropagated    ? [] # -1 -> -1
    , nativeBuildInputs           ? [] # -1 ->  0  N.B. Legacy name
    , propagatedNativeBuildInputs ? [] # -1 ->  0  N.B. Legacy name
    , depsBuildTarget             ? [] # -1 ->  1
    , depsBuildTargetPropagated   ? [] # -1 ->  1

    , depsHostHost                ? [] #  0 ->  0
    , depsHostHostPropagated      ? [] #  0 ->  0
    , buildInputs                 ? [] #  0 ->  1  N.B. Legacy name
    , propagatedBuildInputs       ? [] #  0 ->  1  N.B. Legacy name

    , depsTargetTarget            ? [] #  1 ->  1
    , depsTargetTargetPropagated  ? [] #  1 ->  1

    , checkInputs                 ? []
    , installCheckInputs          ? []

    # Configure Phase
    , configureFlags ? []
    , cmakeFlags ? []
    , # Target is not included by default because most programs don't care.
      # Including it then would cause needless mass rebuilds.
      #
      # TODO(@Ericson2314): Make [ "build" "host" ] always the default.
      configurePlatforms ? lib.optionals
        (stdenv.hostPlatform != stdenv.buildPlatform)
        [ "build" "host" ]

    # TODO(@Ericson2314): Make unconditional / resolve #33599
    # Check phase
    , doCheck ? config.doCheckByDefault or false

    # TODO(@Ericson2314): Make unconditional / resolve #33599
    # InstallCheck phase
    , doInstallCheck ? config.doCheckByDefault or false

    , # TODO(@Ericson2314): Make always true and remove
      strictDeps ? stdenv.hostPlatform != stdenv.buildPlatform
    , meta ? {}
    , passthru ? {}
    , pos ? # position used in error messages and for meta.position
        (if attrs.meta.description or null != null
          then builtins.unsafeGetAttrPos "description" attrs.meta
          else if attrs.version or null != null
          then builtins.unsafeGetAttrPos "version" attrs
          else builtins.unsafeGetAttrPos "name" attrs)
    , separateDebugInfo ? false
    , outputs ? [ "out" ]
    , __impureHostDeps ? []
    , __propagatedImpureHostDeps ? []
    , sandboxProfile ? ""
    , propagatedSandboxProfile ? ""

    , hardeningEnable ? []
    , hardeningDisable ? []

    , ... } @ attrs:

    let
      computedName = if name != "" then name else "${attrs.pname}-${attrs.version}";

      # TODO(@oxij, @Ericson2314): This is here to keep the old semantics, remove when
      # no package has `doCheck = true`.
      doCheck' = doCheck && stdenv.hostPlatform == stdenv.buildPlatform;
      doInstallCheck' = doInstallCheck && stdenv.hostPlatform == stdenv.buildPlatform;

      separateDebugInfo' = separateDebugInfo && stdenv.hostPlatform.isLinux;
      outputs' = outputs ++ lib.optional separateDebugInfo' "debug";

      fixedOutputDrv = attrs ? outputHash;
      noNonNativeDeps = builtins.length (depsBuildTarget ++ depsBuildTargetPropagated
                                      ++ depsHostHost ++ depsHostHostPropagated
                                      ++ buildInputs ++ propagatedBuildInputs
                                      ++ depsTargetTarget ++ depsTargetTargetPropagated) == 0;
      dontAddHostSuffix = attrs ? outputHash && !noNonNativeDeps || stdenv.cc == null;
      supportedHardeningFlags = [ "fortify" "stackprotector" "pie" "pic" "strictoverflow" "format" "relro" "bindnow" ];
      defaultHardeningFlags = if stdenv.hostPlatform.isMusl
                              then supportedHardeningFlags
                              else lib.remove "pie" supportedHardeningFlags;
      enabledHardeningOptions =
        if builtins.elem "all" hardeningDisable
        then []
        else lib.subtractLists hardeningDisable (defaultHardeningFlags ++ hardeningEnable);
      # hardeningDisable additionally supports "all".
      erroneousHardeningFlags = lib.subtractLists supportedHardeningFlags (hardeningEnable ++ lib.remove "all" hardeningDisable);
    in if builtins.length erroneousHardeningFlags != 0
    then abort ("mkDerivation was called with unsupported hardening flags: " + lib.generators.toPretty {} {
      inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
    })
    else let
      doCheck = doCheck';
      doInstallCheck = doInstallCheck';

      outputs = outputs';

      references = nativeBuildInputs ++ buildInputs
                ++ propagatedNativeBuildInputs ++ propagatedBuildInputs;

      dependencies = map (map lib.chooseDevOutputs) [
        [
          (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
          (map (drv: drv.nativeDrv or drv) nativeBuildInputs
             ++ lib.optional separateDebugInfo' ../../build-support/setup-hooks/separate-debug-info.sh
             ++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
          (map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)
        ]
        [
          (map (drv: drv.__spliced.hostHost or drv) depsHostHost)
          (map (drv: drv.crossDrv or drv) (buildInputs
             ++ lib.optionals doCheck checkInputs
             ++ lib.optionals doInstallCheck' installCheckInputs))
        ]
        [
          (map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)
        ]
      ];
      propagatedDependencies = map (map lib.chooseDevOutputs) [
        [
          (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuildPropagated)
          (map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
          (map (drv: drv.__spliced.buildTarget or drv) depsBuildTargetPropagated)
        ]
        [
          (map (drv: drv.__spliced.hostHost or drv) depsHostHostPropagated)
          (map (drv: drv.crossDrv or drv) propagatedBuildInputs)
        ]
        [
          (map (drv: drv.__spliced.targetTarget or drv) depsTargetTargetPropagated)
        ]
      ];

      computedSandboxProfile =
        lib.concatMap (input: input.__propagatedSandboxProfile or [])
          (stdenv.extraNativeBuildInputs
           ++ stdenv.extraBuildInputs
           ++ lib.concatLists dependencies);

      computedPropagatedSandboxProfile =
        lib.concatMap (input: input.__propagatedSandboxProfile or [])
          (lib.concatLists propagatedDependencies);

      computedImpureHostDeps =
        lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
          (stdenv.extraNativeBuildInputs
           ++ stdenv.extraBuildInputs
           ++ lib.concatLists dependencies));

      computedPropagatedImpureHostDeps =
        lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
          (lib.concatLists propagatedDependencies));

      derivationArg =
        (removeAttrs attrs
          ["meta" "passthru" "pos"
           "checkInputs" "installCheckInputs"
           "__impureHostDeps" "__propagatedImpureHostDeps"
           "sandboxProfile" "propagatedSandboxProfile"])
        // {
          # A hack to make `nix-env -qa` and `nix search` ignore broken packages.
          # TODO(@oxij): remove this assert when something like NixOS/nix#1771 gets merged into nix.
          name = assert validity.handled; computedName + lib.optionalString
            # Fixed-output derivations like source tarballs shouldn't get a host
            # suffix. But we have some weird ones with run-time deps that are
            # just used for their side-affects. Those might as well since the
            # hash can't be the same. See #32986.
            (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix)
            ("-" + stdenv.hostPlatform.config);

          builder = attrs.realBuilder or stdenv.shell;
          args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)];
          inherit stdenv;

          # The `system` attribute of a derivation has special meaning to Nix.
          # Derivations set it to choose what sort of machine could be used to
          # execute the build, The build platform entirely determines this,
          # indeed more finely than Nix knows or cares about. The `system`
          # attribute of `buildPlatfom` matches Nix's degree of specificity.
          # exactly.
          inherit (stdenv.buildPlatform) system;

          userHook = config.stdenv.userHook or null;
          __ignoreNulls = true;

          inherit strictDeps;

          depsBuildBuild              = lib.elemAt (lib.elemAt dependencies 0) 0;
          nativeBuildInputs           = lib.elemAt (lib.elemAt dependencies 0) 1;
          depsBuildTarget             = lib.elemAt (lib.elemAt dependencies 0) 2;
          depsHostHost                = lib.elemAt (lib.elemAt dependencies 1) 0;
          buildInputs                 = lib.elemAt (lib.elemAt dependencies 1) 1;<commit_msg>stdenv/make-derivation: don't hide broken packages<commit_after>{ lib, config, stdenv }:

rec {
  # `mkDerivation` wraps the builtin `derivation` function to
  # produce derivations that use this stdenv and its shell.
  #
  # See also:
  #
  # * https://nixos.org/nixpkgs/manual/#sec-using-stdenv
  #   Details on how to use this mkDerivation function
  #
  # * https://nixos.org/nix/manual/#ssec-derivation
  #   Explanation about derivations in general
  mkDerivation =
    { name ? ""

    # These types of dependencies are all exhaustively documented in
    # the "Specifying Dependencies" section of the "Standard
    # Environment" chapter of the Nixpkgs manual.

    # TODO(@Ericson2314): Stop using legacy dep attribute names

    #                           host offset -> target offset
    , depsBuildBuild              ? [] # -1 -> -1
    , depsBuildBuildPropagated    ? [] # -1 -> -1
    , nativeBuildInputs           ? [] # -1 ->  0  N.B. Legacy name
    , propagatedNativeBuildInputs ? [] # -1 ->  0  N.B. Legacy name
    , depsBuildTarget             ? [] # -1 ->  1
    , depsBuildTargetPropagated   ? [] # -1 ->  1

    , depsHostHost                ? [] #  0 ->  0
    , depsHostHostPropagated      ? [] #  0 ->  0
    , buildInputs                 ? [] #  0 ->  1  N.B. Legacy name
    , propagatedBuildInputs       ? [] #  0 ->  1  N.B. Legacy name

    , depsTargetTarget            ? [] #  1 ->  1
    , depsTargetTargetPropagated  ? [] #  1 ->  1

    , checkInputs                 ? []
    , installCheckInputs          ? []

    # Configure Phase
    , configureFlags ? []
    , cmakeFlags ? []
    , # Target is not included by default because most programs don't care.
      # Including it then would cause needless mass rebuilds.
      #
      # TODO(@Ericson2314): Make [ "build" "host" ] always the default.
      configurePlatforms ? lib.optionals
        (stdenv.hostPlatform != stdenv.buildPlatform)
        [ "build" "host" ]

    # TODO(@Ericson2314): Make unconditional / resolve #33599
    # Check phase
    , doCheck ? config.doCheckByDefault or false

    # TODO(@Ericson2314): Make unconditional / resolve #33599
    # InstallCheck phase
    , doInstallCheck ? config.doCheckByDefault or false

    , # TODO(@Ericson2314): Make always true and remove
      strictDeps ? stdenv.hostPlatform != stdenv.buildPlatform
    , meta ? {}
    , passthru ? {}
    , pos ? # position used in error messages and for meta.position
        (if attrs.meta.description or null != null
          then builtins.unsafeGetAttrPos "description" attrs.meta
          else if attrs.version or null != null
          then builtins.unsafeGetAttrPos "version" attrs
          else builtins.unsafeGetAttrPos "name" attrs)
    , separateDebugInfo ? false
    , outputs ? [ "out" ]
    , __impureHostDeps ? []
    , __propagatedImpureHostDeps ? []
    , sandboxProfile ? ""
    , propagatedSandboxProfile ? ""

    , hardeningEnable ? []
    , hardeningDisable ? []

    , ... } @ attrs:

    let
      computedName = if name != "" then name else "${attrs.pname}-${attrs.version}";

      # TODO(@oxij, @Ericson2314): This is here to keep the old semantics, remove when
      # no package has `doCheck = true`.
      doCheck' = doCheck && stdenv.hostPlatform == stdenv.buildPlatform;
      doInstallCheck' = doInstallCheck && stdenv.hostPlatform == stdenv.buildPlatform;

      separateDebugInfo' = separateDebugInfo && stdenv.hostPlatform.isLinux;
      outputs' = outputs ++ lib.optional separateDebugInfo' "debug";

      fixedOutputDrv = attrs ? outputHash;
      noNonNativeDeps = builtins.length (depsBuildTarget ++ depsBuildTargetPropagated
                                      ++ depsHostHost ++ depsHostHostPropagated
                                      ++ buildInputs ++ propagatedBuildInputs
                                      ++ depsTargetTarget ++ depsTargetTargetPropagated) == 0;
      dontAddHostSuffix = attrs ? outputHash && !noNonNativeDeps || stdenv.cc == null;
      supportedHardeningFlags = [ "fortify" "stackprotector" "pie" "pic" "strictoverflow" "format" "relro" "bindnow" ];
      defaultHardeningFlags = if stdenv.hostPlatform.isMusl
                              then supportedHardeningFlags
                              else lib.remove "pie" supportedHardeningFlags;
      enabledHardeningOptions =
        if builtins.elem "all" hardeningDisable
        then []
        else lib.subtractLists hardeningDisable (defaultHardeningFlags ++ hardeningEnable);
      # hardeningDisable additionally supports "all".
      erroneousHardeningFlags = lib.subtractLists supportedHardeningFlags (hardeningEnable ++ lib.remove "all" hardeningDisable);
    in if builtins.length erroneousHardeningFlags != 0
    then abort ("mkDerivation was called with unsupported hardening flags: " + lib.generators.toPretty {} {
      inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
    })
    else let
      doCheck = doCheck';
      doInstallCheck = doInstallCheck';

      outputs = outputs';

      references = nativeBuildInputs ++ buildInputs
                ++ propagatedNativeBuildInputs ++ propagatedBuildInputs;

      dependencies = map (map lib.chooseDevOutputs) [
        [
          (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
          (map (drv: drv.nativeDrv or drv) nativeBuildInputs
             ++ lib.optional separateDebugInfo' ../../build-support/setup-hooks/separate-debug-info.sh
             ++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
          (map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)
        ]
        [
          (map (drv: drv.__spliced.hostHost or drv) depsHostHost)
          (map (drv: drv.crossDrv or drv) (buildInputs
             ++ lib.optionals doCheck checkInputs
             ++ lib.optionals doInstallCheck' installCheckInputs))
        ]
        [
          (map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)
        ]
      ];
      propagatedDependencies = map (map lib.chooseDevOutputs) [
        [
          (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuildPropagated)
          (map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
          (map (drv: drv.__spliced.buildTarget or drv) depsBuildTargetPropagated)
        ]
        [
          (map (drv: drv.__spliced.hostHost or drv) depsHostHostPropagated)
          (map (drv: drv.crossDrv or drv) propagatedBuildInputs)
        ]
        [
          (map (drv: drv.__spliced.targetTarget or drv) depsTargetTargetPropagated)
        ]
      ];

      computedSandboxProfile =
        lib.concatMap (input: input.__propagatedSandboxProfile or [])
          (stdenv.extraNativeBuildInputs
           ++ stdenv.extraBuildInputs
           ++ lib.concatLists dependencies);

      computedPropagatedSandboxProfile =
        lib.concatMap (input: input.__propagatedSandboxProfile or [])
          (lib.concatLists propagatedDependencies);

      computedImpureHostDeps =
        lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
          (stdenv.extraNativeBuildInputs
           ++ stdenv.extraBuildInputs
           ++ lib.concatLists dependencies));

      computedPropagatedImpureHostDeps =
        lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
          (lib.concatLists propagatedDependencies));

      derivationArg =
        (removeAttrs attrs
          ["meta" "passthru" "pos"
           "checkInputs" "installCheckInputs"
           "__impureHostDeps" "__propagatedImpureHostDeps"
           "sandboxProfile" "propagatedSandboxProfile"])
        // {
          name = computedName + lib.optionalString
            # Fixed-output derivations like source tarballs shouldn't get a host
            # suffix. But we have some weird ones with run-time deps that are
            # just used for their side-affects. Those might as well since the
            # hash can't be the same. See #32986.
            (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix)
            ("-" + stdenv.hostPlatform.config);

          builder = attrs.realBuilder or stdenv.shell;
          args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)];
          inherit stdenv;

          # The `system` attribute of a derivation has special meaning to Nix.
          # Derivations set it to choose what sort of machine could be used to
          # execute the build, The build platform entirely determines this,
          # indeed more finely than Nix knows or cares about. The `system`
          # attribute of `buildPlatfom` matches Nix's degree of specificity.
          # exactly.
          inherit (stdenv.buildPlatform) system;

          userHook = config.stdenv.userHook or null;
          __ignoreNulls = true;

          inherit strictDeps;

          depsBuildBuild              = lib.elemAt (lib.elemAt dependencies 0) 0;
          nativeBuildInputs           = lib.elemAt (lib.elemAt dependencies 0) 1;
          depsBuildTarget             = lib.elemAt (lib.elemAt dependencies 0) 2;
          depsHostHost                = lib.elemAt (lib.elemAt dependencies 1) 0;
          buildInputs                 = lib.elemAt (lib.elemAt dependencies 1) 1;
{'fraction_non_alphanumeric': 0.09297619681559188, 'fraction_numerical': 0.00820064966185633, 'mean_word_length': 2.3069202324352878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7645698', 'n_tokens_mistral': 5519, 'n_tokens_neox': 5289, 'n_words': 1349}