<commit_before># This expression returns a list of all fetchurl calls used by ‘expr’.

with import ../.. { };
with lib;

{ expr }:

let

  root = expr;

  uniqueUrls = map (x: x.file) (genericClosure {
    startSet = map (file: { key = file.url; inherit file; }) urls;
    operator = const [ ];
  });

  urls = map (drv: { url = head drv.urls; hash = drv.outputHash; type = drv.outputHashAlgo; }) fetchurlDependencies;

  fetchurlDependencies =
    filter
      (drv: drv.outputHash or "" != "" && drv.outputHashMode or "flat" == "flat"
          && drv.postFetch or "" == "" && drv ? urls)
      dependencies;

  dependencies = map (x: x.value) (genericClosure {
    startSet = map keyDrv (derivationsIn' root);
    operator = { key, value }: map keyDrv (immediateDependenciesOf value);
  });

  derivationsIn' = x:
    if !canEval x then []
    else if isDerivation x then optional (canEval x.drvPath) x
    else if isList x then concatLists (map derivationsIn' x)
    else if isAttrs x then concatLists (mapAttrsToList (n: v: derivationsIn' v) x)
    else [ ];

  keyDrv = drv: if canEval drv.drvPath then { key = drv.drvPath; value = drv; } else { };

  immediateDependenciesOf = drv:
    concatLists (mapAttrsToList (n: v: derivationsIn v) (removeAttrs drv ["meta" "passthru"]));

  derivationsIn = x:
    if !canEval x then []
    else if isDerivation x then optional (canEval x.drvPath) x
    else if isList x then concatLists (map derivationsIn x)
    else [ ];

  canEval = val: (builtins.tryEval val).success;

in uniqueUrls
<commit_msg>find-tarballs.nix: Handle stdenv bootstrap fetchurl<commit_after># This expression returns a list of all fetchurl calls used by ‘expr’.

with import ../.. { };
with lib;

{ expr }:

let

  root = expr;

  uniqueUrls = map (x: x.file) (genericClosure {
    startSet = map (file: { key = file.url; inherit file; }) urls;
    operator = const [ ];
  });

  urls = map (drv: { url = head (drv.urls or [ drv.url ]); hash = drv.outputHash; type = drv.outputHashAlgo; }) fetchurlDependencies;

  fetchurlDependencies =
    filter
      (drv: drv.outputHash or "" != "" && drv.outputHashMode or "flat" == "flat"
          && drv.postFetch or "" == "" && (drv ? url || drv ? urls))
      dependencies;

  dependencies = map (x: x.value) (genericClosure {
    startSet = map keyDrv (derivationsIn' root);
    operator = { key, value }: map keyDrv (immediateDependenciesOf value);
  });

  derivationsIn' = x:
    if !canEval x then []
    else if isDerivation x then optional (canEval x.drvPath) x
    else if isList x then concatLists (map derivationsIn' x)
    else if isAttrs x then concatLists (mapAttrsToList (n: v: derivationsIn' v) x)
    else [ ];

  keyDrv = drv: if canEval drv.drvPath then { key = drv.drvPath; value = drv; } else { };

  immediateDependenciesOf = drv:
    concatLists (mapAttrsToList (n: v: derivationsIn v) (removeAttrs drv ["meta" "passthru"]));

  derivationsIn = x:
    if !canEval x then []
    else if isDerivation x then optional (canEval x.drvPath) x
    else if isList x then concatLists (map derivationsIn x)
    else [ ];

  canEval = val: (builtins.tryEval val).success;

in uniqueUrls

{'fraction_non_alphanumeric': 0.12420986093552465, 'fraction_numerical': 0.0, 'mean_word_length': 3.208776595744681, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5699008', 'n_tokens_mistral': 1115, 'n_tokens_neox': 1065, 'n_words': 346}