<commit_before># General list operations.

with import ./trivial.nix;

rec {

  inherit (builtins) head tail length isList elemAt concatLists filter elem;


  # Create a list consisting of a single element.  `singleton x' is
  # sometimes more convenient with respect to indentation than `[x]'
  # when x spans multiple lines.
  singleton = x: [x];


  # "Fold" a binary function `op' between successive elements of
  # `list' with `nul' as the starting value, i.e., `fold op nul [x_1
  # x_2 ... x_n] == op x_1 (op x_2 ... (op x_n nul))'.  (This is
  # Haskell's foldr).
  fold = op: nul: list:
    let
      len = length list;
      fold' = n:
        if n == len
        then nul
        else op (elemAt list n) (fold' (n + 1));
    in fold' 0;

  # Left fold: `fold op nul [x_1 x_2 ... x_n] == op (... (op (op nul
  # x_1) x_2) ... x_n)'.
  foldl = op: nul: list:
    let
      len = length list;
      foldl' = n:
        if n == -1
        then nul
        else op (foldl' (n - 1)) (elemAt list n);
    in foldl' (length list - 1);


  # Strict version of foldl.
  foldl' = builtins.foldl' or foldl;


  # map with index: `imap (i: v: "${v}-${toString i}") ["a" "b"] ==
  # ["a-1" "b-2"]'
  imap = f: list:
    let
      len = length list;
      imap' = n:
        if n == len
          then []
          else [ (f (n + 1) (elemAt list n)) ] ++ imap' (n + 1);
    in imap' 0;


  # Map and concatenate the result.
  concatMap = f: list: concatLists (map f list);


  # Flatten the argument into a single list; that is, nested lists are
  # spliced into the top-level lists.  E.g., `flatten [1 [2 [3] 4] 5]
  # == [1 2 3 4 5]' and `flatten 1 == [1]'.
  flatten = x:
    if isList x
    then foldl' (x: y: x ++ (flatten y)) [] x
    else [x];


  # Remove elements equal to 'e' from a list.  Useful for buildInputs.
  remove = e: filter (x: x != e);


  # Find the sole element in the list matching the specified
  # predicate, returns `default' if no such element exists, or
  # `multiple' if there are multiple matching elements.
  findSingle = pred: default: multiple: list:
    let found = filter pred list; len = length found;
    in if len == 0 then default
      else if len != 1 then multiple
      else head found;


  # Find the first element in the list matching the specified
  # predicate or returns `default' if no such element exists.
  findFirst = pred: default: list:
    let found = filter pred list;
    in if found == [] then default else head found;


  # Return true iff function `pred' returns true for at least element
  # of `list'.
  any = pred: fold (x: y: if pred x then true else y) false;


  # Return true iff function `pred' returns true for all elements of
  # `list'.
  all = pred: fold (x: y: if pred x then y else false) true;


  # Count how many times function `pred' returns true for the elements
  # of `list'.
  count = pred: foldl' (c: x: if pred x then c + 1 else c) 0;


  # Return a singleton list or an empty list, depending on a boolean
  # value.  Useful when building lists with optional elements
  # (e.g. `++ optional (system == "i686-linux") flashplayer').
  optional = cond: elem: if cond then [elem] else [];


  # Return a list or an empty list, dependening on a boolean value.
  optionals = cond: elems: if cond then elems else [];


  # If argument is a list, return it; else, wrap it in a singleton
  # list.  If you're using this, you should almost certainly
  # reconsider if there isn't a more "well-typed" approach.
  toList = x: if isList x then x else [x];


  # Return a list of integers from `first' up to and including `last'.
  range = first: last:<commit_msg>Use builtin all and any functions<commit_after># General list operations.

with import ./trivial.nix;

rec {

  inherit (builtins) head tail length isList elemAt concatLists filter elem;


  # Create a list consisting of a single element.  `singleton x' is
  # sometimes more convenient with respect to indentation than `[x]'
  # when x spans multiple lines.
  singleton = x: [x];


  # "Fold" a binary function `op' between successive elements of
  # `list' with `nul' as the starting value, i.e., `fold op nul [x_1
  # x_2 ... x_n] == op x_1 (op x_2 ... (op x_n nul))'.  (This is
  # Haskell's foldr).
  fold = op: nul: list:
    let
      len = length list;
      fold' = n:
        if n == len
        then nul
        else op (elemAt list n) (fold' (n + 1));
    in fold' 0;

  # Left fold: `fold op nul [x_1 x_2 ... x_n] == op (... (op (op nul
  # x_1) x_2) ... x_n)'.
  foldl = op: nul: list:
    let
      len = length list;
      foldl' = n:
        if n == -1
        then nul
        else op (foldl' (n - 1)) (elemAt list n);
    in foldl' (length list - 1);


  # Strict version of foldl.
  foldl' = builtins.foldl' or foldl;


  # map with index: `imap (i: v: "${v}-${toString i}") ["a" "b"] ==
  # ["a-1" "b-2"]'
  imap = f: list:
    let
      len = length list;
      imap' = n:
        if n == len
          then []
          else [ (f (n + 1) (elemAt list n)) ] ++ imap' (n + 1);
    in imap' 0;


  # Map and concatenate the result.
  concatMap = f: list: concatLists (map f list);


  # Flatten the argument into a single list; that is, nested lists are
  # spliced into the top-level lists.  E.g., `flatten [1 [2 [3] 4] 5]
  # == [1 2 3 4 5]' and `flatten 1 == [1]'.
  flatten = x:
    if isList x
    then foldl' (x: y: x ++ (flatten y)) [] x
    else [x];


  # Remove elements equal to 'e' from a list.  Useful for buildInputs.
  remove = e: filter (x: x != e);


  # Find the sole element in the list matching the specified
  # predicate, returns `default' if no such element exists, or
  # `multiple' if there are multiple matching elements.
  findSingle = pred: default: multiple: list:
    let found = filter pred list; len = length found;
    in if len == 0 then default
      else if len != 1 then multiple
      else head found;


  # Find the first element in the list matching the specified
  # predicate or returns `default' if no such element exists.
  findFirst = pred: default: list:
    let found = filter pred list;
    in if found == [] then default else head found;


  # Return true iff function `pred' returns true for at least element
  # of `list'.
  any = builtins.any or (pred: fold (x: y: if pred x then true else y) false);


  # Return true iff function `pred' returns true for all elements of
  # `list'.
  all = builtins.all or (pred: fold (x: y: if pred x then y else false) true);


  # Count how many times function `pred' returns true for the elements
  # of `list'.
  count = pred: foldl' (c: x: if pred x then c + 1 else c) 0;


  # Return a singleton list or an empty list, depending on a boolean
  # value.  Useful when building lists with optional elements
  # (e.g. `++ optional (system == "i686-linux") flashplayer').
  optional = cond: elem: if cond then [elem] else [];


  # Return a list or an empty list, dependening on a boolean value.
  optionals = cond: elems: if cond then elems else [];


  # If argument is a list, return it; else, wrap it in a singleton
  # list.  If you're using this, you should almost certainly
  # reconsider if there isn't a more "well-typed" approach.
  toList = x: if isList x then x else [x];


  # Return a list of integers from `first' up to and including `last'.
  range = first: last:
{'fraction_non_alphanumeric': 0.11721311475409836, 'fraction_numerical': 0.010109289617486339, 'mean_word_length': 2.5469961240310077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '709488', 'n_tokens_mistral': 2539, 'n_tokens_neox': 2384, 'n_words': 1102}