<commit_before># Utility functions.

let

  inherit (builtins)
    head tail isList stringLength substring lessThan sub
    listToAttrs attrNames hasAttr;

in

rec {
  listOfListsToAttrs = ll : builtins.listToAttrs (map (l : { name = (head l); value = (head (tail l)); }) ll);


  # Identity function.
  id = x: x;


  # accumulates / merges all attr sets until null is fed.
  # example: sumArgs id { a = 'a'; x = 'x'; } { y = 'y'; x = 'X'; } null
  # result : { a = 'a'; x = 'X'; y = 'Y'; }
  innerSumArgs = f : x : y : (if y == null then (f x)
	else (innerSumArgs f (x // y)));
  sumArgs = f : innerSumArgs f {};

  # Advanced sumArgs version. Hm, twice as slow, I'm afraid.
  # composedArgs id (x:x//{a="b";}) (x:x//{b=x.a + "c";}) null;
  # {a="b" ; b="bc";};
  innerComposedArgs = f : x : y : (if y==null then (f x)
  	else (if (builtins.isAttrs y) then 
		(innerComposedArgs f (x//y))
	else (innerComposedArgs f (y x))));
  composedArgs = f: innerComposedArgs f {};

  defaultMergeArg = x : y: if builtins.isAttrs y then
    y
  else 
    (y x);
  defaultMerge = x: y: x // (defaultMergeArg x y);
  sumTwoArgs = f: x: y: 
    f (defaultMerge x y);
  foldArgs = merger: f: init: x: 
    let arg=(merger init (defaultMergeArg init x)); in  
      # now add the function with composed args already applied to the final attrs
    setAttrMerge "passthru" {} (f arg) ( x : x // { function = foldArgs merger f arg; } );

  # returns f x // { passthru.fun = y : f (merge x y); } while preserving other passthru names.
  # example: let ex = applyAndFun (x : removeAttrs x ["fixed"])  (mergeOrApply mergeAttr) {name = 6;};
  #              usage1 = ex.passthru.fun { name = 7; };    # result: { name = 7;}
  #              usage2 = ex.passthru.fun (a: a // {name = __add a.name 1; }); # result: { a = 7; }
  # fix usage:
  #              usage3a = ex.passthru.fun (a: a // {name2 = a.fixed.toBePassed; }); # usage3a will fail because toBePassed is not yet given
  #              usage3b usage3a.passthru.fun { toBePassed = "foo";}; # result { name = 7; name2 = "foo"; toBePassed = "foo"; fixed = <this attrs>; }
  applyAndFun = f : merge : x : assert (__isAttrs x || __isFunction x);
    let takeFix = if (__isFunction x) then x else (attr: merge attr x); in
    setAttrMerge "passthru" {} (fix (fixed : f (takeFix {inherit fixed;})))
      ( y : y //
        {
          fun = z : applyAndFun f merge (fixed: merge (takeFix fixed) z);
          funMerge = z : applyAndFun f merge (fixed: let e = takeFix fixed; in e // merge e z);
        } );
  mergeOrApply = merge : x : y : if (__isFunction y) then  y x else merge x y;

  # rec { # an example of how composedArgsAndFun can be used
  #  a  = composedArgsAndFun (x : x) { a = ["2"]; meta = { d = "bar";}; };
  #  # meta.d will be lost ! It's your task to preserve it (eg using a merge function)
  #  b  = a.passthru.function { a = [ "3" ]; meta = { d2 = "bar2";}; };
  #  # instead of passing/ overriding values you can use a merge function:
  #  c  = b.passthru.function ( x: { a = x.a  ++ ["4"]; }); # consider using (maybeAttr "a" [] x)
  # }
  # result:
  # {
  #   a = { a = ["2"];     meta = { d = "bar"; }; passthru = { function = .. }; };
  #   b = { a = ["3"];     meta = { d2 = "bar2"; }; passthru = { function = .. }; };
  #   c = { a = ["3" "4"]; meta = { d2 = "bar2"; }; passthru = { function = .. }; };
  #   # c2 is equal to c
  # }
  composedArgsAndFun = f: foldArgs defaultMerge f {};

  # example a = pairMap (x : y : x + y) ["a" "b" "c" "d"];
  # result: ["ab" "cd"]
  innerPairMap = acc: f: l: 
  	if l == [] then acc else
	innerPairMap (acc ++ [(f (head l)(head (tail l)))])
		f (tail (tail l));
  pairMap = innerPairMap [];

  
  
  # "Fold" a binary function `op' between successive elements of
  # `list' with `nul' as the starting value, i.e., `fold op nul [x_1
  # x_2 ... x_n] == op x_1 (op x_2 ... (op x_n nul))'.  (This is
  # Haskell's foldr).
  fold = op: nul: list:
    if list == []
    then nul
    else op (head list) (fold op nul (tail list));
<commit_msg>added Haskell's foldl<commit_after># Utility functions.

let

  inherit (builtins)
    head tail isList stringLength substring lessThan sub
    listToAttrs attrNames hasAttr;

in

rec {
  listOfListsToAttrs = ll : builtins.listToAttrs (map (l : { name = (head l); value = (head (tail l)); }) ll);


  # Identity function.
  id = x: x;


  # accumulates / merges all attr sets until null is fed.
  # example: sumArgs id { a = 'a'; x = 'x'; } { y = 'y'; x = 'X'; } null
  # result : { a = 'a'; x = 'X'; y = 'Y'; }
  innerSumArgs = f : x : y : (if y == null then (f x)
	else (innerSumArgs f (x // y)));
  sumArgs = f : innerSumArgs f {};

  # Advanced sumArgs version. Hm, twice as slow, I'm afraid.
  # composedArgs id (x:x//{a="b";}) (x:x//{b=x.a + "c";}) null;
  # {a="b" ; b="bc";};
  innerComposedArgs = f : x : y : (if y==null then (f x)
  	else (if (builtins.isAttrs y) then 
		(innerComposedArgs f (x//y))
	else (innerComposedArgs f (y x))));
  composedArgs = f: innerComposedArgs f {};

  defaultMergeArg = x : y: if builtins.isAttrs y then
    y
  else 
    (y x);
  defaultMerge = x: y: x // (defaultMergeArg x y);
  sumTwoArgs = f: x: y: 
    f (defaultMerge x y);
  foldArgs = merger: f: init: x: 
    let arg=(merger init (defaultMergeArg init x)); in  
      # now add the function with composed args already applied to the final attrs
    setAttrMerge "passthru" {} (f arg) ( x : x // { function = foldArgs merger f arg; } );

  # returns f x // { passthru.fun = y : f (merge x y); } while preserving other passthru names.
  # example: let ex = applyAndFun (x : removeAttrs x ["fixed"])  (mergeOrApply mergeAttr) {name = 6;};
  #              usage1 = ex.passthru.fun { name = 7; };    # result: { name = 7;}
  #              usage2 = ex.passthru.fun (a: a // {name = __add a.name 1; }); # result: { a = 7; }
  # fix usage:
  #              usage3a = ex.passthru.fun (a: a // {name2 = a.fixed.toBePassed; }); # usage3a will fail because toBePassed is not yet given
  #              usage3b usage3a.passthru.fun { toBePassed = "foo";}; # result { name = 7; name2 = "foo"; toBePassed = "foo"; fixed = <this attrs>; }
  applyAndFun = f : merge : x : assert (__isAttrs x || __isFunction x);
    let takeFix = if (__isFunction x) then x else (attr: merge attr x); in
    setAttrMerge "passthru" {} (fix (fixed : f (takeFix {inherit fixed;})))
      ( y : y //
        {
          fun = z : applyAndFun f merge (fixed: merge (takeFix fixed) z);
          funMerge = z : applyAndFun f merge (fixed: let e = takeFix fixed; in e // merge e z);
        } );
  mergeOrApply = merge : x : y : if (__isFunction y) then  y x else merge x y;

  # rec { # an example of how composedArgsAndFun can be used
  #  a  = composedArgsAndFun (x : x) { a = ["2"]; meta = { d = "bar";}; };
  #  # meta.d will be lost ! It's your task to preserve it (eg using a merge function)
  #  b  = a.passthru.function { a = [ "3" ]; meta = { d2 = "bar2";}; };
  #  # instead of passing/ overriding values you can use a merge function:
  #  c  = b.passthru.function ( x: { a = x.a  ++ ["4"]; }); # consider using (maybeAttr "a" [] x)
  # }
  # result:
  # {
  #   a = { a = ["2"];     meta = { d = "bar"; }; passthru = { function = .. }; };
  #   b = { a = ["3"];     meta = { d2 = "bar2"; }; passthru = { function = .. }; };
  #   c = { a = ["3" "4"]; meta = { d2 = "bar2"; }; passthru = { function = .. }; };
  #   # c2 is equal to c
  # }
  composedArgsAndFun = f: foldArgs defaultMerge f {};

  # example a = pairMap (x : y : x + y) ["a" "b" "c" "d"];
  # result: ["ab" "cd"]
  innerPairMap = acc: f: l: 
  	if l == [] then acc else
	innerPairMap (acc ++ [(f (head l)(head (tail l)))])
		f (tail (tail l));
  pairMap = innerPairMap [];

  
  
  # "Fold" a binary function `op' between successive elements of
  # `list' with `nul' as the starting value, i.e., `fold op nul [x_1
  # x_2 ... x_n] == op x_1 (op x_2 ... (op x_n nul))'.  (This is
  # Haskell's foldr).
  fold = op: nul: list:
    if list == []
    then nul
    else op (head list) (fold op nul (tail list));

  # Haskell's fold
  foldl = op: nul: list:
    if list == []
    then nul
    else fold op (op nul (head list)) (tail list);

{'fraction_non_alphanumeric': 0.1686746987951807, 'fraction_numerical': 0.007788730680296945, 'mean_word_length': 2.5014912654452495, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5407878', 'n_tokens_mistral': 3104, 'n_tokens_neox': 3028, 'n_words': 1059}