<commit_before>-module(wilcog).
-export([build/1, rebuild/2, rebuild/3]).


build(Path) ->
  OldTree = gb_trees:empty(),
  rebuild(Path, OldTree).


rebuild(Path, OldTree)->
  NewTree = gb_trees:empty(),
  PathProperties = [{"path", Path}],
  RootTree = gb_trees:enter(Path, PathProperties, NewTree),
  rebuild(Path, RootTree, OldTree).


rebuild(Path, Tree, OldTree)->
  {ok, Items} = file:list_dir_all(Path),

  ItemFolder = fun(Item, Acc)->
    {ParentPath, AccumulatedTree, OldTree} = Acc,
    ItemPath = filename:absname_join(ParentPath, Item),
    NewStamp = filelib:last_modified(ItemPath),
    ItemProps = [{"path", ItemPath}],

    case filelib:is_dir(ItemPath) of
      true -> % It's a dir. Recurse through it and update tree.
        NewTree = gb_trees:enter(ItemPath, ItemProps, AccumulatedTree),
        UpdatedTree = rebuild(ItemPath, NewTree, OldTree),
        {ParentPath, UpdatedTree, OldTree};
      false -> % It's a file. Return updated tree.
        case NewStamp of
          0 ->
            % File deleted. So just return whatever required
            {ParentPath, AccumulatedTree, OldTree};
          _ ->
            % File exists, Add output to tree
            FileProps = [{"modified_at", NewStamp} | ItemProps],
            Output = get_output_for(ItemPath, NewStamp, OldTree),
            UpdatedTree = gb_trees:enter(ItemPath, [{"compiled", Output} | FileProps], AccumulatedTree),
            {ParentPath, UpdatedTree, OldTree}
        end
    end
  end,

  % The accumulator is weird, because we want function to be free of parent scope.
  {_, FinalTree, _} = lists:foldl(ItemFolder, {Path, Tree, OldTree}, Items),
  FinalTree.


get_output_for(ItemPath, NewStamp, OldTree) ->
  case gb_trees:lookup(ItemPath, OldTree) of
    {value, OldProps} ->
      OldStamp = proplists:get_value("modified_at", OldProps),
      case NewStamp of<commit_msg>Some var renames<commit_after>-module(wilcog).
-export([build/1, rebuild/2, rebuild/3]).


build(Path) ->
  OldTree = gb_trees:empty(),
  rebuild(Path, OldTree).


rebuild(Path, OldTree)->
  NewTree = gb_trees:empty(),
  PathProperties = [{"path", Path}],
  RootTree = gb_trees:enter(Path, PathProperties, NewTree),
  rebuild(Path, RootTree, OldTree).


rebuild(Path, Tree, OldTree)->
  {ok, Items} = file:list_dir_all(Path),

  ItemFolder = fun(Item, Acc)->
    {ParentPath, AccumulatedTree, OldTree} = Acc,
    ItemPath = filename:absname_join(ParentPath, Item),
    DefaultProps = [{"path", ItemPath}],

    case filelib:is_dir(ItemPath) of
      true -> % It's a dir. Recurse through it and update tree.
        NewTree = gb_trees:enter(ItemPath, DefaultProps, AccumulatedTree),
        UpdatedTree = rebuild(ItemPath, NewTree, OldTree),
        {ParentPath, UpdatedTree, OldTree};
      false -> % It's a file. Return updated tree.
        NewStamp = filelib:last_modified(ItemPath),
        case NewStamp of
          0 ->
            % File deleted. So just return whatever required
            {ParentPath, AccumulatedTree, OldTree};
          _ ->
            % File exists, Add output to tree
            Output = get_output_for(ItemPath, NewStamp, OldTree),
            FileProps = [{"modified_at", NewStamp}, {"compiled", Output}] ++ DefaultProps,
            UpdatedTree = gb_trees:enter(ItemPath, FileProps, AccumulatedTree),
            {ParentPath, UpdatedTree, OldTree}
        end
    end
  end,

  % The accumulator is weird, because we want function to be free of parent scope.
  % Or atleast try to.
  {_, FinalTree, _} = lists:foldl(ItemFolder, {Path, Tree, OldTree}, Items),
  FinalTree.


get_output_for(ItemPath, NewStamp, OldTree) ->
  case gb_trees:lookup(ItemPath, OldTree) of
    {value, OldProps} ->
      OldStamp = proplists:get_value("modified_at", OldProps),
      case NewStamp of
{'fraction_non_alphanumeric': 0.12259425257052466, 'fraction_numerical': 0.0021091484313208544, 'mean_word_length': 3.2155555555555555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295615', 'n_tokens_mistral': 1248, 'n_tokens_neox': 1198, 'n_words': 307}