<commit_before>-module(web_utils).
-include ("beehive.hrl").
-include ("http.hrl").
-compile (export_all).
    
% Private
convert_to_struct(RawData) ->
  lists:map(fun({BinKey, BinVal}) ->
      Key = misc_utils:to_atom(BinKey),
      Val = misc_utils:to_list(BinVal),
      {Key, Val}
    end, RawData).
    
% jsonify the struct
jsonify(JsonifiableBody) ->
  [ ?JSON_ENCODE({
        struct, [
          JsonifiableBody
        ]
    })
  ].<commit_msg>Added binify to web_utils to help out with binifying the proplists<commit_after>-module(web_utils).
-include ("beehive.hrl").
-include ("http.hrl").
-compile (export_all).
    
% Private
convert_to_struct(RawData) ->
  lists:map(fun({BinKey, BinVal}) ->
      Key = misc_utils:to_atom(BinKey),
      Val = misc_utils:to_list(BinVal),
      {Key, Val}
    end, RawData).
    
% jsonify the struct
jsonify(JsonifiableBody) ->
  [ ?JSON_ENCODE({
        struct, [
          JsonifiableBody
        ]
    })
  ].

binify(Proplist) ->
  lists:map(fun({Key, V}) ->
    case V of
      List when is_list(List) ->
        case io_lib:char_list(List) of
          true -> {Key, misc_utils:to_bin(List)};
          false -> 
            % check to see if the head is lists of strings
            case V of
              ListOfStrings when is_list(hd(ListOfStrings)) ->
                {Key, lists:map(fun(E) -> misc_utils:to_bin(E) end, V)};
              _Else ->
                {Key, {struct, binify(V)}}
            end
        end;
      Tuple when is_tuple(Tuple) -> {Key, binify(Tuple)};
      E -> {Key, misc_utils:to_bin(E)}
    end
  end, Proplist).
{'fraction_non_alphanumeric': 0.1415211970074813, 'fraction_numerical': 0.0, 'mean_word_length': 2.2688391038696536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3291648', 'n_tokens_mistral': 567, 'n_tokens_neox': 550, 'n_words': 116}