<commit_before>%% @author <NAME> <<EMAIL>>
%% @copyright 2007 Mochi Media, Inc.

%% @doc Yet another JSON (RFC 4627) library for Erlang. mochijson2 works
%%      with binaries as strings, arrays as lists (without an {array, _})
%%      wrapper and it only knows how to decode UTF-8 (and ASCII).
%%
%%      JSON terms are decoded as follows (javascript -> erlang):
%%      <ul>
%%          <li>{"key": "value"} ->
%%              {struct, [{&lt;&lt;"key">>, &lt;&lt;"value">>}]}</li>
%%          <li>["array", 123, 12.34, true, false, null] ->
%%              [&lt;&lt;"array">>, 123, 12.34, true, false, null]
%%          </li>
%%      </ul>
%%      <ul>
%%          <li>Strings in JSON decode to UTF-8 binaries in Erlang</li>
%%          <li>Objects decode to {struct, PropList}</li>
%%          <li>Numbers decode to integer or float</li>
%%          <li>true, false, null decode to their respective terms.</li>
%%      </ul>
%%      The encoder will accept the same format that the decoder will produce,
%%      but will also allow additional cases for leniency:
%%      <ul>
%%          <li>atoms other than true, false, null will be considered UTF-8
%%              strings (even as a proplist key)
%%          </li>
%%          <li>{json, IoList} will insert IoList directly into the output
%%              with no validation
%%          </li>
%%          <li>{array, Array} will be encoded as Array
%%              (legacy mochijson style)
%%          </li>
%%          <li>A non-empty raw proplist will be encoded as an object as long
%%              as the first pair does not have an atom key of json, struct,
%%              or array
%%          </li>
%%      </ul>

-module(nitro_mochijson2).
-author('<EMAIL>').
-export([encoder/1, encode/1]).
-export([decoder/1, decode/1, decode/2]).

%% This is a macro to placate syntax highlighters..
-define(Q, $\").
-define(ADV_COL(S, N), S#decoder{offset=N+S#decoder.offset,
                                 column=N+S#decoder.column}).
-define(INC_COL(S), S#decoder{offset=1+S#decoder.offset,
                              column=1+S#decoder.column}).
-define(INC_LINE(S), S#decoder{offset=1+S#decoder.offset,
                               column=1,
                               line=1+S#decoder.line}).
-define(INC_CHAR(S, C),
        case C of
            $\n ->
                S#decoder{column=1,
                          line=1+S#decoder.line,
                          offset=1+S#decoder.offset};
            _ ->
                S#decoder{column=1+S#decoder.column,
                          offset=1+S#decoder.offset}
        end).
-define(IS_WHITESPACE(C),
        (C =:= $\s orelse C =:= $\t orelse C =:= $\r orelse C =:= $\n)).

%% @type json_string() = atom | binary()
%% @type json_number() = integer() | float()
%% @type json_array() = [json_term()]
%% @type json_object() = {struct, [{json_string(), json_term()}]}
%% @type json_eep18_object() = {[{json_string(), json_term()}]}
%% @type json_iolist() = {json, iolist()}
%% @type json_term() = json_string() | json_number() | json_array() |
%%                     json_object() | json_eep18_object() | json_iolist()

-record(encoder, {handler=null,
                  utf8=false}).

-record(decoder, {object_hook=null,
                  offset=0,
                  line=1,
                  column=1,
                  state=null}).

%% @spec encoder([encoder_option()]) -> function()
%% @doc Create an encoder/1 with the given options.
%% @type encoder_option() = handler_option() | utf8_option()
%% @type utf8_option() = boolean(). Emit unicode as utf8 (default - false)
encoder(Options) ->
    State = parse_encoder_options(Options, #encoder{}),
    fun (O) -> json_encode(O, State) end.

%% @spec encode(json_term()) -> iolist()
%% @doc Encode the given as JSON to an iolist.
encode(Any) ->
    json_encode(Any, #encoder{}).

%% @spec decoder([decoder_option()]) -> function()
%% @doc Create a decoder/1 with the given options.
decoder(Options) ->
    State = parse_decoder_options(Options, #decoder{}),
    fun (O) -> json_decode(O, State) end.

%% @spec decode(iolist(), [{format, proplist | eep18 | struct}]) -> json_term()
%% @doc Decode the given iolist to Erlang terms using the given object format
%%      for decoding, where proplist returns JSON objects as [{binary(), json_term()}]
%%      proplists, eep18 returns JSON objects as {[binary(), json_term()]}, and struct
%%      returns them as-is.
decode(S, Options) ->
    json_decode(S, parse_decoder_options(Options, #decoder{})).

%% @spec decode(iolist()) -> json_term()
%% @doc Decode the given iolist to Erlang terms.
decode(S) ->
    json_decode(S, #decoder{}).

%% Internal API

parse_encoder_options([], State) ->
    State;
parse_encoder_options([{handler, Handler} | Rest], State) ->
    parse_encoder_options(Rest, State#encoder{handler=Handler});
parse_encoder_options([{utf8, Switch} | Rest], State) ->
    parse_encoder_options(Rest, State#encoder{utf8=Switch}).

parse_decoder_options([], State) ->
    State;
parse_decoder_options([{object_hook, Hook} | Rest], State) ->
    parse_decoder_options(Rest, State#decoder{object_hook=Hook});
parse_decoder_options([{format, Format} | Rest], State)
  when Format =:= struct orelse Format =:= eep18 orelse Format =:= proplist ->
    parse_decoder_options(Rest, State#decoder{object_hook=Format}).

json_encode(true, _State) ->
    <<"true">>;
json_encode(false, _State) ->
    <<"false">>;
json_encode(null, _State) ->
    <<"null">>;
json_encode(I, _State) when is_integer(I) ->
    integer_to_list(I);
json_encode(F, _State) when is_float(F) ->
    mochinum:digits(F);
json_encode(S, State) when is_binary(S); is_atom(S) ->
    json_encode_string(S, State);
json_encode([{K, _}|_] = Props, State) when (K =/= struct andalso
                                             K =/= array andalso
                                             K =/= json) ->
    json_encode_proplist(Props, State);
json_encode({struct, Props}, State) when is_list(Props) ->
    json_encode_proplist(Props, State);
json_encode({Props}, State) when is_list(Props) ->
    json_encode_proplist(Props, State);
json_encode({}, State) ->
    json_encode_proplist([], State);
json_encode(Array, State) when is_list(Array) ->
    json_encode_array(Array, State);
json_encode({array, Array}, State) when is_list(Array) ->
    json_encode_array(Array, State);
json_encode({json, IoList}, _State) ->
    IoList;
json_encode(Bad, #encoder{handler=null}) ->
    exit({json_encode, {bad_term, Bad}});
json_encode(Bad, State=#encoder{handler=Handler}) ->
    json_encode(Handler(Bad), State).

json_encode_array([], _State) ->
    <<"[]">>;<commit_msg>Using 'nitro_mochinum' instead of 'mochinum'<commit_after>%% @author <NAME> <<EMAIL>>
%% @copyright 2007 Mochi Media, Inc.

%% @doc Yet another JSON (RFC 4627) library for Erlang. mochijson2 works
%%      with binaries as strings, arrays as lists (without an {array, _})
%%      wrapper and it only knows how to decode UTF-8 (and ASCII).
%%
%%      JSON terms are decoded as follows (javascript -> erlang):
%%      <ul>
%%          <li>{"key": "value"} ->
%%              {struct, [{&lt;&lt;"key">>, &lt;&lt;"value">>}]}</li>
%%          <li>["array", 123, 12.34, true, false, null] ->
%%              [&lt;&lt;"array">>, 123, 12.34, true, false, null]
%%          </li>
%%      </ul>
%%      <ul>
%%          <li>Strings in JSON decode to UTF-8 binaries in Erlang</li>
%%          <li>Objects decode to {struct, PropList}</li>
%%          <li>Numbers decode to integer or float</li>
%%          <li>true, false, null decode to their respective terms.</li>
%%      </ul>
%%      The encoder will accept the same format that the decoder will produce,
%%      but will also allow additional cases for leniency:
%%      <ul>
%%          <li>atoms other than true, false, null will be considered UTF-8
%%              strings (even as a proplist key)
%%          </li>
%%          <li>{json, IoList} will insert IoList directly into the output
%%              with no validation
%%          </li>
%%          <li>{array, Array} will be encoded as Array
%%              (legacy mochijson style)
%%          </li>
%%          <li>A non-empty raw proplist will be encoded as an object as long
%%              as the first pair does not have an atom key of json, struct,
%%              or array
%%          </li>
%%      </ul>

-module(nitro_mochijson2).
-author('<EMAIL>').
-export([encoder/1, encode/1]).
-export([decoder/1, decode/1, decode/2]).

%% This is a macro to placate syntax highlighters..
-define(Q, $\").
-define(ADV_COL(S, N), S#decoder{offset=N+S#decoder.offset,
                                 column=N+S#decoder.column}).
-define(INC_COL(S), S#decoder{offset=1+S#decoder.offset,
                              column=1+S#decoder.column}).
-define(INC_LINE(S), S#decoder{offset=1+S#decoder.offset,
                               column=1,
                               line=1+S#decoder.line}).
-define(INC_CHAR(S, C),
        case C of
            $\n ->
                S#decoder{column=1,
                          line=1+S#decoder.line,
                          offset=1+S#decoder.offset};
            _ ->
                S#decoder{column=1+S#decoder.column,
                          offset=1+S#decoder.offset}
        end).
-define(IS_WHITESPACE(C),
        (C =:= $\s orelse C =:= $\t orelse C =:= $\r orelse C =:= $\n)).

%% @type json_string() = atom | binary()
%% @type json_number() = integer() | float()
%% @type json_array() = [json_term()]
%% @type json_object() = {struct, [{json_string(), json_term()}]}
%% @type json_eep18_object() = {[{json_string(), json_term()}]}
%% @type json_iolist() = {json, iolist()}
%% @type json_term() = json_string() | json_number() | json_array() |
%%                     json_object() | json_eep18_object() | json_iolist()

-record(encoder, {handler=null,
                  utf8=false}).

-record(decoder, {object_hook=null,
                  offset=0,
                  line=1,
                  column=1,
                  state=null}).

%% @spec encoder([encoder_option()]) -> function()
%% @doc Create an encoder/1 with the given options.
%% @type encoder_option() = handler_option() | utf8_option()
%% @type utf8_option() = boolean(). Emit unicode as utf8 (default - false)
encoder(Options) ->
    State = parse_encoder_options(Options, #encoder{}),
    fun (O) -> json_encode(O, State) end.

%% @spec encode(json_term()) -> iolist()
%% @doc Encode the given as JSON to an iolist.
encode(Any) ->
    json_encode(Any, #encoder{}).

%% @spec decoder([decoder_option()]) -> function()
%% @doc Create a decoder/1 with the given options.
decoder(Options) ->
    State = parse_decoder_options(Options, #decoder{}),
    fun (O) -> json_decode(O, State) end.

%% @spec decode(iolist(), [{format, proplist | eep18 | struct}]) -> json_term()
%% @doc Decode the given iolist to Erlang terms using the given object format
%%      for decoding, where proplist returns JSON objects as [{binary(), json_term()}]
%%      proplists, eep18 returns JSON objects as {[binary(), json_term()]}, and struct
%%      returns them as-is.
decode(S, Options) ->
    json_decode(S, parse_decoder_options(Options, #decoder{})).

%% @spec decode(iolist()) -> json_term()
%% @doc Decode the given iolist to Erlang terms.
decode(S) ->
    json_decode(S, #decoder{}).

%% Internal API

parse_encoder_options([], State) ->
    State;
parse_encoder_options([{handler, Handler} | Rest], State) ->
    parse_encoder_options(Rest, State#encoder{handler=Handler});
parse_encoder_options([{utf8, Switch} | Rest], State) ->
    parse_encoder_options(Rest, State#encoder{utf8=Switch}).

parse_decoder_options([], State) ->
    State;
parse_decoder_options([{object_hook, Hook} | Rest], State) ->
    parse_decoder_options(Rest, State#decoder{object_hook=Hook});
parse_decoder_options([{format, Format} | Rest], State)
  when Format =:= struct orelse Format =:= eep18 orelse Format =:= proplist ->
    parse_decoder_options(Rest, State#decoder{object_hook=Format}).

json_encode(true, _State) ->
    <<"true">>;
json_encode(false, _State) ->
    <<"false">>;
json_encode(null, _State) ->
    <<"null">>;
json_encode(I, _State) when is_integer(I) ->
    integer_to_list(I);
json_encode(F, _State) when is_float(F) ->
    nitro_mochinum:digits(F);
json_encode(S, State) when is_binary(S); is_atom(S) ->
    json_encode_string(S, State);
json_encode([{K, _}|_] = Props, State) when (K =/= struct andalso
                                             K =/= array andalso
                                             K =/= json) ->
    json_encode_proplist(Props, State);
json_encode({struct, Props}, State) when is_list(Props) ->
    json_encode_proplist(Props, State);
json_encode({Props}, State) when is_list(Props) ->
    json_encode_proplist(Props, State);
json_encode({}, State) ->
    json_encode_proplist([], State);
json_encode(Array, State) when is_list(Array) ->
    json_encode_array(Array, State);
json_encode({array, Array}, State) when is_list(Array) ->
    json_encode_array(Array, State);
json_encode({json, IoList}, _State) ->
    IoList;
json_encode(Bad, #encoder{handler=null}) ->
    exit({json_encode, {bad_term, Bad}});
json_encode(Bad, State=#encoder{handler=Handler}) ->
    json_encode(Handler(Bad), State).

json_encode_array([], _State) ->
    <<"[]">>;
{'fraction_non_alphanumeric': 0.16503182328715837, 'fraction_numerical': 0.00943466866342194, 'mean_word_length': 2.8724267903740213, 'pattern_counts': {'":': 2, '<': 79, '<?xml version=': 0, '>': 169, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3923814', 'n_tokens_mistral': 4443, 'n_tokens_neox': 4232, 'n_words': 1080}