<commit_before>%%% Copyright (C) 2010-2011  <NAME>
%%%
%%% Author: <NAME> <<EMAIL>>
%%%
%%% This library is free software; you can redistribute it and/or
%%% modify it under the terms of the GNU Lesser General Public
%%% License as published by the Free Software Foundation; either
%%% version 2.1 of the License, or (at your option) any later version.
%%%
%%% This library is distributed in the hope that it will be useful,
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
%%% Lesser General Public License for more details.
%%%
%%% You should have received a copy of the GNU Lesser General Public
%%% License along with this library; if not, write to the Free Software
%%% Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
%%% MA  02110-1301  USA

-module(gpb_compile_tests).

-include_lib("kernel/include/file.hrl").
-include_lib("eunit/include/eunit.hrl").
-include("../include/gpb.hrl").

-export([test_nifs/1]). %% set whether to test nifs or not

-export([compile_iolist/2]).
-export([unload_code/1]).

%% NIF related
-export([nif_tests_check_prerequisites/1]).
-export([nif_oneof_tests_check_prerequisites/1]).
-export([nif_mapfield_tests_check_prerequisites/1]).
-export([nif_proto3_tests_check_prerequisites/1]).
-export([increase_timeouts/1]).
-export([with_tmpdir/1]).
-export([in_separate_vm/4]).
-export([compile_nif_msg_defs/3, compile_nif_msg_defs/4]).
-export([check_protoc_can_do_oneof/0]).
-export([check_protoc_can_do_mapfields/0]).
-export([check_protoc_can_do_proto3/0]).

%% internally used
-export([main_in_separate_vm/1]).

%% Include a bunch of tests from gpb_tests.
%% The shared tests are for stuff that must work both
%% for gpb and for the code that gpb_compile generates.
%% (I know it is a bit unorthodox to include .erl files,
%% but actually seems to work. Better than duplicating tests)
-define(gpb_compile_common_tests, true).
-ifdef(gpb_compile_common_tests).
-include("gpb_tests.erl").
-else.  %% gpb_compile_common_tests
-record(m1,{a}).
-endif. %% gpb_compile_common_tests

parses_non_importing_file_test() ->
    Contents = iolist_to_binary(
                 ["message Msg { required uint32 field1 = 1; }\n"]),
    ok = gpb_compile:file(
           "X.proto",
           [mk_fileop_opt([{read_file, fun(_) -> {ok, Contents} end}]),
            mk_defs_probe_sender_opt(self()),
            {i,"."}]),
    [{{msg,'Msg'},_}] = receive_filter_sort_msgs_defs().


parses_importing_file_test() ->
    ContentsX = iolist_to_binary(
                  ["import \"Y.proto\";\n"
                   "message X { required Y f1 = 1; }\n"]),
    ContentsY = iolist_to_binary(
                  ["message Y { required uint32 f1 = 1; }\n"]),
    ok = gpb_compile:file(
           "X.proto",
           [mk_fileop_opt([{read_file, fun("X.proto") -> {ok, ContentsX};
                                          ("Y.proto") -> {ok, ContentsY}
                                       end}]),
            mk_defs_probe_sender_opt(self()),
            {i,"."}]),
    [{{msg,'X'},_}, {{msg,'Y'},_}] = receive_filter_sort_msgs_defs().


parses_file_to_binary_test() ->
    Contents = <<"message Msg { required uint32 field1 = 1; }\n">>,
    {ok, 'X', Code, []} =
        gpb_compile:file(
          "X.proto",
          [mk_fileop_opt([{read_file, fun(_) -> {ok, Contents} end}]),
           mk_defs_probe_sender_opt(self()),
           {i,"."},
           binary, return_warnings]),
    true = is_binary(Code),
    [{{msg,'Msg'},_}] = receive_filter_sort_msgs_defs().

parses_file_to_msg_defs_test() ->
    Contents = <<"message Msg { required uint32 field1 = 1; }\n">>,
    {ok, [{{msg,'Msg'},[#?gpb_field{}]}]=MsgDefs} =
        gpb_compile:file(
          "X.proto",
          [mk_fileop_opt([{read_file, fun(_) -> {ok, Contents} end}]),
           {i,"."},
           to_proto_defs, report_warnings]),
    %% Check that the returned msgdefs are usable
    M = compile_defs(MsgDefs),
    ?assertMatch(<<_/binary>>, M:encode_msg({'Msg',33})),
    unload_code(M).

parses_msgdefs_to_binary_test() ->
    Defs = [{{msg,'Msg'},
             [#?gpb_field{name=field1, rnum=2, fnum=1, type=uint32,
                          occurrence=required, opts=[]}]}],
    M = find_unused_module(),
    {ok, M, Code} = gpb_compile:proto_defs(M, Defs, [binary]),
    true = is_binary(Code).

parses_and_generates_good_code_also_for_reserved_keywords_test() ->
    %% use erlang reserved words on as many .proto locations as possible
    %% to verify that the generated code compiles and works.
    M = compile_iolist(["enum if { begin=1; end=2; }"
                        "message catch { required if case = 1; }\n"]),
    ?assertMatch(true, is_binary(M:encode_msg({'catch', 'begin'}))),
    unload_code(M).

parses_and_generates_good_code_also_for_empty_msgs_test() ->
    M = compile_iolist(["message m1 { }\n"]),
    ?assertMatch(true, is_binary(M:encode_msg({m1}))),
    ?assertMatch({m1}, M:decode_msg(M:encode_msg({m1}), m1)),
    unload_code(M).

field_pass_as_params_test() ->
    MsgDef = ["message m2 { required uint32 f22 = 1; }"
              "message m1 { required uint32  f1 = 1;",
              "             optional fixed32 f2 = 2;",
              "             repeated fixed32 f3 = 3;",
              "             repeated fixed32 f4 = 4 [packed];",
              "             repeated uint32  f5 = 5;",
              "             repeated uint32  f6 = 5 [packed];",
              "             optional string  f7 = 7;",
              "             optional m2      f8 = 8;",
              "             oneof o1 { m2     x1 = 15;",
              "                        uint32 y1 = 16; };",
              "             oneof o2 { m2     x2 = 25;",
              "                        uint32 y2 = 26; }",
              "             oneof o3 { m2     x3 = 35;",
              "                        uint32 y3 = 36; }",
              "}"],
    Msg = {m1, 4711, undefined,      %% f1,f2
           [4713,4714], [4715,4716], %% f3,f4
           [4717,4718], [4719,4720], %% f5,f6
           "abc", {m2,33}, {x1,{m2,45}}, {y2,226}, undefined},
    lists:foreach(
      fun(Opts) ->
              ?assertMatch({Msg,_},
                           {encode_decode_round_trip(MsgDef, Msg, Opts), Opts})
      end,
      [[{field_pass_method,pass_as_params}],
       [{field_pass_method,pass_as_record}],
       [{{field_pass_method,m1},pass_as_record},
        {{field_pass_method,m2},pass_as_params}],
       [{{field_pass_method,m1},pass_as_params},
        {{field_pass_method,m2},pass_as_record}]]).

encode_decode_round_trip(MsgDefAsIoList, Msg, Opts) ->
    M = compile_iolist(MsgDefAsIoList, Opts),
    MsgName = element(1, Msg),
    Result = M:decode_msg(M:encode_msg(Msg), MsgName),
    unload_code(M),
    Result.

mk_fileop_opt(NonDefaults) ->
    NonDefaults1 = [case Op of
                        read_file_info -> {Op, mk_with_basename_1(Fn)};
                        read_file      -> {Op, mk_with_basename_1(Fn)};
                        write_file     -> {Op, mk_with_basename_2(Fn)}
                    end
                    || {Op, Fn} <- NonDefaults],
    {file_op, NonDefaults1 ++ mk_default_file_ops()}.

mk_with_basename_1(Fn) -> fun(Path) -> Fn(filename:basename(Path)) end.
mk_with_basename_2(Fn) -> fun(Path, A) -> Fn(filename:basename(Path), A) end.

mk_default_file_ops() ->
    [{read_file_info, fun(_FileName) -> {ok, #file_info{access=read}} end},
     {read_file,      fun(_FileName) -> {error, enoent} end},
     {write_file,     fun(_FileName, _Bin) -> ok end}].

mk_defs_probe_sender_opt(SendTo) ->
    {probe_defs, fun(Defs) -> SendTo ! {defs, Defs} end}.

receive_filter_sort_msgs_defs() ->
    lists:sort([Msg || {{msg,_},_} = Msg <- receive {defs, Defs} -> Defs end]).

-record(m9,{aa, bb, cc, dd}).
-record(m10,{aaa}).

code_generation_when_submsg_size_is_known_at_compile_time_test() ->
    KnownSizeM9 =
        [{{msg,m9}, [#?gpb_field{name=aa, type={enum,e}, occurrence=required,
                                 fnum=1, rnum=#m9.aa, opts=[]},
                     #?gpb_field{name=bb, type=fixed32, occurrence=required,
                                 fnum=2, rnum=#m9.bb, opts=[]},
                     #?gpb_field{name=cc, type=fixed64, occurrence=required,
                                 fnum=3, rnum=#m9.cc, opts=[]},
                     #?gpb_field{name=dd, type={msg,m10}, occurrence=required,
                                 fnum=4, rnum=#m9.dd, opts=[]}]}],
    UnknownSizeM9 =
        [{{msg,m9}, [#?gpb_field{name=aa, type={enum,e}, occurrence=optional,
                                 fnum=1, rnum=#m9.aa, opts=[]},
                     #?gpb_field{name=bb, type=fixed32, occurrence=optional,
                                 fnum=2, rnum=#m9.bb, opts=[]},
                     #?gpb_field{name=cc, type=fixed64, occurrence=optional,
                                 fnum=3, rnum=#m9.cc, opts=[]},
                     #?gpb_field{name=dd, type={msg,m10}, occurrence=required,
                                 fnum=4, rnum=#m9.dd, opts=[]}]}],

    CommonDefs =
        [{{msg,m1}, [#?gpb_field{name=a, type={msg,m9}, occurrence=required,
                                 fnum=1, rnum=2, opts=[]}]},
         {{msg,m10},[#?gpb_field{name=aaa, type=bool, occurrence=required,
                                 fnum=1, rnum=#m10.aaa, opts=[]}]},
         {{enum,e}, [{x1, 1}, {x2, 2}]} %% all enum values same encode size
        ],

    M1 = compile_defs(CommonDefs++KnownSizeM9),
    M2 = compile_defs(CommonDefs++UnknownSizeM9),
    Msg = #m1{a=#m9{aa=x1, bb=33, cc=44, dd=#m10{aaa=true}}},
    Encoded1 = M1:encode_msg(Msg),
    Encoded2 = M2:encode_msg(Msg),
    Encoded1 = Encoded2,
    unload_code(M1),
    unload_code(M2).

%% --- introspection ---------------

introspection_package_name_test() ->
    M = compile_iolist(["package foo.bar;",
                        "message M { required uint32 f1=1; }"]),
    'foo.bar' = M:get_package_name(),
    unload_code(M),
    M = compile_iolist(["message M { required uint32 f1=1; }"]),
    undefined = M:get_package_name(),
    unload_code(M).

introspection_msgs_test() ->
    M = compile_iolist(["message msg1 { required uint32 f1=1; }"]),
    [msg1] = M:get_msg_names(),
    [#?gpb_field{name=f1, type=uint32, fnum=1}] = M:find_msg_def(msg1),
    [#?gpb_field{name=f1, type=uint32, fnum=1}] = M:fetch_msg_def(msg1),
    error = M:find_msg_def(msg_ee),
    ?assertError(_, M:fetch_msg_def(msg_ee)),
    unload_code(M).

introspection_enums_test() ->
    %% Names
    M = compile_iolist(["enum e1 { n1=1; n2=2; }",
                        "message msg1 { required uint32 f1=1; }"]),
    [e1] = M:get_enum_names(),
    %% find and fetch
    [{n1,1},{n2,2}] = M:find_enum_def(e1),
    error = M:find_enum_def(ee),
    [{n1,1},{n2,2}] = M:fetch_enum_def(e1),
    ?assertError(_, M:fetch_enum_def(ee)),
    %% symbol <--> value mapping
    n1 = M:enum_symbol_by_value(e1, 1),
    n2 = M:enum_symbol_by_value(e1, 2),
    n1 = M:enum_symbol_by_value_e1(1),
    n2 = M:enum_symbol_by_value_e1(2),
    1  = M:enum_value_by_symbol(e1, n1),
    2  = M:enum_value_by_symbol(e1, n2),
    1  = M:enum_value_by_symbol_e1(n1),
    2  = M:enum_value_by_symbol_e1(n2),
    unload_code(M).

introspection_defs_as_proplists_test() ->
    Proto = ["message msg1 { required uint32 f1=1; }",
             "service s1 {",
             "  rpc req1(msg1) returns (msg1);",
             "  rpc req2(msg1) returns (msg1);",
             "}",
             "service s2 {",
             "  rpc req2(msg1) returns (msg1);",
             "}"],
    %% With the defs_as_proplists option
    M = compile_iolist(Proto, [defs_as_proplists]),
    [[{name,       f1},
      {fnum,       1},
      {rnum,       2},
      {type,       uint32},
      {occurrence, required},
      {opts,       []}]] = PL = M:find_msg_def(msg1),
    [{{msg, msg1}, PL}] = M:get_msg_defs(),
    [s1, s2] = M:get_service_names(),
    {{service, s1}, [[{name, req1}, {input, 'msg1'}, {output, 'msg1'}], 
                     [{name, req2}, {input, 'msg1'}, {output, 'msg1'}]]} = M:get_service_def(s1),
    {{service, s2}, [[{name, req2}, {input, 'msg1'}, {output, 'msg1'}]]} = M:get_service_def(s2),
    [{name, req1}, {input, 'msg1'}, {output, 'msg1'}] = M:find_rpc_def(s1, req1),
    [{name, req2}, {input, 'msg1'}, {output, 'msg1'}] = M:find_rpc_def(s2, req2),
    unload_code(M),

    %% No defs_as_proplists option
    M = compile_iolist(Proto, [{defs_as_proplists, false}]),
    [#?gpb_field{name       = f1,
                 fnum       = 1,
                 rnum       = 2,
                 type       = uint32,
                 occurrence = required,
                 opts       = []}] = Fs = M:find_msg_def(msg1),
    [{{msg, msg1}, Fs}] = Defs = M:get_msg_defs(),
    {{service, s1},
     [#?gpb_rpc{name=req1, input='msg1', output='msg1'},
      #?gpb_rpc{name=req2, input='msg1', output='msg1'}]} =
        M:get_service_def(s1),
    {{service, s2},
     [#?gpb_rpc{name=req2, input='msg1', output='msg1'}]} =
        M:get_service_def(s2),
    #?gpb_rpc{name=req1, input='msg1', output='msg1'} = M:find_rpc_def(s1, req1),
    #?gpb_rpc{name=req2, input='msg1', output='msg1'} = M:find_rpc_def(s2, req2),
    unload_code(M),

    %% make sure the generated erl file does not -include[_lib] "gpb.hrl"
    Master = self(),
    ReportOutput = fun(FName, Contents) ->
                           Master ! {filename:extension(FName), Contents},
                           ok
                   end,
    FileOpOpt = mk_fileop_opt([{write_file, ReportOutput}]),
    ok = gpb_compile:proto_defs(M, Defs, [FileOpOpt, defs_as_proplists]),
    receive {".hrl", Hrl1} -> nomatch = re:run(Hrl1, "\"gpb.hrl\"") end,
    receive {".erl", Erl1} -> nomatch = re:run(Erl1, "\"gpb.hrl\"") end,
    ok = gpb_compile:proto_defs(M, Defs, [FileOpOpt, defs_as_proplists,
                                          include_as_lib]),
    receive {".hrl", Hrl2} -> nomatch = re:run(Hrl2, "\"gpb.hrl\"") end,
    receive {".erl", Erl2} -> nomatch = re:run(Erl2, "\"gpb.hrl\"") end.

introspection_rpcs_test() ->
    Proto = ["message m1 { required uint32 f1=1; }",
             "message m2 { required uint32 f2=1; }",
             "service s1 {",
             "  rpc req1(m1) returns (m2);",
             "  rpc req2(m2) returns (m1);",
             "}"],
    M = compile_iolist(Proto),
    {{service, s1},
     [#?gpb_rpc{name=req1, input='m1', output='m2'},
      #?gpb_rpc{name=req2, input='m2', output='m1'}]} = M:get_service_def(s1),
    [req1, req2] = M:get_rpc_names(s1),
    #?gpb_rpc{name=req1, input='m1', output='m2'} = M:find_rpc_def(s1, req1),
    #?gpb_rpc{name=req1, input='m1', output='m2'} = M:fetch_rpc_def(s1, req1),
    #?gpb_rpc{name=req2, input='m2', output='m1'} = M:fetch_rpc_def(s1, req2),
    #?gpb_rpc{name=req2, input='m2', output='m1'} = M:find_rpc_def(s1, req2),
    error = M:find_rpc_def(s1, req_ee),
    ?assertError(_, M:fetch_rpc_def(s2, req_ee)),
    unload_code(M).

introspection_multiple_rpcs_test() ->
    Proto = ["message m1 { required uint32 f1=1; }",
             "message m2 { required uint32 f2=1; }",
             "service s1 {",
             "  rpc req1(m1) returns (m2);",
             "  rpc req2(m2) returns (m1);",
             "}",
             "service s2 {",
             "  rpc req21(m2) returns (m1);",
             "  rpc req22(m1) returns (m2);",
             "}"],
    M = compile_iolist(Proto),
    [s1, s2] = M:get_service_names(),
    {{service, s1},
     [#?gpb_rpc{name=req1, input='m1', output='m2'},
      #?gpb_rpc{name=req2, input='m2', output='m1'}]} = M:get_service_def(s1),
    {{service, s2},
     [#?gpb_rpc{name=req21, input='m2', output='m1'},
      #?gpb_rpc{name=req22, input='m1', output='m2'}]} = M:get_service_def(s2),
    #?gpb_rpc{name=req21,  input='m2', output='m1'} = M:find_rpc_def(s2, req21),
    #?gpb_rpc{name=req1, input='m1', output='m2'} = M:fetch_rpc_def(s1, req1),
    #?gpb_rpc{name=req2,  input='m2', output='m1'} = M:fetch_rpc_def(s1, req2),
    #?gpb_rpc{name=req22, input='m1', output='m2'} = M:find_rpc_def(s2, req22),
    error = M:find_rpc_def(s1, req_ee),
    error = M:find_rpc_def(s2, req_aa),
    ?assertError(_, M:fetch_rpc_def(s2, req_ee)),
    ?assertError(_, M:fetch_rpc_def(s1, req_aa)),
    unload_code(M).

%% --- decoder tests ---------------

decodes_overly_long_varints_test() ->
    M = compile_defs([{{msg,m1}, [#?gpb_field{name=a, type=int32,
                                              fnum=1, rnum=#m1.a,
                                              occurrence=required, opts=[]}]}]),
    #m1{a=54} = M:decode_msg(<<8, 54>>, m1), %% canonically encoded
    #m1{a=54} = M:decode_msg(<<8, (128+54), 128, 128, 0>>, m1),
    unload_code(M).

%% --- scoped messages ---------------

dotted_names_gives_no_compilation_error_test() ->
    %% make sure dotted names does not give compilation errors,
    %% for instance if some generated code would rely on names
    %% having the same syntax as erlang atoms, or, when prepended
    %% with an upper case character, having the same syntax as an
    %% erlang variable
    M = compile_iolist(["message m1 {"
                        "  message m2 { required uint32 x = 1; }",
                        "  enum    e1 { ea = 17; eb = 18; }",
                        "  required m2     y = 1;",
                        "  required .m1.m2 z = 2;",
                        "  required e1     w = 3;",
                        "}",
                        "message m3 { required m1.m2 b = 1; }"]),
    M1Msg = {m1, {'m1.m2', 1}, {'m1.m2', 2}, ea},
    Data = M:encode_msg(M1Msg),
    M1Msg = M:decode_msg(Data, m1),
    unload_code(M).

%% --- module/msg name prefix/suffix ---------------
module_msg_name_prefix_test() ->
    Proto = <<"message msg1 { required uint32 f1=1; }\n">>,
    Master = self(),
    ReadInput = fun(FName) -> Master ! {read, FName}, {ok, Proto} end,
    ReportOutput = fun(FName, Contents) ->
                           Ext = list_to_atom(tl(filename:extension(FName))),
                           Master ! {write, {Ext, FName, Contents}},
                           ok
                   end,
    FileOpOpt = mk_fileop_opt([{read_file, ReadInput},
                               {write_file, ReportOutput}]),
    ModPrefix = "mp_",
    MsgPrefix = "mm_",
    ModSuffix = "_xp",
    MsgSuffix = "_xm",
    ok = gpb_compile:file("m.proto",
                          [FileOpOpt, {i,"."},
                           {module_name_prefix, ModPrefix},
                           {msg_name_prefix, MsgPrefix},
                           {module_name_suffix, ModSuffix},
                           {msg_name_suffix, MsgSuffix}]),
    receive
        {read, "m.proto"} -> ok;
        {read, X} -> erlang:error("reading from odd file", X)
    end,
    receive
        {write, {hrl, "mp_m_xp.hrl", Hrl}} ->
            assert_contains_regexp(Hrl, "mm_msg1_xm"),
            ok;
        {write, {hrl, "m.hrl", _}} ->
            erlang:error("hrl file not prefixed or suffixed!");
        {write, {hrl, X2, C2}} ->
            erlang:error({"writing odd hrl file!", X2, C2})
    end,
    receive
        {write, {erl, "mp_m_xp.erl", Erl}} ->
            assert_contains_regexp(Erl, "-include.*\"mp_m_xp.hrl\""),
            assert_contains_regexp(Erl, "-module.*mp_m_xp"),
            assert_contains_regexp(Erl, "mm_msg1_xm"),
            ok;
        {write, {erl, "m.erl", _}} ->
            erlang:error("erl file not prefixed or suffixed!");
        {write, {erl, X3, C3}} ->
            erlang:error({"writing odd erl file!", X3, C3})
    end,
    ok.

assert_contains_regexp(IoData, Re) ->
    case re:run(IoData, Re) of
        {match, _} -> ok;
        nomatch    ->
            ?debugFmt("~nERROR: Regexp ~s not found in:~n~s~n", [Re, IoData]),
            erlang:error({"Re ", Re, "not found in", IoData})
    end.

%% --- bytes ----------

copy_bytes_unconditionally_test() ->
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>, % binary exists since R14A
    if HasBinary ->
            M = compile_iolist(["message m1 { required bytes f1 = 1; }"],
                               [{copy_bytes, true}]),
            Data = M:encode_msg({m1, <<"d">>}),
            {m1, <<"d">>=Bs} = M:decode_msg(Data, m1),
            %% If the Bs has not been copied, then it is a sub-binary
            %% of a larger binary: of the message, ie of Data.
            %% So verify copying by verifying size of referenced data.
            ?assertEqual(byte_size(Bs), binary:referenced_byte_size(Bs)),
            unload_code(M);
       true ->
            %% nothing to test
            ok
    end.

copy_bytes_false_test() ->
    M = compile_iolist(["message m1 { required bytes f1 = 1; }"],
                       [{copy_bytes, false}]),
    Data = M:encode_msg({m1, <<"d">>}),
    {m1, <<"d">>=Bs} = M:decode_msg(Data, m1),
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>, % binary exists since R14A
    if HasBinary ->
            %% If the StrBin has not been copied, then it is a sub-binary
            %% of a larger binary: of the message, ie of Data.
            %% So verify copying by verifying size of referenced data.
            ?assertEqual(byte_size(Data), binary:referenced_byte_size(Bs));
       true ->
            ok
    end,
    unload_code(M).

copy_bytes_auto_test() ->
    M = compile_iolist(["message m1 { required bytes f1 = 1; }"],
                       [{copy_bytes, auto}]),
    Data = M:encode_msg({m1, <<"d">>}),
    {m1, <<"d">>=Bs} = M:decode_msg(Data, m1),
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>,
    if HasBinary ->
            ?assertEqual(byte_size(Bs), binary:referenced_byte_size(Bs));
       true ->
            ok %% cannot test more if we don't have the binary module
    end,
    unload_code(M).

copy_bytes_fraction_test() ->
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>,
    if HasBinary ->
            Proto = ["message m1 {",
                     "  required bytes f1 = 1;",
                     "  required bytes f2 = 2;",
                     "}"],
            M1 = compile_iolist(Proto, [{copy_bytes, 2}]),   % fraction as int
            M2 = compile_iolist(Proto, [{copy_bytes, 2.0}]), % fraction as float
            D1 = <<"d">>, %% small
            D2 = <<"dddddddddddddddddddddddddddd">>, %% large
            Data = M1:encode_msg({m1, D1, D2}),
            ?assert(byte_size(Data) > (2 * byte_size(D1))),
            ?assert(byte_size(Data) < (2 * byte_size(D2))),
            {m1, D1Bs, D2Bs} = M1:decode_msg(Data, m1),
            ?assertEqual(D1, D1Bs),
            ?assertEqual(D2, D2Bs),
            %% The small data should have been copied, but not the larger
            ?assertEqual(byte_size(D1Bs), binary:referenced_byte_size(D1Bs)),
            ?assertEqual(byte_size(Data), binary:referenced_byte_size(D2Bs)),

            {m1, D3Bs, D4Bs} = M2:decode_msg(Data, m1),
            ?assertEqual(D1, D3Bs),
            ?assertEqual(D2, D4Bs),
            ?assertEqual(byte_size(D3Bs), binary:referenced_byte_size(D3Bs)),
            ?assertEqual(byte_size(Data), binary:referenced_byte_size(D4Bs)),

            unload_code(M1),
            unload_code(M2);
       true ->
            ok
    end.

%% --- strings ----------

strings_as_binaries_option_produces_bins_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "}"],
                       [strings_as_binaries]),
    Data = M:encode_msg({m1, "some string"}),
    {m1, <<"some string">>} = M:decode_msg(Data, m1),
    unload_code(M).

strings_as_lists_is_the_default_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "}"],
                       []),
    Data = M:encode_msg({m1, "some string"}),
    {m1, "some string"} = M:decode_msg(Data, m1),
    unload_code(M).

strings_as_binaries_opt_together_with_copy_bytes_opt_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "}"],
                       [strings_as_binaries, {copy_bytes, auto}]),
    Data = M:encode_msg({m1, "some string"}),
    {m1, <<"some string">>=StrBin} = M:decode_msg(Data, m1),
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>, % binary exists since R14A
    if HasBinary ->
            ?assertEqual(byte_size(StrBin),
                         binary:referenced_byte_size(StrBin));
       true ->
            ok
    end,
    unload_code(M).

accepts_both_strings_and_binaries_as_input_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "  required string f2 = 2;",
                        "}"]),
    Data = M:encode_msg({m1, "some string", <<"some other string">>}),
    {m1, "some string", "some other string"} = M:decode_msg(Data, m1),
    unload_code(M).

verifies_both_strings_and_binaries_as_input_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "  required string f2 = 2;",
                        "}"],
                        [strings_as_binaries]),
    R = {m1, "some string", <<"some other string">>},
    ok = M:verify_msg(R),
    ?assertError(_, M:verify_msg({m1, "a", <<97,98,99,255,191>>})),
    unload_code(M).

%% --- misc ----------

only_enums_no_msgs_test() ->
    M = compile_iolist(["enum e {"
                        "  a = 1;",
                        "}"]),
    ?assertError({gpb_error, no_messages}, M:encode_msg({x})),
    ?assertError({gpb_error, no_messages}, M:encode_msg({x}, [])),
    ?assertError({gpb_error, no_messages}, M:decode_msg(<<>>, x)),
    ?assertError({gpb_error, no_messages}, M:merge_msgs({x}, {x})),
    ?assertError({gpb_type_error, {not_a_known_message, _}}, M:verify_msg({x})),
    [] = M:get_msg_names(),
    [e] = M:get_enum_names(),
    unload_code(M).

%% --- Returning/reporting warnings/errors (and warnings_as_errors) tests -----
%% ... when compiling to file/binary/defs
%% ... when compiling from file/defs
%% ... when there are/aren't warnings/errors

report_or_return_warnings_or_errors_test_() ->
    %% Without increased timeout, this test sometimes times out
    %% on my slow machine (1.6 GHz Atom N270)
    {timeout,58,fun report_or_return_warnings_or_errors_test_aux/0}.

report_or_return_warnings_or_errors_test_aux() ->
    [begin
         Options = WarningOptions ++ ErrorOptions ++ WarnsAsErrsOpts,
         rwre_go(Options, CompileTo, SrcType, SrcQuality)
     end
     || WarningOptions     <- [[], [report_warnings], [return_warnings],
                               [report_warnings, return_warnings]],
        ErrorOptions       <- [[], [report_errors], [return_errors],
                               [report_errors, return_errors]],
        WarnsAsErrsOpts    <- [[], [warnings_as_errors]],
        CompileTo          <- [to_binary, to_file, to_proto_defs],
        SrcType            <- [from_file, from_defs],<commit_msg>Aid debugging of the report_or_return_warnings_or_errors test<commit_after>%%% Copyright (C) 2010-2011  <NAME>
%%%
%%% Author: <NAME> <<EMAIL>>
%%%
%%% This library is free software; you can redistribute it and/or
%%% modify it under the terms of the GNU Lesser General Public
%%% License as published by the Free Software Foundation; either
%%% version 2.1 of the License, or (at your option) any later version.
%%%
%%% This library is distributed in the hope that it will be useful,
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
%%% Lesser General Public License for more details.
%%%
%%% You should have received a copy of the GNU Lesser General Public
%%% License along with this library; if not, write to the Free Software
%%% Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
%%% MA  02110-1301  USA

-module(gpb_compile_tests).

-include_lib("kernel/include/file.hrl").
-include_lib("eunit/include/eunit.hrl").
-include("../include/gpb.hrl").

-export([test_nifs/1]). %% set whether to test nifs or not

-export([compile_iolist/2]).
-export([unload_code/1]).

%% NIF related
-export([nif_tests_check_prerequisites/1]).
-export([nif_oneof_tests_check_prerequisites/1]).
-export([nif_mapfield_tests_check_prerequisites/1]).
-export([nif_proto3_tests_check_prerequisites/1]).
-export([increase_timeouts/1]).
-export([with_tmpdir/1]).
-export([in_separate_vm/4]).
-export([compile_nif_msg_defs/3, compile_nif_msg_defs/4]).
-export([check_protoc_can_do_oneof/0]).
-export([check_protoc_can_do_mapfields/0]).
-export([check_protoc_can_do_proto3/0]).

%% internally used
-export([main_in_separate_vm/1]).

%% Include a bunch of tests from gpb_tests.
%% The shared tests are for stuff that must work both
%% for gpb and for the code that gpb_compile generates.
%% (I know it is a bit unorthodox to include .erl files,
%% but actually seems to work. Better than duplicating tests)
-define(gpb_compile_common_tests, true).
-ifdef(gpb_compile_common_tests).
-include("gpb_tests.erl").
-else.  %% gpb_compile_common_tests
-record(m1,{a}).
-endif. %% gpb_compile_common_tests

parses_non_importing_file_test() ->
    Contents = iolist_to_binary(
                 ["message Msg { required uint32 field1 = 1; }\n"]),
    ok = gpb_compile:file(
           "X.proto",
           [mk_fileop_opt([{read_file, fun(_) -> {ok, Contents} end}]),
            mk_defs_probe_sender_opt(self()),
            {i,"."}]),
    [{{msg,'Msg'},_}] = receive_filter_sort_msgs_defs().


parses_importing_file_test() ->
    ContentsX = iolist_to_binary(
                  ["import \"Y.proto\";\n"
                   "message X { required Y f1 = 1; }\n"]),
    ContentsY = iolist_to_binary(
                  ["message Y { required uint32 f1 = 1; }\n"]),
    ok = gpb_compile:file(
           "X.proto",
           [mk_fileop_opt([{read_file, fun("X.proto") -> {ok, ContentsX};
                                          ("Y.proto") -> {ok, ContentsY}
                                       end}]),
            mk_defs_probe_sender_opt(self()),
            {i,"."}]),
    [{{msg,'X'},_}, {{msg,'Y'},_}] = receive_filter_sort_msgs_defs().


parses_file_to_binary_test() ->
    Contents = <<"message Msg { required uint32 field1 = 1; }\n">>,
    {ok, 'X', Code, []} =
        gpb_compile:file(
          "X.proto",
          [mk_fileop_opt([{read_file, fun(_) -> {ok, Contents} end}]),
           mk_defs_probe_sender_opt(self()),
           {i,"."},
           binary, return_warnings]),
    true = is_binary(Code),
    [{{msg,'Msg'},_}] = receive_filter_sort_msgs_defs().

parses_file_to_msg_defs_test() ->
    Contents = <<"message Msg { required uint32 field1 = 1; }\n">>,
    {ok, [{{msg,'Msg'},[#?gpb_field{}]}]=MsgDefs} =
        gpb_compile:file(
          "X.proto",
          [mk_fileop_opt([{read_file, fun(_) -> {ok, Contents} end}]),
           {i,"."},
           to_proto_defs, report_warnings]),
    %% Check that the returned msgdefs are usable
    M = compile_defs(MsgDefs),
    ?assertMatch(<<_/binary>>, M:encode_msg({'Msg',33})),
    unload_code(M).

parses_msgdefs_to_binary_test() ->
    Defs = [{{msg,'Msg'},
             [#?gpb_field{name=field1, rnum=2, fnum=1, type=uint32,
                          occurrence=required, opts=[]}]}],
    M = find_unused_module(),
    {ok, M, Code} = gpb_compile:proto_defs(M, Defs, [binary]),
    true = is_binary(Code).

parses_and_generates_good_code_also_for_reserved_keywords_test() ->
    %% use erlang reserved words on as many .proto locations as possible
    %% to verify that the generated code compiles and works.
    M = compile_iolist(["enum if { begin=1; end=2; }"
                        "message catch { required if case = 1; }\n"]),
    ?assertMatch(true, is_binary(M:encode_msg({'catch', 'begin'}))),
    unload_code(M).

parses_and_generates_good_code_also_for_empty_msgs_test() ->
    M = compile_iolist(["message m1 { }\n"]),
    ?assertMatch(true, is_binary(M:encode_msg({m1}))),
    ?assertMatch({m1}, M:decode_msg(M:encode_msg({m1}), m1)),
    unload_code(M).

field_pass_as_params_test() ->
    MsgDef = ["message m2 { required uint32 f22 = 1; }"
              "message m1 { required uint32  f1 = 1;",
              "             optional fixed32 f2 = 2;",
              "             repeated fixed32 f3 = 3;",
              "             repeated fixed32 f4 = 4 [packed];",
              "             repeated uint32  f5 = 5;",
              "             repeated uint32  f6 = 5 [packed];",
              "             optional string  f7 = 7;",
              "             optional m2      f8 = 8;",
              "             oneof o1 { m2     x1 = 15;",
              "                        uint32 y1 = 16; };",
              "             oneof o2 { m2     x2 = 25;",
              "                        uint32 y2 = 26; }",
              "             oneof o3 { m2     x3 = 35;",
              "                        uint32 y3 = 36; }",
              "}"],
    Msg = {m1, 4711, undefined,      %% f1,f2
           [4713,4714], [4715,4716], %% f3,f4
           [4717,4718], [4719,4720], %% f5,f6
           "abc", {m2,33}, {x1,{m2,45}}, {y2,226}, undefined},
    lists:foreach(
      fun(Opts) ->
              ?assertMatch({Msg,_},
                           {encode_decode_round_trip(MsgDef, Msg, Opts), Opts})
      end,
      [[{field_pass_method,pass_as_params}],
       [{field_pass_method,pass_as_record}],
       [{{field_pass_method,m1},pass_as_record},
        {{field_pass_method,m2},pass_as_params}],
       [{{field_pass_method,m1},pass_as_params},
        {{field_pass_method,m2},pass_as_record}]]).

encode_decode_round_trip(MsgDefAsIoList, Msg, Opts) ->
    M = compile_iolist(MsgDefAsIoList, Opts),
    MsgName = element(1, Msg),
    Result = M:decode_msg(M:encode_msg(Msg), MsgName),
    unload_code(M),
    Result.

mk_fileop_opt(NonDefaults) ->
    NonDefaults1 = [case Op of
                        read_file_info -> {Op, mk_with_basename_1(Fn)};
                        read_file      -> {Op, mk_with_basename_1(Fn)};
                        write_file     -> {Op, mk_with_basename_2(Fn)}
                    end
                    || {Op, Fn} <- NonDefaults],
    {file_op, NonDefaults1 ++ mk_default_file_ops()}.

mk_with_basename_1(Fn) -> fun(Path) -> Fn(filename:basename(Path)) end.
mk_with_basename_2(Fn) -> fun(Path, A) -> Fn(filename:basename(Path), A) end.

mk_default_file_ops() ->
    [{read_file_info, fun(_FileName) -> {ok, #file_info{access=read}} end},
     {read_file,      fun(_FileName) -> {error, enoent} end},
     {write_file,     fun(_FileName, _Bin) -> ok end}].

mk_defs_probe_sender_opt(SendTo) ->
    {probe_defs, fun(Defs) -> SendTo ! {defs, Defs} end}.

receive_filter_sort_msgs_defs() ->
    lists:sort([Msg || {{msg,_},_} = Msg <- receive {defs, Defs} -> Defs end]).

-record(m9,{aa, bb, cc, dd}).
-record(m10,{aaa}).

code_generation_when_submsg_size_is_known_at_compile_time_test() ->
    KnownSizeM9 =
        [{{msg,m9}, [#?gpb_field{name=aa, type={enum,e}, occurrence=required,
                                 fnum=1, rnum=#m9.aa, opts=[]},
                     #?gpb_field{name=bb, type=fixed32, occurrence=required,
                                 fnum=2, rnum=#m9.bb, opts=[]},
                     #?gpb_field{name=cc, type=fixed64, occurrence=required,
                                 fnum=3, rnum=#m9.cc, opts=[]},
                     #?gpb_field{name=dd, type={msg,m10}, occurrence=required,
                                 fnum=4, rnum=#m9.dd, opts=[]}]}],
    UnknownSizeM9 =
        [{{msg,m9}, [#?gpb_field{name=aa, type={enum,e}, occurrence=optional,
                                 fnum=1, rnum=#m9.aa, opts=[]},
                     #?gpb_field{name=bb, type=fixed32, occurrence=optional,
                                 fnum=2, rnum=#m9.bb, opts=[]},
                     #?gpb_field{name=cc, type=fixed64, occurrence=optional,
                                 fnum=3, rnum=#m9.cc, opts=[]},
                     #?gpb_field{name=dd, type={msg,m10}, occurrence=required,
                                 fnum=4, rnum=#m9.dd, opts=[]}]}],

    CommonDefs =
        [{{msg,m1}, [#?gpb_field{name=a, type={msg,m9}, occurrence=required,
                                 fnum=1, rnum=2, opts=[]}]},
         {{msg,m10},[#?gpb_field{name=aaa, type=bool, occurrence=required,
                                 fnum=1, rnum=#m10.aaa, opts=[]}]},
         {{enum,e}, [{x1, 1}, {x2, 2}]} %% all enum values same encode size
        ],

    M1 = compile_defs(CommonDefs++KnownSizeM9),
    M2 = compile_defs(CommonDefs++UnknownSizeM9),
    Msg = #m1{a=#m9{aa=x1, bb=33, cc=44, dd=#m10{aaa=true}}},
    Encoded1 = M1:encode_msg(Msg),
    Encoded2 = M2:encode_msg(Msg),
    Encoded1 = Encoded2,
    unload_code(M1),
    unload_code(M2).

%% --- introspection ---------------

introspection_package_name_test() ->
    M = compile_iolist(["package foo.bar;",
                        "message M { required uint32 f1=1; }"]),
    'foo.bar' = M:get_package_name(),
    unload_code(M),
    M = compile_iolist(["message M { required uint32 f1=1; }"]),
    undefined = M:get_package_name(),
    unload_code(M).

introspection_msgs_test() ->
    M = compile_iolist(["message msg1 { required uint32 f1=1; }"]),
    [msg1] = M:get_msg_names(),
    [#?gpb_field{name=f1, type=uint32, fnum=1}] = M:find_msg_def(msg1),
    [#?gpb_field{name=f1, type=uint32, fnum=1}] = M:fetch_msg_def(msg1),
    error = M:find_msg_def(msg_ee),
    ?assertError(_, M:fetch_msg_def(msg_ee)),
    unload_code(M).

introspection_enums_test() ->
    %% Names
    M = compile_iolist(["enum e1 { n1=1; n2=2; }",
                        "message msg1 { required uint32 f1=1; }"]),
    [e1] = M:get_enum_names(),
    %% find and fetch
    [{n1,1},{n2,2}] = M:find_enum_def(e1),
    error = M:find_enum_def(ee),
    [{n1,1},{n2,2}] = M:fetch_enum_def(e1),
    ?assertError(_, M:fetch_enum_def(ee)),
    %% symbol <--> value mapping
    n1 = M:enum_symbol_by_value(e1, 1),
    n2 = M:enum_symbol_by_value(e1, 2),
    n1 = M:enum_symbol_by_value_e1(1),
    n2 = M:enum_symbol_by_value_e1(2),
    1  = M:enum_value_by_symbol(e1, n1),
    2  = M:enum_value_by_symbol(e1, n2),
    1  = M:enum_value_by_symbol_e1(n1),
    2  = M:enum_value_by_symbol_e1(n2),
    unload_code(M).

introspection_defs_as_proplists_test() ->
    Proto = ["message msg1 { required uint32 f1=1; }",
             "service s1 {",
             "  rpc req1(msg1) returns (msg1);",
             "  rpc req2(msg1) returns (msg1);",
             "}",
             "service s2 {",
             "  rpc req2(msg1) returns (msg1);",
             "}"],
    %% With the defs_as_proplists option
    M = compile_iolist(Proto, [defs_as_proplists]),
    [[{name,       f1},
      {fnum,       1},
      {rnum,       2},
      {type,       uint32},
      {occurrence, required},
      {opts,       []}]] = PL = M:find_msg_def(msg1),
    [{{msg, msg1}, PL}] = M:get_msg_defs(),
    [s1, s2] = M:get_service_names(),
    {{service, s1}, [[{name, req1}, {input, 'msg1'}, {output, 'msg1'}], 
                     [{name, req2}, {input, 'msg1'}, {output, 'msg1'}]]} = M:get_service_def(s1),
    {{service, s2}, [[{name, req2}, {input, 'msg1'}, {output, 'msg1'}]]} = M:get_service_def(s2),
    [{name, req1}, {input, 'msg1'}, {output, 'msg1'}] = M:find_rpc_def(s1, req1),
    [{name, req2}, {input, 'msg1'}, {output, 'msg1'}] = M:find_rpc_def(s2, req2),
    unload_code(M),

    %% No defs_as_proplists option
    M = compile_iolist(Proto, [{defs_as_proplists, false}]),
    [#?gpb_field{name       = f1,
                 fnum       = 1,
                 rnum       = 2,
                 type       = uint32,
                 occurrence = required,
                 opts       = []}] = Fs = M:find_msg_def(msg1),
    [{{msg, msg1}, Fs}] = Defs = M:get_msg_defs(),
    {{service, s1},
     [#?gpb_rpc{name=req1, input='msg1', output='msg1'},
      #?gpb_rpc{name=req2, input='msg1', output='msg1'}]} =
        M:get_service_def(s1),
    {{service, s2},
     [#?gpb_rpc{name=req2, input='msg1', output='msg1'}]} =
        M:get_service_def(s2),
    #?gpb_rpc{name=req1, input='msg1', output='msg1'} = M:find_rpc_def(s1, req1),
    #?gpb_rpc{name=req2, input='msg1', output='msg1'} = M:find_rpc_def(s2, req2),
    unload_code(M),

    %% make sure the generated erl file does not -include[_lib] "gpb.hrl"
    Master = self(),
    ReportOutput = fun(FName, Contents) ->
                           Master ! {filename:extension(FName), Contents},
                           ok
                   end,
    FileOpOpt = mk_fileop_opt([{write_file, ReportOutput}]),
    ok = gpb_compile:proto_defs(M, Defs, [FileOpOpt, defs_as_proplists]),
    receive {".hrl", Hrl1} -> nomatch = re:run(Hrl1, "\"gpb.hrl\"") end,
    receive {".erl", Erl1} -> nomatch = re:run(Erl1, "\"gpb.hrl\"") end,
    ok = gpb_compile:proto_defs(M, Defs, [FileOpOpt, defs_as_proplists,
                                          include_as_lib]),
    receive {".hrl", Hrl2} -> nomatch = re:run(Hrl2, "\"gpb.hrl\"") end,
    receive {".erl", Erl2} -> nomatch = re:run(Erl2, "\"gpb.hrl\"") end.

introspection_rpcs_test() ->
    Proto = ["message m1 { required uint32 f1=1; }",
             "message m2 { required uint32 f2=1; }",
             "service s1 {",
             "  rpc req1(m1) returns (m2);",
             "  rpc req2(m2) returns (m1);",
             "}"],
    M = compile_iolist(Proto),
    {{service, s1},
     [#?gpb_rpc{name=req1, input='m1', output='m2'},
      #?gpb_rpc{name=req2, input='m2', output='m1'}]} = M:get_service_def(s1),
    [req1, req2] = M:get_rpc_names(s1),
    #?gpb_rpc{name=req1, input='m1', output='m2'} = M:find_rpc_def(s1, req1),
    #?gpb_rpc{name=req1, input='m1', output='m2'} = M:fetch_rpc_def(s1, req1),
    #?gpb_rpc{name=req2, input='m2', output='m1'} = M:fetch_rpc_def(s1, req2),
    #?gpb_rpc{name=req2, input='m2', output='m1'} = M:find_rpc_def(s1, req2),
    error = M:find_rpc_def(s1, req_ee),
    ?assertError(_, M:fetch_rpc_def(s2, req_ee)),
    unload_code(M).

introspection_multiple_rpcs_test() ->
    Proto = ["message m1 { required uint32 f1=1; }",
             "message m2 { required uint32 f2=1; }",
             "service s1 {",
             "  rpc req1(m1) returns (m2);",
             "  rpc req2(m2) returns (m1);",
             "}",
             "service s2 {",
             "  rpc req21(m2) returns (m1);",
             "  rpc req22(m1) returns (m2);",
             "}"],
    M = compile_iolist(Proto),
    [s1, s2] = M:get_service_names(),
    {{service, s1},
     [#?gpb_rpc{name=req1, input='m1', output='m2'},
      #?gpb_rpc{name=req2, input='m2', output='m1'}]} = M:get_service_def(s1),
    {{service, s2},
     [#?gpb_rpc{name=req21, input='m2', output='m1'},
      #?gpb_rpc{name=req22, input='m1', output='m2'}]} = M:get_service_def(s2),
    #?gpb_rpc{name=req21,  input='m2', output='m1'} = M:find_rpc_def(s2, req21),
    #?gpb_rpc{name=req1, input='m1', output='m2'} = M:fetch_rpc_def(s1, req1),
    #?gpb_rpc{name=req2,  input='m2', output='m1'} = M:fetch_rpc_def(s1, req2),
    #?gpb_rpc{name=req22, input='m1', output='m2'} = M:find_rpc_def(s2, req22),
    error = M:find_rpc_def(s1, req_ee),
    error = M:find_rpc_def(s2, req_aa),
    ?assertError(_, M:fetch_rpc_def(s2, req_ee)),
    ?assertError(_, M:fetch_rpc_def(s1, req_aa)),
    unload_code(M).

%% --- decoder tests ---------------

decodes_overly_long_varints_test() ->
    M = compile_defs([{{msg,m1}, [#?gpb_field{name=a, type=int32,
                                              fnum=1, rnum=#m1.a,
                                              occurrence=required, opts=[]}]}]),
    #m1{a=54} = M:decode_msg(<<8, 54>>, m1), %% canonically encoded
    #m1{a=54} = M:decode_msg(<<8, (128+54), 128, 128, 0>>, m1),
    unload_code(M).

%% --- scoped messages ---------------

dotted_names_gives_no_compilation_error_test() ->
    %% make sure dotted names does not give compilation errors,
    %% for instance if some generated code would rely on names
    %% having the same syntax as erlang atoms, or, when prepended
    %% with an upper case character, having the same syntax as an
    %% erlang variable
    M = compile_iolist(["message m1 {"
                        "  message m2 { required uint32 x = 1; }",
                        "  enum    e1 { ea = 17; eb = 18; }",
                        "  required m2     y = 1;",
                        "  required .m1.m2 z = 2;",
                        "  required e1     w = 3;",
                        "}",
                        "message m3 { required m1.m2 b = 1; }"]),
    M1Msg = {m1, {'m1.m2', 1}, {'m1.m2', 2}, ea},
    Data = M:encode_msg(M1Msg),
    M1Msg = M:decode_msg(Data, m1),
    unload_code(M).

%% --- module/msg name prefix/suffix ---------------
module_msg_name_prefix_test() ->
    Proto = <<"message msg1 { required uint32 f1=1; }\n">>,
    Master = self(),
    ReadInput = fun(FName) -> Master ! {read, FName}, {ok, Proto} end,
    ReportOutput = fun(FName, Contents) ->
                           Ext = list_to_atom(tl(filename:extension(FName))),
                           Master ! {write, {Ext, FName, Contents}},
                           ok
                   end,
    FileOpOpt = mk_fileop_opt([{read_file, ReadInput},
                               {write_file, ReportOutput}]),
    ModPrefix = "mp_",
    MsgPrefix = "mm_",
    ModSuffix = "_xp",
    MsgSuffix = "_xm",
    ok = gpb_compile:file("m.proto",
                          [FileOpOpt, {i,"."},
                           {module_name_prefix, ModPrefix},
                           {msg_name_prefix, MsgPrefix},
                           {module_name_suffix, ModSuffix},
                           {msg_name_suffix, MsgSuffix}]),
    receive
        {read, "m.proto"} -> ok;
        {read, X} -> erlang:error("reading from odd file", X)
    end,
    receive
        {write, {hrl, "mp_m_xp.hrl", Hrl}} ->
            assert_contains_regexp(Hrl, "mm_msg1_xm"),
            ok;
        {write, {hrl, "m.hrl", _}} ->
            erlang:error("hrl file not prefixed or suffixed!");
        {write, {hrl, X2, C2}} ->
            erlang:error({"writing odd hrl file!", X2, C2})
    end,
    receive
        {write, {erl, "mp_m_xp.erl", Erl}} ->
            assert_contains_regexp(Erl, "-include.*\"mp_m_xp.hrl\""),
            assert_contains_regexp(Erl, "-module.*mp_m_xp"),
            assert_contains_regexp(Erl, "mm_msg1_xm"),
            ok;
        {write, {erl, "m.erl", _}} ->
            erlang:error("erl file not prefixed or suffixed!");
        {write, {erl, X3, C3}} ->
            erlang:error({"writing odd erl file!", X3, C3})
    end,
    ok.

assert_contains_regexp(IoData, Re) ->
    case re:run(IoData, Re) of
        {match, _} -> ok;
        nomatch    ->
            ?debugFmt("~nERROR: Regexp ~s not found in:~n~s~n", [Re, IoData]),
            erlang:error({"Re ", Re, "not found in", IoData})
    end.

%% --- bytes ----------

copy_bytes_unconditionally_test() ->
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>, % binary exists since R14A
    if HasBinary ->
            M = compile_iolist(["message m1 { required bytes f1 = 1; }"],
                               [{copy_bytes, true}]),
            Data = M:encode_msg({m1, <<"d">>}),
            {m1, <<"d">>=Bs} = M:decode_msg(Data, m1),
            %% If the Bs has not been copied, then it is a sub-binary
            %% of a larger binary: of the message, ie of Data.
            %% So verify copying by verifying size of referenced data.
            ?assertEqual(byte_size(Bs), binary:referenced_byte_size(Bs)),
            unload_code(M);
       true ->
            %% nothing to test
            ok
    end.

copy_bytes_false_test() ->
    M = compile_iolist(["message m1 { required bytes f1 = 1; }"],
                       [{copy_bytes, false}]),
    Data = M:encode_msg({m1, <<"d">>}),
    {m1, <<"d">>=Bs} = M:decode_msg(Data, m1),
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>, % binary exists since R14A
    if HasBinary ->
            %% If the StrBin has not been copied, then it is a sub-binary
            %% of a larger binary: of the message, ie of Data.
            %% So verify copying by verifying size of referenced data.
            ?assertEqual(byte_size(Data), binary:referenced_byte_size(Bs));
       true ->
            ok
    end,
    unload_code(M).

copy_bytes_auto_test() ->
    M = compile_iolist(["message m1 { required bytes f1 = 1; }"],
                       [{copy_bytes, auto}]),
    Data = M:encode_msg({m1, <<"d">>}),
    {m1, <<"d">>=Bs} = M:decode_msg(Data, m1),
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>,
    if HasBinary ->
            ?assertEqual(byte_size(Bs), binary:referenced_byte_size(Bs));
       true ->
            ok %% cannot test more if we don't have the binary module
    end,
    unload_code(M).

copy_bytes_fraction_test() ->
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>,
    if HasBinary ->
            Proto = ["message m1 {",
                     "  required bytes f1 = 1;",
                     "  required bytes f2 = 2;",
                     "}"],
            M1 = compile_iolist(Proto, [{copy_bytes, 2}]),   % fraction as int
            M2 = compile_iolist(Proto, [{copy_bytes, 2.0}]), % fraction as float
            D1 = <<"d">>, %% small
            D2 = <<"dddddddddddddddddddddddddddd">>, %% large
            Data = M1:encode_msg({m1, D1, D2}),
            ?assert(byte_size(Data) > (2 * byte_size(D1))),
            ?assert(byte_size(Data) < (2 * byte_size(D2))),
            {m1, D1Bs, D2Bs} = M1:decode_msg(Data, m1),
            ?assertEqual(D1, D1Bs),
            ?assertEqual(D2, D2Bs),
            %% The small data should have been copied, but not the larger
            ?assertEqual(byte_size(D1Bs), binary:referenced_byte_size(D1Bs)),
            ?assertEqual(byte_size(Data), binary:referenced_byte_size(D2Bs)),

            {m1, D3Bs, D4Bs} = M2:decode_msg(Data, m1),
            ?assertEqual(D1, D3Bs),
            ?assertEqual(D2, D4Bs),
            ?assertEqual(byte_size(D3Bs), binary:referenced_byte_size(D3Bs)),
            ?assertEqual(byte_size(Data), binary:referenced_byte_size(D4Bs)),

            unload_code(M1),
            unload_code(M2);
       true ->
            ok
    end.

%% --- strings ----------

strings_as_binaries_option_produces_bins_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "}"],
                       [strings_as_binaries]),
    Data = M:encode_msg({m1, "some string"}),
    {m1, <<"some string">>} = M:decode_msg(Data, m1),
    unload_code(M).

strings_as_lists_is_the_default_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "}"],
                       []),
    Data = M:encode_msg({m1, "some string"}),
    {m1, "some string"} = M:decode_msg(Data, m1),
    unload_code(M).

strings_as_binaries_opt_together_with_copy_bytes_opt_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "}"],
                       [strings_as_binaries, {copy_bytes, auto}]),
    Data = M:encode_msg({m1, "some string"}),
    {m1, <<"some string">>=StrBin} = M:decode_msg(Data, m1),
    HasBinary = (catch binary:copy(<<1>>)) == <<1>>, % binary exists since R14A
    if HasBinary ->
            ?assertEqual(byte_size(StrBin),
                         binary:referenced_byte_size(StrBin));
       true ->
            ok
    end,
    unload_code(M).

accepts_both_strings_and_binaries_as_input_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "  required string f2 = 2;",
                        "}"]),
    Data = M:encode_msg({m1, "some string", <<"some other string">>}),
    {m1, "some string", "some other string"} = M:decode_msg(Data, m1),
    unload_code(M).

verifies_both_strings_and_binaries_as_input_test() ->
    M = compile_iolist(["message m1 {"
                        "  required string f1 = 1;",
                        "  required string f2 = 2;",
                        "}"],
                        [strings_as_binaries]),
    R = {m1, "some string", <<"some other string">>},
    ok = M:verify_msg(R),
    ?assertError(_, M:verify_msg({m1, "a", <<97,98,99,255,191>>})),
    unload_code(M).

%% --- misc ----------

only_enums_no_msgs_test() ->
    M = compile_iolist(["enum e {"
                        "  a = 1;",
                        "}"]),
    ?assertError({gpb_error, no_messages}, M:encode_msg({x})),
    ?assertError({gpb_error, no_messages}, M:encode_msg({x}, [])),
    ?assertError({gpb_error, no_messages}, M:decode_msg(<<>>, x)),
    ?assertError({gpb_error, no_messages}, M:merge_msgs({x}, {x})),
    ?assertError({gpb_type_error, {not_a_known_message, _}}, M:verify_msg({x})),
    [] = M:get_msg_names(),
    [e] = M:get_enum_names(),
    unload_code(M).

%% --- Returning/reporting warnings/errors (and warnings_as_errors) tests -----
%% ... when compiling to file/binary/defs
%% ... when compiling from file/defs
%% ... when there are/aren't warnings/errors

report_or_return_warnings_or_errors_test_() ->
    %% Without increased timeout, this test sometimes times out
    %% on my slow machine (1.6 GHz Atom N270)
    {timeout,58,fun report_or_return_warnings_or_errors_test_aux/0}.

report_or_return_warnings_or_errors_test_aux() ->
    [begin
         Options = WarningOptions ++ ErrorOptions ++ WarnsAsErrsOpts,
         try
             rwre_go(Options, CompileTo, SrcType, SrcQuality)
         catch Class:Reason ->
                 Stack = erlang:get_stacktrace(),
                 %% Need some trouble shooting info for the failing combination
                 %% This could have been made into a test generator,
                 %% with each combination its won test,
                 %% but in total 544 tests are executed, and if running
                 %% with verbose mode, it'll always be half a thousand lines
                 %% of (almost) non-interesting info.
                 ?debugFmt("~nFailed for~n"
                           "   Options=~p~n"
                           "   CompileTo=~p~n"
                           "   SrcType=~p~n"
                           "   SrcQuality=~p~n",
                           [Options, CompileTo, SrcType, SrcQuality]),
                 erlang:raise(Class, Reason, Stack)
         end
     end
     || WarningOptions     <- [[], [report_warnings], [return_warnings],
                               [report_warnings, return_warnings]],
        ErrorOptions       <- [[], [report_errors], [return_errors],
                               [report_errors, return_errors]],
        WarnsAsErrsOpts    <- [[], [warnings_as_errors]],
        CompileTo          <- [to_binary, to_file, to_proto_defs],
        SrcType            <- [from_file, from_defs],
{'fraction_non_alphanumeric': 0.16496265680516278, 'fraction_numerical': 0.027717351896164165, 'mean_word_length': 2.2686496415239676, 'pattern_counts': {'":': 0, '<': 149, '<?xml version=': 0, '>': 298, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3071701', 'n_tokens_mistral': 20239, 'n_tokens_neox': 19278, 'n_words': 3821}