<commit_before>#!/usr/bin/env escript
%% -*- erlang -*-
%%! -smp enable -sname kpro_gen

%%%
%%%   Copyright (c) 2014-2016, Klarna AB
%%%
%%%   Licensed under the Apache License, Version 2.0 (the "License");
%%%   you may not use this file except in compliance with the License.
%%%   You may obtain a copy of the License at
%%%
%%%       http://www.apache.org/licenses/LICENSE-2.0
%%%
%%%   Unless required by applicable law or agreed to in writing, software
%%%   distributed under the License is distributed on an "AS IS" BASIS,
%%%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%%%   See the License for the specific language governing permissions and
%%%   limitations under the License.
%%%

-mode(compile).

-include("../include/kpro_common.hrl").

main(_) ->
  ok = file:set_cwd(this_dir()),
  {ok, _} = leex:file(kpro_scanner),
  {ok, _} = compile:file(kpro_scanner, [debug_info]),
  {ok, _} = yecc:file(kpro_parser),
  {ok, _} = compile:file(kpro_parser, [debug_info]),
  {ok, Contents} = file:read_file("kafka.bnf"),
  {ok, Tokens, _EndLine} = kpro_scanner:string(binary_to_list(Contents)),
  {ok, DefGroups} = kpro_parser:parse(Tokens),
  Records = to_records(DefGroups),
  generate_code(Records).

this_dir() ->
  ThisScript = escript:script_name(),
  filename:dirname(ThisScript).
<commit_msg>Add some documentation to priv/kpro_gen.escript<commit_after>#!/usr/bin/env escript
%% -*- erlang -*-
%%! -smp enable -sname kpro_gen

%%%
%%%   Copyright (c) 2014-2016, Klarna AB
%%%
%%%   Licensed under the Apache License, Version 2.0 (the "License");
%%%   you may not use this file except in compliance with the License.
%%%   You may obtain a copy of the License at
%%%
%%%       http://www.apache.org/licenses/LICENSE-2.0
%%%
%%%   Unless required by applicable law or agreed to in writing, software
%%%   distributed under the License is distributed on an "AS IS" BASIS,
%%%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%%%   See the License for the specific language governing permissions and
%%%   limitations under the License.
%%%

%% This script reads definitions of kafka data structures from priv/kafka.bnf
%% and generates:
%%   include/kpro.hrl -- erlang record definitions of and type annotations
%%   src/kpro_structs.erl -- encoders and decoders to the wire format
%%   src/kpro_records.erl -- fields() helper to convert records to maps of maps
%%
%% Assumptions about priv/kafka.bnf: field names are unique across each top
%% level definition, so we're able to not nest names for subrecords of
%% subrecords. Subrecords and subsubrecords are indented, top level records are not.
%%
%% Top level record and type names are lowercased, changed from plural to singular,
%% "underscored", and prefixed with "kpro_":
%%   ProduceRequestV1 becomes kpro_request_v1().
%%
%% Record names for subsubrecords and types additionaly prefixed with top level
%% record name:
%%   partition_responses in responses in ProduceResponseV1 becomes
%%     kpro_produce_response_v1_partition_response().
%%
%% Field names are left exactly as they defined in kafka.bnf.

-mode(compile).

-include("../include/kpro_common.hrl").

main(_) ->
  ok = file:set_cwd(this_dir()),
  {ok, _} = leex:file(kpro_scanner),
  {ok, _} = compile:file(kpro_scanner, [debug_info]),
  {ok, _} = yecc:file(kpro_parser),
  {ok, _} = compile:file(kpro_parser, [debug_info]),
  {ok, Contents} = file:read_file("kafka.bnf"),
  {ok, Tokens, _EndLine} = kpro_scanner:string(binary_to_list(Contents)),
  {ok, DefGroups} = kpro_parser:parse(Tokens),
  Records = to_records(DefGroups),
  generate_code(Records).

this_dir() ->
  ThisScript = escript:script_name(),
  filename:dirname(ThisScript).

{'fraction_non_alphanumeric': 0.13408420488066505, 'fraction_numerical': 0.007508715473317243, 'mean_word_length': 4.477239353891337, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '620904', 'n_tokens_mistral': 1222, 'n_tokens_neox': 1205, 'n_words': 370}