<commit_before>-define(key(M, K), maps:get(K, M)).
-define(ann(Opts), elixir_erl:get_ann(Opts)).
-define(line(Opts), elixir_utils:get_line(Opts)).
-define(generated(Meta), [{generated, true} | Meta]).
-define(var_context, ?MODULE).
-define(remote(Ann, Module, Function, Args), {call, Ann, {remote, Ann, {atom, Ann, Module}, {atom, Ann, Function}}, Args}).

-record(elixir_erl, {
  context=nil,             %% can be match, guards or nil
  extra=nil,               %% extra information about the context, like pin_guard and map_key
  caller=false,            %% when true, it means caller was invoked
  vars={#{}, false},       %% maps of defined variables and their alias
  backup_vars=nil,         %% a copy of vars to be used on ^var
  extra_guards=[],         %% extra guards from args expansion
  counter=#{},             %% a map counting the variables defined
  expand_captures=false,   %% a boolean to control if captures should be expanded
  stacktrace=false         %% holds information about the stacktrace variable
}).

-record(elixir_quote, {
  line=false,
  file=nil,
  context=nil,
  vars_hygiene=true,
  aliases_hygiene=true,
  imports_hygiene=true,
  unquote=true,
  generated=false
}).

-record(elixir_tokenizer, {
  file=(<<"nofile">>),
  terminators=[],
  unescape=true,
  check_terminators=true,
  existing_atoms_only=false,
  static_atoms_encoder=nil,
  preserve_comments=nil,
  identifier_tokenizer=elixir_tokenizer,
  indentation=0,
  mismatch_hints=[],
  warn_on_unnecessary_quotes=true
}).

%% TODO: Remove this once we support Erlang/OTP 22+ exclusively.
%% See https://github.com/erlang/otp/pull/1972
-if(?OTP_RELEASE >= 22).
  -define(NO_SPAWN_COMPILER_PROCESS, no_spawn_compiler_process).
-elif(?OTP_RELEASE >= 21).
  -define(NO_SPAWN_COMPILER_PROCESS, dialyzer, no_spawn_compiler_process).
-endif.<commit_msg>elif to else<commit_after>-define(key(M, K), maps:get(K, M)).
-define(ann(Opts), elixir_erl:get_ann(Opts)).
-define(line(Opts), elixir_utils:get_line(Opts)).
-define(generated(Meta), [{generated, true} | Meta]).
-define(var_context, ?MODULE).
-define(remote(Ann, Module, Function, Args), {call, Ann, {remote, Ann, {atom, Ann, Module}, {atom, Ann, Function}}, Args}).

-record(elixir_erl, {
  context=nil,             %% can be match, guards or nil
  extra=nil,               %% extra information about the context, like pin_guard and map_key
  caller=false,            %% when true, it means caller was invoked
  vars={#{}, false},       %% maps of defined variables and their alias
  backup_vars=nil,         %% a copy of vars to be used on ^var
  extra_guards=[],         %% extra guards from args expansion
  counter=#{},             %% a map counting the variables defined
  expand_captures=false,   %% a boolean to control if captures should be expanded
  stacktrace=false         %% holds information about the stacktrace variable
}).

-record(elixir_quote, {
  line=false,
  file=nil,
  context=nil,
  vars_hygiene=true,
  aliases_hygiene=true,
  imports_hygiene=true,
  unquote=true,
  generated=false
}).

-record(elixir_tokenizer, {
  file=(<<"nofile">>),
  terminators=[],
  unescape=true,
  check_terminators=true,
  existing_atoms_only=false,
  static_atoms_encoder=nil,
  preserve_comments=nil,
  identifier_tokenizer=elixir_tokenizer,
  indentation=0,
  mismatch_hints=[],
  warn_on_unnecessary_quotes=true
}).

%% TODO: Remove this once we support Erlang/OTP 22+ exclusively.
%% See https://github.com/erlang/otp/pull/1972
-if(?OTP_RELEASE >= 22).
  -define(NO_SPAWN_COMPILER_PROCESS, no_spawn_compiler_process).
-else.
  -define(NO_SPAWN_COMPILER_PROCESS, dialyzer, no_spawn_compiler_process).
-endif.
{'fraction_non_alphanumeric': 0.1320754716981132, 'fraction_numerical': 0.005468963631391851, 'mean_word_length': 4.263309352517986, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7026084', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1238, 'n_words': 292}