<commit_before>defmodule CucumberGherkin.Lexicon do
  @moduledoc false
  @default_lexicon_path [File.cwd!(), "priv", "gherkin_languages.json"] |> Path.join()
  @feature_keywords ["feature"]
  @scens_both ["scenario", "scenarioOutline"]
  @step_keywords ["given", "when", "then", "and", "but"]
  @background_keywords ["background"]
  @rule_keywords ["rule"]
  @examples_keywords ["examples"]

  # Credit goes to Alvivi for loading the resource at compile time.
  @external_resource @default_lexicon_path

  @lexicon @default_lexicon_path |> File.read!() |> Jason.decode!()

  def load_lang(lang) when is_binary(lang), do: Map.fetch(@lexicon, lang)

  def load_keywords(FeatureLine, lex), do: fetch_and_flatten(@feature_keywords, lex)
  def load_keywords(ScenarioLine, lex), do: fetch_and_flatten(@scens_both, lex)
  def load_keywords(StepLine, lex), do: fetch_and_flatten(@step_keywords, lex)
  def load_keywords(BackgroundLine, lex), do: fetch_and_flatten(@background_keywords, lex)
  def load_keywords(RuleLine, lex), do: fetch_and_flatten(@rule_keywords, lex)
  def load_keywords(ExamplesLine, lex), do: fetch_and_flatten(@examples_keywords, lex)

  defp fetch_and_flatten(keywords, lexicon) do
    keywords |> Enum.map(&Map.fetch!(lexicon, &1)) |> List.flatten()
  end
end
<commit_msg>use  instead of "priv"<commit_after>defmodule CucumberGherkin.Lexicon do
  @moduledoc false
  @default_lexicon_path [:code.priv_dir(:cucumber_gherkin), "gherkin_languages.json"]
                        |> Path.join()
  @feature_keywords ["feature"]
  @scens_both ["scenario", "scenarioOutline"]
  @step_keywords ["given", "when", "then", "and", "but"]
  @background_keywords ["background"]
  @rule_keywords ["rule"]
  @examples_keywords ["examples"]

  # Credit goes to Alvivi for loading the resource at compile time.
  @external_resource @default_lexicon_path

  @lexicon @default_lexicon_path |> File.read!() |> Jason.decode!()

  def load_lang(lang) when is_binary(lang), do: Map.fetch(@lexicon, lang)

  def load_keywords(FeatureLine, lex), do: fetch_and_flatten(@feature_keywords, lex)
  def load_keywords(ScenarioLine, lex), do: fetch_and_flatten(@scens_both, lex)
  def load_keywords(StepLine, lex), do: fetch_and_flatten(@step_keywords, lex)
  def load_keywords(BackgroundLine, lex), do: fetch_and_flatten(@background_keywords, lex)
  def load_keywords(RuleLine, lex), do: fetch_and_flatten(@rule_keywords, lex)
  def load_keywords(ExamplesLine, lex), do: fetch_and_flatten(@examples_keywords, lex)

  defp fetch_and_flatten(keywords, lexicon) do
    keywords |> Enum.map(&Map.fetch!(lexicon, &1)) |> List.flatten()
  end
end

{'fraction_non_alphanumeric': 0.13114754098360656, 'fraction_numerical': 0.0007624857033930614, 'mean_word_length': 6.4124293785310735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4985071', 'n_tokens_mistral': 964, 'n_tokens_neox': 877, 'n_words': 196}