<commit_before>defmodule PublicSuffix do
  @moduledoc """
  Implements the publicsuffix algorithm described at https://publicsuffix.org/list/.
  Comments throughout this module are direct quotes from https://publicsuffix.org/list/,
  showing how individual lines of code relate to the specification.
  """

  @doc """
  Extracts the public suffix from the provided domain based on the publicsuffix.org rules.

  ## Examples
    iex> public_suffix("foo.bar.com")
    "com"

  You can use the `ignore_private` keyword to exclude private (non-ICANN) domains.

    iex> public_suffix("foo.github.io", ignore_private: false)
    "github.io"
    iex> public_suffix("foo.github.io", ignore_private: true)
    "io"
    iex> public_suffix("foo.github.io")
    "github.io"
  """
  def public_suffix(domain, options \\ [ignore_private: false]) when is_binary(domain) do
    parse_domain(domain, options, 0)
  end

  @doc """
  Extracts the _registrable_ part of the provided domain. The registrable
  part is the public suffix plus one additional domain part. For example,
  given a public suffix of `co.uk`, so `example.co.uk` would be the registrable
  domain part.

  ## Examples
    iex> registrable_domain("foo.bar.com")
    "bar.com"

  You can use the `ignore_private` keyword to exclude private (non-ICANN) domains.

    iex> registrable_domain("foo.github.io", ignore_private: false)
    "foo.github.io"
    iex> registrable_domain("foo.github.io", ignore_private: true)
    "github.io"
    iex> registrable_domain("foo.github.io")
    "foo.github.io"
  """
  @spec registrable_domain(String.t) :: nil | String.t
  @spec registrable_domain(String.t, ignore_private: boolean) :: nil | String.t
  def registrable_domain(domain, options \\ [ignore_private: false]) when is_binary(domain) do
    # "The registered or registrable domain is the public suffix plus one additional label."
    parse_domain(domain, options, 1)
  end

  defp parse_domain(domain, options, extra_label_parts) do
    domain
    # "The domain...must be canonicalized in the normal way for hostnames - lower-case"
    |> String.downcase
    # "Empty labels are not permitted, meaning that leading and trailing dots are ignored."
    |> String.strip(?.)
    # "A domain or rule can be split into a list of labels using the separator "." (dot)."
    |> String.split(".")
    |> extract_labels_using_rules(extra_label_parts, options)
    |> case do
         nil -> nil
         labels -> Enum.join(labels, ".")
       end
  end

  defp extract_labels_using_rules(labels, extra_label_parts, options) do
    allowed_rule_types = allowed_rule_types_for(options)
<commit_msg>PR comment follow-ups<commit_after>defmodule PublicSuffix do
  @moduledoc """
  Implements the publicsuffix algorithm described at https://publicsuffix.org/list/.
  Comments throughout this module are direct quotes from https://publicsuffix.org/list/,
  showing how individual lines of code relate to the specification.
  """

  @type options :: [ignore_private: boolean]

  @doc """
  Extracts the public suffix from the provided domain based on the publicsuffix.org rules.

  ## Examples
    iex> public_suffix("foo.bar.com")
    "com"

  You can use the `ignore_private` keyword to exclude private (non-ICANN) domains.

    iex> public_suffix("foo.github.io", ignore_private: false)
    "github.io"
    iex> public_suffix("foo.github.io", ignore_private: true)
    "io"
    iex> public_suffix("foo.github.io")
    "github.io"
  """
  @spec public_suffix(String.t) :: nil | String.t
  @spec public_suffix(String.t, options) :: nil | String.t
  def public_suffix(domain, options \\ []) when is_binary(domain) do
    parse_domain(domain, options, 0)
  end

  @doc """
  Extracts the _registrable_ part of the provided domain. The registrable
  part is the public suffix plus one additional domain part. For example,
  given a public suffix of `co.uk`, so `example.co.uk` would be the registrable
  domain part. If the domain does not contain a registrable part (for example,
  if the domain is itself a public suffix), this function will return `nil`.

  ## Examples
    iex> registrable_domain("foo.bar.com")
    "bar.com"
    iex> registrable_domain("com")
    nil

  You can use the `ignore_private` keyword to exclude private (non-ICANN) domains.

    iex> registrable_domain("foo.github.io", ignore_private: false)
    "foo.github.io"
    iex> registrable_domain("foo.github.io", ignore_private: true)
    "github.io"
    iex> registrable_domain("foo.github.io")
    "foo.github.io"
  """
  @spec registrable_domain(String.t) :: nil | String.t
  @spec registrable_domain(String.t, options) :: nil | String.t
  def registrable_domain(domain, options \\ []) when is_binary(domain) do
    # "The registered or registrable domain is the public suffix plus one additional label."
    parse_domain(domain, options, 1)
  end

  defp parse_domain(domain, options, extra_label_parts) do
    domain
    # "The domain...must be canonicalized in the normal way for hostnames - lower-case"
    |> String.downcase
    # "Empty labels are not permitted, meaning that leading and trailing dots are ignored."
    |> String.strip(?.)
    # "A domain or rule can be split into a list of labels using the separator "." (dot)."
    |> String.split(".")
    |> extract_labels_using_rules(extra_label_parts, options)
    |> case do
         nil -> nil
         labels -> Enum.join(labels, ".")
       end
  end

  defp extract_labels_using_rules(labels, extra_label_parts, options) do
    allowed_rule_types = allowed_rule_types_for(options)

{'fraction_non_alphanumeric': 0.1052347544522396, 'fraction_numerical': 0.0007195538765965102, 'mean_word_length': 3.695945945945946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 34, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1637978', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1676, 'n_words': 537}