<commit_before>defmodule Set do
  @moduledoc ~S"""
  WARNING: this module is deprecated.

  Use the `MapSet` module instead.
  """

  @type value :: any
  @type values :: [ value ]
  @type t :: map

  # TODO: Remove callbacks on 1.3
  # TODO: Deprecate every function on 1.3
  @callback new :: t
  @callback delete(t, value) :: t
  @callback difference(t, t) :: t
  @callback disjoint?(t, t) :: boolean
  @callback equal?(t, t) :: boolean
  @callback intersection(t, t) :: t
  @callback member?(t, value) :: boolean
  @callback put(t, value) :: t
  @callback size(t) :: non_neg_integer
  @callback subset?(t, t) :: boolean
  @callback to_list(t) :: list()
  @callback union(t, t) :: t

  defmacrop target(set) do
    quote do
      case unquote(set) do
        %{__struct__: x} when is_atom(x) ->
          x
        x ->
          unsupported_set(x)
      end
    end
  end

  def delete(set, value) do
    target(set).delete(set, value)
  end

  def difference(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.difference(set1, set2)
    else
      Enumerable.reduce(set2, {:cont, set1}, fn v, acc ->
        {:cont, target1.delete(acc, v)}
      end) |> elem(1)
    end
  end

  def disjoint?(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.disjoint?(set1, set2)
    else
      Enumerable.reduce(set2, {:cont, true}, fn member, acc ->
        case target1.member?(set1, member) do
          false -> {:cont, acc}
          _     -> {:halt, false}
        end
      end) |> elem(1)
    end
  end

  @doc false
  def empty(set) do
    target(set).empty(set)
  end

  def equal?(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    cond do
      target1 == target2 ->
        target1.equal?(set1, set2)

      target1.size(set1) == target2.size(set2) ->
        do_subset?(target1, target2, set1, set2)

      true ->
        false
    end
  end


  def intersection(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.intersection(set1, set2)
    else
      Enumerable.reduce(set1, {:cont, target1.new}, fn v, acc ->
        {:cont, if(target2.member?(set2, v), do: target1.put(acc, v), else: acc)}
      end) |> elem(1)
    end
  end


  def member?(set, value) do
    target(set).member?(set, value)
  end


  def put(set, value) do
    target(set).put(set, value)
  end

  def size(set) do
    target(set).size(set)
  end

  def subset?(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.subset?(set1, set2)
    else
      do_subset?(target1, target2, set1, set2)
    end
  end

  def to_list(set) do
    target(set).to_list(set)
  end

  def union(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.union(set1, set2)
    else
      Enumerable.reduce(set2, {:cont, set1}, fn v, acc ->
        {:cont, target1.put(acc, v)}
      end) |> elem(1)
    end
  end

  defp do_subset?(target1, target2, set1, set2) do
    Enumerable.reduce(set1, {:cont, true}, fn member, acc ->
      case target2.member?(set2, member) do
        true -> {:cont, acc}
        _    -> {:halt, false}
      end
    end) |> elem(1)
  end

  defp unsupported_set(set) do
    raise ArgumentError, "unsupported set: #{inspect set}"
  end
end<commit_msg>Suppress warning in set file<commit_after>defmodule Set do
  @moduledoc ~S"""
  WARNING: this module is deprecated.

  Use the `MapSet` module instead.
  """

  @type value :: any
  @type values :: [ value ]
  @type t :: map

  # TODO: Remove callbacks on 1.3
  # TODO: Deprecate every function on 1.3
  @callback new :: t
  @callback delete(t, value) :: t
  @callback difference(t, t) :: t
  @callback disjoint?(t, t) :: boolean
  @callback equal?(t, t) :: boolean
  @callback intersection(t, t) :: t
  @callback member?(t, value) :: boolean
  @callback put(t, value) :: t
  @callback size(t) :: non_neg_integer
  @callback subset?(t, t) :: boolean
  @callback to_list(t) :: list()
  @callback union(t, t) :: t

  defmacrop target(set) do
    quote do
      case unquote(set) do
        %{__struct__: x} when is_atom(x) ->
          x
        x ->
          unsupported_set(x)
      end
    end
  end

  def delete(set, value) do
    target(set).delete(set, value)
  end

  def difference(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.difference(set1, set2)
    else
      Enumerable.reduce(set2, {:cont, set1}, fn v, acc ->
        {:cont, target1.delete(acc, v)}
      end) |> elem(1)
    end
  end

  def disjoint?(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.disjoint?(set1, set2)
    else
      Enumerable.reduce(set2, {:cont, true}, fn member, acc ->
        case target1.member?(set1, member) do
          false -> {:cont, acc}
          _     -> {:halt, false}
        end
      end) |> elem(1)
    end
  end

  @doc false
  def empty(set) do
    target(set).empty(set)
  end

  def equal?(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    cond do
      target1 == target2 ->
        target1.equal?(set1, set2)

      target1.size(set1) == target2.size(set2) ->
        do_subset?(target1, target2, set1, set2)

      true ->
        false
    end
  end


  def intersection(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.intersection(set1, set2)
    else
      Enumerable.reduce(set1, {:cont, target1.new}, fn v, acc ->
        {:cont, if(target2.member?(set2, v), do: target1.put(acc, v), else: acc)}
      end) |> elem(1)
    end
  end


  def member?(set, value) do
    target(set).member?(set, value)
  end


  def put(set, value) do
    target(set).put(set, value)
  end

  def size(set) do
    target(set).size(set)
  end

  def subset?(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.subset?(set1, set2)
    else
      do_subset?(target1, target2, set1, set2)
    end
  end

  def to_list(set) do
    target(set).to_list(set)
  end

  def union(set1, set2) do
    target1 = target(set1)
    target2 = target(set2)

    if target1 == target2 do
      target1.union(set1, set2)
    else
      Enumerable.reduce(set2, {:cont, set1}, fn v, acc ->
        {:cont, target1.put(acc, v)}
      end) |> elem(1)
    end
  end

  defp do_subset?(_target1, target2, set1, set2) do
    Enumerable.reduce(set1, {:cont, true}, fn member, acc ->
      case target2.member?(set2, member) do
        true -> {:cont, acc}
        _    -> {:halt, false}
      end
    end) |> elem(1)
  end

  defp unsupported_set(set) do
    raise ArgumentError, "unsupported set: #{inspect set}"
  end
end
{'fraction_non_alphanumeric': 0.13260305563562985, 'fraction_numerical': 0.031132891323147883, 'mean_word_length': 2.440257808626673, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5591802', 'n_tokens_mistral': 2584, 'n_tokens_neox': 2505, 'n_words': 685}