<commit_before>defmodule GraphemesTest do
  def run do
    IO.puts("Running GraphemeBreakTest.txt")
    count = run_grapheme_break()
    IO.puts("Got #{count} failures")
  end

  defp run_grapheme_break do
    Path.join(__DIR__, "GraphemeBreakTest.txt")
    |> File.stream!()
    |> Stream.filter(&match?("÷" <> _, &1))
    |> Stream.reject(&(&1 =~ "D800"))
    |> Enum.reduce(0, fn line, acc ->
      [string | _] = String.split(line, "#", parts: 2)
      {string, graphemes} = parse_grapheme_break(string)

      if String.graphemes(string) == graphemes do
        acc
      else
        acc = acc + 1

        IO.puts("""
        ============== Failure ##{acc} ==============

            String.graphemes(#{inspect(string)})

        must be:

            #{inspect(graphemes)}

        got:

            #{inspect(String.graphemes(string))}

        On line:

            #{line}
        """)

        acc
      end
    end)
  end

  defp parse_grapheme_break(string) do
    string
    |> String.trim()
    |> String.trim_leading("÷ ")
    |> String.trim_trailing(" ÷")<commit_msg>Print number of tests from graphemes tests<commit_after>defmodule GraphemesTest do
  def run do
    IO.puts("Running GraphemeBreakTest.txt")
    {cases, failures} = run_grapheme_break()
    IO.puts("Ran #{cases} tests, got #{failures} failures")
  end

  defp run_grapheme_break do
    Path.join(__DIR__, "GraphemeBreakTest.txt")
    |> File.stream!()
    |> Stream.filter(&match?("÷" <> _, &1))
    |> Stream.reject(&(&1 =~ "D800"))
    |> Enum.reduce({0, 0}, fn line, {cases, failures} ->
      [string | _] = String.split(line, "#", parts: 2)
      {string, graphemes} = parse_grapheme_break(string)

      if String.graphemes(string) == graphemes do
        {cases + 1, failures}
      else
        IO.puts("""
        ============== Failure ##{failures + 1} ==============

            String.graphemes(#{inspect(string)})

        must be:

            #{inspect(graphemes)}

        got:

            #{inspect(String.graphemes(string))}

        On line:

            #{line}
        """)

        {cases + 1, failures + 1}
      end
    end)
  end

  defp parse_grapheme_break(string) do
    string
    |> String.trim()
    |> String.trim_leading("÷ ")
    |> String.trim_trailing(" ÷")
{'fraction_non_alphanumeric': 0.17404647084612013, 'fraction_numerical': 0.008768084173608066, 'mean_word_length': 2.3807407407407406, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7042754', 'n_tokens_mistral': 837, 'n_tokens_neox': 784, 'n_words': 150}