<commit_before>defmodule ExTikaTest do
  use ExUnit.Case
  doctest ExTika

  @files_dir Path.join(Path.expand(__DIR__), "test-files")

  test "get_text" do
    get_text = fn(name) ->
      name
      |> test_file
      |> ExTika.get_text
      |> trim
    end

    assert get_text.("test.doc") == {:ok, "This is a DOC file."}
    assert get_text.("test.docx") == {:ok, "This is a DOCX file."}
  end

  test "get_text!" do
    text = "test.doc"
            |> test_file
            |> ExTika.get_text!
            |> String.trim
    assert text == "This is a DOC file."
  end


  defp trim({:ok, text}) do
    {:ok, trim(text)}
  end

  defp trim({:error, msg}) do
    {:error, msg}
  end

  defp trim(s) when is_binary(s) do
    if :erlang.function_exported(String, :trim, 1) do
      String.trim(s)
    else
      String.strip(s)
    end
  end

  defp test_file(name) do
    Path.join(@files_dir, name)
  end
end
<commit_msg>Actually use our trim function<commit_after>defmodule ExTikaTest do
  use ExUnit.Case
  doctest ExTika

  @files_dir Path.join(Path.expand(__DIR__), "test-files")

  test "get_text" do
    get_text = fn(name) ->
      name
      |> test_file
      |> ExTika.get_text
      |> trim
    end

    assert get_text.("test.doc") == {:ok, "This is a DOC file."}
    assert get_text.("test.docx") == {:ok, "This is a DOCX file."}
  end

  test "get_text!" do
    text = "test.doc"
            |> test_file
            |> ExTika.get_text!
            |> trim
    assert text == "This is a DOC file."
  end


  defp trim({:ok, text}) do
    {:ok, trim(text)}
  end

  defp trim({:error, msg}) do
    {:error, msg}
  end

  defp trim(s) when is_binary(s) do
    if :erlang.function_exported(String, :trim, 1) do
      String.trim(s)
    else
      String.strip(s)
    end
  end

  defp test_file(name) do
    Path.join(@files_dir, name)
  end
end

{'fraction_non_alphanumeric': 0.1348012889366273, 'fraction_numerical': 0.0010741138560687433, 'mean_word_length': 2.3031914893617023, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1316992', 'n_tokens_mistral': 736, 'n_tokens_neox': 692, 'n_words': 182}