<commit_before>defmodule FileProcessor do
  use Application

  @file_checks [
    {&FileProcessor.file_exists?/1, :not_found}
  ]

  # See http://elixir-lang.org/docs/stable/elixir/Application.html
  # for more information on OTP Applications
  def start(_type, _args) do
    import Supervisor.Spec, warn: false

    # Define workers and child supervisors to be supervised
    children = [
      # Starts a worker by calling: FileProcessor.Worker.start_link(arg1, arg2, arg3)
      # worker(FileProcessor.Worker, [arg1, arg2, arg3]),
    ]

    # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
    # for other strategies and supported options
    opts = [strategy: :one_for_one, name: FileProcessor.Supervisor]
    Supervisor.start_link(children, opts)
  end

  def process(file_metadata) do
    file_metadata
    |> do_file_checks
    |> do_process
  end

  defp do_file_checks(file_metadata), do: do_file_checks(@file_checks, file_metadata)
  defp do_file_checks([], file_metadata), do: {:ok, file_metadata}
  defp do_file_checks([{check?, error} | remaining_checks], file_metadata) do
    if check?.(file_metadata) do
      do_file_checks(remaining_checks, file_metadata)
    else
      {:error, error, file_metadata}
    end
  end

  defp do_process({:error, error, file_metadata}) do
    log_bad_file(error, file_metadata)
    {:error, file_metadata}
  end

  defp do_process({:ok, file_metadata}) do
    file_metadata
    |> inspect
    |> IO.puts
    {:ok, file_metadata}
  end

  def file_exists?(file_metadata) do
    File.exists? file_metadata.filename
  end
<commit_msg>Refactor<commit_after>defmodule FileProcessor do
  use Application

  @processing_steps [
    :check_file_exists,
    :check_data_type,
    :clean_file
  ]

  # See http://elixir-lang.org/docs/stable/elixir/Application.html
  # for more information on OTP Applications
  def start(_type, _args) do
    import Supervisor.Spec, warn: false

    # Define workers and child supervisors to be supervised
    children = [
      # Starts a worker by calling: FileProcessor.Worker.start_link(arg1, arg2, arg3)
      # worker(FileProcessor.Worker, [arg1, arg2, arg3]),
    ]

    # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
    # for other strategies and supported options
    opts = [strategy: :one_for_one, name: FileProcessor.Supervisor]
    Supervisor.start_link(children, opts)
  end

  def process(file_metadata) do
    case do_process @processing_steps, file_metadata do
    {:ok, file_metadata} ->
      {:ok, file_metadata}
    {:error, error, file_metadata} ->
      log_bad_file(error, file_metadata)
      {:error, error, file_metadata}
    end
  end

  defp do_process([], file_metadata), do: {:ok, file_metadata}

  defp do_process([:check_file_exists | remaining_steps], file_metadata) do
    if File.exists? file_metadata.filename do
      do_process remaining_steps, file_metadata
    else
      {:error, :not_found, file_metadata}
    end
  end

{'fraction_non_alphanumeric': 0.09831649831649832, 'fraction_numerical': 0.004377104377104377, 'mean_word_length': 3.7384370015948964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '938385', 'n_tokens_mistral': 999, 'n_tokens_neox': 959, 'n_words': 230}