<commit_before>defmodule ParseLogs do
  use Application

  def start(_type, _args) do
    files = [
      {"base64-video", ParseLogs.Base64Video},
      {"base64-image", ParseLogs.Base64Image},
      {"sent-200", ParseLogs.Sent200},
    ]

    for {path, module} <- files, do: process(path, module)
    wait(0)
  end

  defp process(log_name, mod) do
    parent = self
    in_file = File.open! "log_data/#{log_name}.log", [:read, :utf8]
    out_file = File.open! "log_data/#{log_name}.csv", [:write, :utf8]

    spawn fn ->
      IO.puts out_file, mod.header

      for line <- IO.stream(in_file, :line) do
        parsed_line = mod.parse_line(line)
        if parsed_line, do: IO.puts(out_file, parsed_line)
      end

      File.close in_file
      File.close out_file
      send parent, :done
    end
  end

  defp wait(n) do
    if n == 2 do
      {:ok, self}
    else
      receive do
        :done ->
          IO.puts "Received a :done"
          wait(n+1)
      end
    end
  end<commit_msg>Change async code to use Task<commit_after>defmodule ParseLogs do
  use Application

  def start(_type, _args) do
    files = [
      {"base64-video", ParseLogs.Base64Video},
      {"base64-image", ParseLogs.Base64Image},
      {"sent-200", ParseLogs.Sent200},
    ]

    pids = Enum.map(files, fn({path, module}) ->
      Task.async(fn ->
        process(path, module)
      end)
    end)

    pids |> Enum.each(fn(pid) -> Task.await(pid, 30_000) end)
    {:ok, self}
  end

  defp process(log_name, mod) do
    in_file = File.open! "log_data/#{log_name}.log", [:read, :utf8]
    out_file = File.open! "log_data/#{log_name}.csv", [:write, :utf8]

    IO.puts out_file, mod.header

    for line <- IO.stream(in_file, :line) do
      parsed_line = mod.parse_line(line)
      if parsed_line, do: IO.puts(out_file, parsed_line)
    end

    File.close in_file
    File.close out_file
  end
{'fraction_non_alphanumeric': 0.1336870026525199, 'fraction_numerical': 0.021220159151193633, 'mean_word_length': 2.4925925925925925, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4072871', 'n_tokens_mistral': 725, 'n_tokens_neox': 690, 'n_words': 161}