<commit_before># A module to include for easy access to writing to a transient filesystem
module LocalLogFile
  def working_directory
    @working_directory ||= "/tmp/" + \
      Digest::SHA1.hexdigest([name_with_owner, github_token].join)
  end

  def checkout_directory
    "#{working_directory}/checkout"
  end

  def stdout_file
    "#{working_directory}/stdout.#{guid}.log"
  end

  def stderr_file
    "#{working_directory}/stderr.#{guid}.log"
  end

  def log_stdout(out)
    File.open(stdout_file, "a") { |f| f.write(out.force_encoding("utf-8")) }
  end

  def log_stderr(err)
    File.open(stderr_file, "a") { |f| f.write(err.force_encoding("utf-8")) }
  end

  def execute_and_log(cmds, env = {})
    @last_child = POSIX::Spawn::Child.new(env.merge("HOME" => working_directory), *cmds)

    log_stdout(last_child.out)
    log_stderr(last_child.err)

    unless last_child.success?
      fail StandardError "Task failed: #{cmds.join(" ")}"
    end

    last_child
  end
end<commit_msg>Add missing comma for fail in LocalLogFile#execute_and_log<commit_after># A module to include for easy access to writing to a transient filesystem
module LocalLogFile
  def working_directory
    @working_directory ||= "/tmp/" + \
      Digest::SHA1.hexdigest([name_with_owner, github_token].join)
  end

  def checkout_directory
    "#{working_directory}/checkout"
  end

  def stdout_file
    "#{working_directory}/stdout.#{guid}.log"
  end

  def stderr_file
    "#{working_directory}/stderr.#{guid}.log"
  end

  def log_stdout(out)
    File.open(stdout_file, "a") { |f| f.write(out.force_encoding("utf-8")) }
  end

  def log_stderr(err)
    File.open(stderr_file, "a") { |f| f.write(err.force_encoding("utf-8")) }
  end

  def execute_and_log(cmds, env = {})
    @last_child = POSIX::Spawn::Child.new(env.merge("HOME" => working_directory), *cmds)

    log_stdout(last_child.out)
    log_stderr(last_child.err)

    unless last_child.success?
      fail StandardError, "Task failed: #{cmds.join(" ")}"
    end

    last_child
  end
end
{'fraction_non_alphanumeric': 0.13470993117010815, 'fraction_numerical': 0.0029498525073746312, 'mean_word_length': 3.927360774818402, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4653001', 'n_tokens_mistral': 729, 'n_tokens_neox': 711, 'n_words': 143}