<commit_before># Based on the rake remote task code

require 'rubygems'
require 'stringio'
require 'open4'

class RubyTikaApp

  class Error < RuntimeError; end

  class CommandFailedError < Error
    attr_reader :status
    def initialize(status)
      @status = status
    end
  end

  def initialize(document)
    if (document =~ /https?:\/\/[\S]+/) == 0
      @document = document
    else
      @document = "file://#{document}"
    end

    java_cmd = 'java'
    java_args = '-server -Djava.awt.headless=true'
    tika_path = "#{File.join(File.dirname(__FILE__))}/../ext/tika-app-1.14.jar"

    @tika_cmd = "#{java_cmd} #{java_args} -jar '#{tika_path}'"
  end

  def to_xml
    run_tika('--xml')
  end

  def to_html
    run_tika('--html')
  end

  def to_json
    run_tika('--json')
  end

  def to_text
    run_tika('--text')
  end

  def to_text_main
    run_tika('--text-main')
  end

  def to_metadata
    run_tika('--metadata')
  end

  private

  def run_tika(option)
    final_cmd = "#{@tika_cmd} #{option} '#{@document}'"

    pid, stdin, stdout, stderr = Open4::popen4(final_cmd)

    stdout_result = stdout.read.strip
    stderr_result = stderr.read.strip

    unless strip_stderr(stderr_result).empty?
      raise(CommandFailedError.new(stderr_result),
            "execution failed with status #{stderr_result}: #{final_cmd}")
    end

    stdout_result
  ensure
    stdin.close
    stdout.close
    stderr.close
  end

  def strip_stderr(s)
    s.gsub(/^(info|warn) - .*$/i, '').strip
  end

end<commit_msg>strip picked up message from heroku<commit_after># Based on the rake remote task code

require 'rubygems'
require 'stringio'
require 'open4'

class RubyTikaApp

  class Error < RuntimeError; end

  class CommandFailedError < Error
    attr_reader :status
    def initialize(status)
      @status = status
    end
  end

  def initialize(document)
    if (document =~ /https?:\/\/[\S]+/) == 0
      @document = document
    else
      @document = "file://#{document}"
    end

    java_cmd = 'java'
    java_args = '-server -Djava.awt.headless=true'
    tika_path = "#{File.join(File.dirname(__FILE__))}/../ext/tika-app-1.14.jar"

    @tika_cmd = "#{java_cmd} #{java_args} -jar '#{tika_path}'"
  end

  def to_xml
    run_tika('--xml')
  end

  def to_html
    run_tika('--html')
  end

  def to_json
    run_tika('--json')
  end

  def to_text
    run_tika('--text')
  end

  def to_text_main
    run_tika('--text-main')
  end

  def to_metadata
    run_tika('--metadata')
  end

  private

  def run_tika(option)
    final_cmd = "#{@tika_cmd} #{option} '#{@document}'"

    pid, stdin, stdout, stderr = Open4::popen4(final_cmd)

    stdout_result = stdout.read.strip
    stderr_result = stderr.read.strip

    unless strip_stderr(stderr_result).empty?
      raise(CommandFailedError.new(stderr_result),
            "execution failed with status #{stderr_result}: #{final_cmd}")
    end

    stdout_result
  ensure
    stdin.close
    stdout.close
    stderr.close
  end

  def strip_stderr(s)
    s
      .gsub(/^(info|warn) - .*$/i, '')
      .strip
      .gsub(/Picked up JAVA_TOOL_OPTIONS: .+ -Dfile.encoding=UTF-8/i, '')
      .strip
  end
end
{'fraction_non_alphanumeric': 0.13610586011342155, 'fraction_numerical': 0.004725897920604915, 'mean_word_length': 2.7707838479809976, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2184172', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1135, 'n_words': 248}