<commit_before>require "foreman"
require "rubygems"
require "spoon" if RUBY_PLATFORM == "java"

class Foreman::Process

  attr_reader :entry
  attr_reader :num
  attr_reader :pid
  attr_reader :port

  def initialize(entry, num, port)
    @entry = entry
    @num = num
    @port = port
  end

  def run(pipe, basedir, environment)
    Dir.chdir(basedir) do
      with_environment(environment.merge("PORT" => port.to_s)) do
        run_process entry.command, pipe
      end
    end
  end

  def name
    "%s.%s" % [ entry.name, num ]
  end

private

  def jruby?
    defined?(RUBY_PLATFORM) and RUBY_PLATFORM == "java"
  end

  def fork_with_io(command)
    reader, writer = IO.pipe
    command = replace_command_env(command)
    pid = if jruby?
      Spoon.spawnp Foreman.runner, command
    else
      fork do
        trap("INT", "IGNORE")
        writer.sync = true
        $stdout.reopen writer
        $stderr.reopen writer
        reader.close
        exec Foreman.runner, command
      end
    end
    [ reader, pid ]
  end

  def run_process(command, pipe)
    io, @pid = fork_with_io(command)
    output pipe, "started with pid %d" % @pid
    Thread.new do
      until io.eof?
        output pipe, io.gets
      end
    end
  end

  def output(pipe, message)
    pipe.puts "%s,%s" % [ name, message ]
  end

  def replace_command_env(command)
    command.gsub(/\$(\w+)/) { |e| ENV[e[1..-1]] }
  end

  def with_environment(environment)
    old_env = ENV.each_pair.inject({}) { |h,(k,v)| h.update(k => v) }
    environment.each { |k,v| ENV[k] = v }
    ret = yield
    ENV.clear
    old_env.each { |k,v| ENV[k] = v}
    ret<commit_msg>pass basedir along to the runner script<commit_after>require "foreman"
require "rubygems"
require "spoon" if RUBY_PLATFORM == "java"

class Foreman::Process

  attr_reader :entry
  attr_reader :num
  attr_reader :pid
  attr_reader :port

  def initialize(entry, num, port)
    @entry = entry
    @num = num
    @port = port
  end

  def run(pipe, basedir, environment)
    with_environment(environment.merge("PORT" => port.to_s)) do
      run_process basedir, entry.command, pipe
    end
  end

  def name
    "%s.%s" % [ entry.name, num ]
  end

private

  def jruby?
    defined?(RUBY_PLATFORM) and RUBY_PLATFORM == "java"
  end

  def fork_with_io(command, basedir)
    reader, writer = IO.pipe
    command = replace_command_env(command)
    pid = if jruby?
      Spoon.spawnp Foreman.runner, command
      Spoon.spawnp Foreman.runner, "-d", basedir, command
    else
      fork do
        trap("INT", "IGNORE")
        writer.sync = true
        $stdout.reopen writer
        $stderr.reopen writer
        reader.close
        exec Foreman.runner, "-d", basedir, command
      end
    end
    [ reader, pid ]
  end

  def run_process(basedir, command, pipe)
    io, @pid = fork_with_io(command, basedir)
    output pipe, "started with pid %d" % @pid
    Thread.new do
      until io.eof?
        output pipe, io.gets
      end
    end
  end

  def output(pipe, message)
    pipe.puts "%s,%s" % [ name, message ]
  end

  def replace_command_env(command)
    command.gsub(/\$(\w+)/) { |e| ENV[e[1..-1]] }
  end

  def with_environment(environment)
    old_env = ENV.each_pair.inject({}) { |h,(k,v)| h.update(k => v) }
    environment.each { |k,v| ENV[k] = v }
    ret = yield
    ENV.clear
    old_env.each { |k,v| ENV[k] = v}
    ret
{'fraction_non_alphanumeric': 0.12015389168393016, 'fraction_numerical': 0.0011837821840781297, 'mean_word_length': 2.5504201680672267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3705379', 'n_tokens_mistral': 1242, 'n_tokens_neox': 1207, 'n_words': 309}