<commit_before>#!/usr/bin/env ruby

# This wrapper script runs a command and lets standard out and error flow
# through. However, it does limit the number of lines of output. This is
# used by morph as a wrapper around running scrapers to ensure that they
# can't fill up the docker container log file (and hence the server disk).

require 'optparse'
require 'open3'

command = nil
exit_status = nil

OptionParser.new do |opts|
  opts.banner = 'Usage: ./limit_output.rb [command to run]'

  command = ARGV[0]
  if command.nil?
    STDERR.puts 'Please give me a command to run'
    puts opts
    exit
  end
end.parse!

Open3.popen3(command) do |_stdin, stdout, stderr, wait_thr|
  streams = [stdout, stderr]
  until streams.empty?
    IO.select(streams).flatten.compact.each do |io|
      if io.eof?
        streams.delete io
        next
      end

      # Just send this stuff straight through
      STDOUT << io.readpartial(1024) if io.fileno == stdout.fileno
      STDERR << io.readpartial(1024) if io.fileno == stderr.fileno
    end
  end

  exit_status = wait_thr.value.exitstatus
end

exit(exit_status)<commit_msg>Disable output buffering on limit_output.rb<commit_after>#!/usr/bin/env ruby

# This wrapper script runs a command and lets standard out and error flow
# through. However, it does limit the number of lines of output. This is
# used by morph as a wrapper around running scrapers to ensure that they
# can't fill up the docker container log file (and hence the server disk).

require 'optparse'
require 'open3'

command = nil
exit_status = nil

OptionParser.new do |opts|
  opts.banner = 'Usage: ./limit_output.rb [command to run]'

  command = ARGV[0]
  if command.nil?
    STDERR.puts 'Please give me a command to run'
    puts opts
    exit
  end
end.parse!

# Disable output buffering
STDOUT.sync = true
STDERR.sync = true

Open3.popen3(command) do |_stdin, stdout, stderr, wait_thr|
  streams = [stdout, stderr]
  until streams.empty?
    IO.select(streams).flatten.compact.each do |io|
      if io.eof?
        streams.delete io
        next
      end

      # Just send this stuff straight through
      STDOUT << io.readpartial(1024) if io.fileno == stdout.fileno
      STDERR << io.readpartial(1024) if io.fileno == stderr.fileno
    end
  end

  exit_status = wait_thr.value.exitstatus
end

exit(exit_status)
{'fraction_non_alphanumeric': 0.08561643835616438, 'fraction_numerical': 0.010273972602739725, 'mean_word_length': 3.2568306010928962, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4118789', 'n_tokens_mistral': 784, 'n_tokens_neox': 748, 'n_words': 270}