<commit_before>class Helper

  @@spec_dir = File.absolute_path(File.dirname(__FILE__))+"/"
  @@pwrake = @@spec_dir+'../bin/pwrake'

  @@show_command = false
  @@show_result = false

  def initialize(dir=nil,args=nil)
    @dir = @@spec_dir+(dir||"")
    @args = args
  end

  attr_reader :n_files, :filelist, :result, :status
  attr_reader :elapsed_time

  def clean
    Dir.chdir(@dir) do
      `rake -q clean`
    end
    self
  end

  def run
    if @@show_command
      puts "@dir=#{@dir}"
      puts "@args=#{@args}"
    end
    Dir.chdir(@dir) do
      #system "rm -f *.dat"
      #`rake clean`
      tm = Time.now
      @result = `#{@@pwrake} -q #{@args}`
      @status = $?
      @elapsed_time = Time.now - tm
      system "touch dummy; rm dummy"
      @filelist = Dir.glob("*")
      @n_files = @filelist.size
    end
    if @@show_result
      puts @result
      puts "@status=#{@status.inspect}"
    end
    self
  end

  def success?
    @status && @status.success?
  end

  def output_lines
    @result.split("\n")
  end
end

def read_hosts(file)
  cores = []
  open(file) do |f|
    while l = f.gets
      l = $1 if /^([^#]*)#/ =~ l
      host, ncore, group = l.split
      if host
        ncore = (ncore || 1).to_i
        cores.concat( [host] * ncore )
      end
    end
  end
  cores
end
<commit_msg>add option to show messages<commit_after>class Helper

  @@spec_dir = File.absolute_path(File.dirname(__FILE__))+"/"
  @@pwrake = @@spec_dir+'../bin/pwrake -q --pwrake-conf=../pwrake_conf.yaml'

  @@show_command = false
  @@show_result = false

  def self.show=(f)
    if f
      @@show_command = true
      @@show_result = true
    else
      @@show_command = false
      @@show_result = false
    end
  end

  def initialize(dir=nil,args=nil)
    @dir = @@spec_dir+(dir||"")
    @args = args
  end

  attr_reader :n_files, :filelist, :result, :status
  attr_reader :elapsed_time

  def clean
    Dir.chdir(@dir) do
      `rake -q clean`
    end
    self
  end

  def run
    cmd = "#{@@pwrake} #{@args}"
    if @@show_command
      puts
      puts "-- dir: #{@dir}"
      puts "-- cmd: #{cmd}"
    end
    Dir.chdir(@dir) do
      #system "rm -f *.dat"
      #`rake clean`
      tm = Time.now
      @result = `#{cmd}`
      @status = $?
      @elapsed_time = Time.now - tm
      system "touch dummy; rm dummy"
      @filelist = Dir.glob("*")
      @n_files = @filelist.size
    end
    if @@show_result
      puts @result
      puts "-- status: #{@status.inspect}\n"
    end
    self
  end

  def success?
    @status && @status.success?
  end

  def output_lines
    @result.split("\n")
  end
end

def read_hosts(file)
  cores = []
  open(file) do |f|
    while l = f.gets
      l = $1 if /^([^#]*)#/ =~ l
      host, ncore, group = l.split
      if host
        ncore = (ncore || 1).to_i
        cores.concat( [host] * ncore )
      end
    end
  end
  cores
end

{'fraction_non_alphanumeric': 0.15365938258758238, 'fraction_numerical': 0.001387443635102324, 'mean_word_length': 2.124593716143012, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704312', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1070, 'n_words': 266}