<commit_before>#!/usr/bin/env ruby
require 'faker'
require 'fileutils'
require_relative '../queue-helper'

# TODO: check existing worker ids. it'd be disastrous to have two sharing a workspace
@worker_id = ARGV[0] || "#{Faker::Name.first_name}"

def from_workspace(named:)
  path = "#{__dir__}/workspaces/#{named}-#{@worker_id}"
  FileUtils.mkpath path
  Dir.chdir(path) { yield if block_given? }
end

def execute(command)
  puts "Executing '#{command}'..."
  # TODO: path.split(' ') to bypass the shell when we're not using env vars
  IO.popen(command) do |io|
    while line = io.gets # do
      log.info line
    end
  end
  puts "Execution complete with exit status: #{$CHILD_STATUS.exitstatus}"
end

QueueHelper.wait_for('jobs') do |data|<commit_msg> needs 'English' and 0 is probably just as good<commit_after>#!/usr/bin/env ruby
require 'faker'
require 'fileutils'
require_relative '../queue-helper'

# TODO: check existing worker ids. it'd be disastrous to have two sharing a workspace
@worker_id = ARGV[0] || "#{Faker::Name.first_name}"

def from_workspace(named:)
  path = "#{__dir__}/workspaces/#{named}-#{@worker_id}"
  FileUtils.mkpath path
  Dir.chdir(path) { yield if block_given? }
end

def execute(command)
  puts "Executing '#{command}'..."
  # TODO: path.split(' ') to bypass the shell when we're not using env vars
  IO.popen(command) do |io|
    while line = io.gets # do
      log.info line
    end
  end
  puts "Execution complete with exit status: #{$?.exitstatus}"
end

QueueHelper.wait_for('jobs') do |data|
{'fraction_non_alphanumeric': 0.14024787997390736, 'fraction_numerical': 0.0019569471624266144, 'mean_word_length': 4.2534246575342465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6672447', 'n_tokens_mistral': 532, 'n_tokens_neox': 511, 'n_words': 158}