<commit_before>
require 'pstore'

def reset_test_state
  FileUtils.mkdir_p 'tmp'
  FileUtils.rm_f 'tmp/state.pstore'
  FileUtils.touch 'tmp/state.pstore'
end

def incr_test_state(key)
  ps = PStore.new('tmp/state.pstore')
  ps.transaction do
    ps[key] ||= 0
    ps[key] += 1
  end
end

def test_state(key)
  ps = PStore.new('tmp/state.pstore')
  ps.transaction do
    return ps[key] || 0
  end
end

shared_context 'test server and worker' do
  before { reset_test_state }

  def wait_for_fork
    sleep 0.2
  end

  def wait_for_stop
    sleep 0.8
  end

  def wait_for_restart
    sleep 1.5<commit_msg>updated spec to implement concurrency control by itself because PStore didn't work with ruby 2.0<commit_after>
require 'thread'
require 'yaml'

def reset_test_state
  FileUtils.mkdir_p 'tmp'
  FileUtils.rm_f 'tmp/state.yml'
  FileUtils.touch 'tmp/state.yml'
  $state_file_mutex = Mutex.new
end

def incr_test_state(key)
  File.open('tmp/state.yml', 'r+') do |f|
    f.flock(File::LOCK_EX)

    $state_file_mutex.synchronize do
      data = YAML.load(f.read) || {} rescue {}
      data[key] ||= 0
      data[key] += 1

      f.pos = 0
      f.write YAML.dump(data)
    end
  end
end

def test_state(key)
  data = YAML.load_file('tmp/state.yml') || {} rescue {}
  return data[key] || 0
end

shared_context 'test server and worker' do
  before { reset_test_state }

  def wait_for_fork
    sleep 0.2
  end

  def wait_for_stop
    sleep 0.8
  end

  def wait_for_restart
    sleep 1.5
{'fraction_non_alphanumeric': 0.10767160161507403, 'fraction_numerical': 0.014131897711978465, 'mean_word_length': 3.165266106442577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '98172', 'n_tokens_mistral': 603, 'n_tokens_neox': 567, 'n_words': 140}