<commit_before>require 'tmpdir'
require 'listen'
require 'fileutils'

if ENV['USE_LISTEN'] == '1'
  ('a'..'z').each do |i|
    begin
      puts "Watching tmp/#{i}... "
      Listen.to("tmp/#{i}", &proc{}).start
      sleep 1
    ensure
      Listen.stop
      puts "Stopped watching tmp/#{i}"
    end
  end
end

100_000.times do |i|
  File.open("tmp/#{i}.txt") { |f| puts "Opened tmp/#{i}.txt: #{f.read}" }
end<commit_msg>Add more work to increase the chance of triggering GC<commit_after>require 'tmpdir'
require 'listen'
require 'fileutils'

if ENV['USE_LISTEN'] == '1'
  ('a'..'z').each do |i|
    begin
      puts "Watching tmp/#{i}... "
      Listen.to("tmp/#{i}", &proc{}).start
      sleep 1
    ensure
      Listen.stop
      puts "Stopped watching tmp/#{i}"
    end
  end
end

str = []

1000.times do
  str << Array.new(10_000, "a").map { |i| "b" }.join * 10
  str.pop
end

5000.times do |i|
  str << Array.new(10_000, "a").map { |i| "b" }.join * 10
  str.pop
  File.open("tmp/#{i}.txt") { |f| puts "Opened tmp/#{i}.txt: #{f.read}" }
end
{'fraction_non_alphanumeric': 0.2045889101338432, 'fraction_numerical': 0.030592734225621414, 'mean_word_length': 2.752688172043011, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4896056', 'n_tokens_mistral': 438, 'n_tokens_neox': 418, 'n_words': 106}