<commit_before>require 'json'
require 'active_support'
require 'active_support/number_helper'

EXT = '.cache.json'
$total_paths = 0
$trimmed_paths = 0

def main
  Dir.glob("*" + EXT).each do |cache_file|
    trim_cache(cache_file)
  end
end

def trim_cache(path)
  puts "trimming #{path}"
  json = JSON.parse(File.read(path))
  $total_paths += before_size = json.size
  json.select! do |path, _data|
    still_exists?(path)
  end
  $trimmed_paths += before_size - json.size
  File.write(path, JSON.generate(json))
end

def still_exists?(path)
  File.exist?(path) && !File.directory?(path)
end

start = Time.now
main
duration = Time.now - start
avg_duration = duration / $total_paths * 1000
puts "\nTrimmed #{ActiveSupport::NumberHelper.number_to_delimited $trimmed_paths} paths"
puts "took #{duration} seconds for #{ActiveSupport::NumberHelper.number_to_delimited $total_paths} paths (avg #{avg_duration.round(3)} ms per file)"<commit_msg>trim cache on overmind as well<commit_after>require 'json'
require 'active_support'
require 'active_support/number_helper'

if ARGV.empty?
  require 'dotenv/load'
  require 'net/ssh'
  require 'net/sftp'
end

SSH_OPTIONS = {compression: true, config: true, timeout: 1, max_pkt_size: 0x10000}
EXT = '.cache.json'
$total_paths = 0
$trimmed_paths = 0

def main
  Dir.glob("*" + EXT).each do |cache_file|
    trim_cache(cache_file)
  end
end

def trim_cache(path)
  puts "trimming #{path}"
  json = JSON.parse(File.read(path))
  $total_paths += before_size = json.size
  json.select! do |path, _data|
    still_exists?(path)
  end
  $trimmed_paths += before_size - json.size
  File.write(path, JSON.generate(json))
end

def still_exists?(path)
  File.exist?(path) && !File.directory?(path)
end

def remote_trim
  begin
    Net::SSH.start(ENV['OVERMIND_SSH_HOST'], nil, SSH_OPTIONS) do |ssh|
      ssh.sftp.upload!(__FILE__, 'remote.rb')
      puts "\n*************** running on remote ***************\n\n"
      puts ssh.exec! "source load_rbenv && ruby remote.rb remote"
      ssh.exec! "rm remote.rb"
    end
  rescue Errno::EAGAIN => e
    puts 'could not connect to overmind'
  end
end


start = Time.now
main
duration = Time.now - start
avg_duration = duration / $total_paths * 1000
puts "\nTrimmed #{ActiveSupport::NumberHelper.number_to_delimited $trimmed_paths} paths"
puts "took #{duration} seconds for #{ActiveSupport::NumberHelper.number_to_delimited $total_paths} paths (avg #{avg_duration.round(3)} ms per file)"

if ARGV.empty?
  remote_trim
end
{'fraction_non_alphanumeric': 0.13793103448275862, 'fraction_numerical': 0.008420208500400962, 'mean_word_length': 4.762124711316397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4510253', 'n_tokens_mistral': 907, 'n_tokens_neox': 885, 'n_words': 216}