<commit_before>#!/usr/bin/env ruby

require 'json'

#
# Lista las imagenes de cierto path que no estan siendo referenciadas por Caliboro
#

dir = ARGV.shift
if dir.nil?
  puts "usage: #{$0} <path>"
  exit 1
end


def used_files(path)
  json = JSON.load(IO.read(File.join(path, 'bones.json')))

  json['bones'].map { |x| x['images'] }.map do |img|
    img.map { |a| a['imagePath'] }
  end.flatten
end


Dir.glob(File.join(dir, '*')).each do |path|
  next unless File.directory?(path)

  puts "-"*60
  puts path
  used = used_files(path)
  Dir.glob(File.join(path, '*')).each do |subpath|
    name = File.basename(subpath)
    next unless name =~ /\.jpg$/i

    puts subpath unless used.include?(name)
  end


  used.each do |img_path|
    raise "#{img_path} not found" unless File.exist?(File.join(path, img_path))
  end

end
<commit_msg>opcion para eliminar las imagenes no usadas<commit_after>#!/usr/bin/env ruby

require 'json'
require 'fileutils'

#
# Lista las imagenes de cierto path que no estan siendo referenciadas por Caliboro
#

dir = ARGV.shift
remove = (ARGV.shift == "-r")
if dir.nil?
  puts "usage: #{$0} <path> [-r]"
  exit 1
end


def used_files(path)
  json = JSON.load(IO.read(File.join(path, 'bones.json')))

  json['bones'].map { |x| x['images'] }.map do |img|
    img.map { |a| a['imagePath'] }
  end.flatten
end


Dir.glob(File.join(dir, '*')).each do |path|
  next unless File.directory?(path)

  puts "-"*60
  puts path
  used = used_files(path)
  Dir.glob(File.join(path, '*')).each do |subpath|
    name = File.basename(subpath)
    next unless name =~ /\.jpg$/i

    unless used.include?(name)
      puts subpath
      FileUtils.rm(subpath)
    end
  end


  used.each do |img_path|
    raise "#{img_path} not found" unless File.exist?(File.join(path, img_path))
  end

end

{'fraction_non_alphanumeric': 0.16046640755136035, 'fraction_numerical': 0.004441976679622432, 'mean_word_length': 3.449382716049383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6559824', 'n_tokens_mistral': 715, 'n_tokens_neox': 696, 'n_words': 182}