<commit_before>require 'fileutils'

namespace :dict do
  desc "Install hunspell lib"
  task :install do
    hunspell = File.join(File.dirname(__FILE__), "../hunspell")
    Dir.chdir( hunspell )
    bundle_file = File.join(hunspell, "hunspell.o")
    File.delete( bundle_file ) if File.exist?( bundle_file )
    system "ruby extconf.rb && make"
  end
  
  desc "Generate dictionaries for all models"
  task :generate => :environment do
    # Borrow from ThinkingSphinx's configure.rb
    base = "#{Rails.root}/app/models/"
    Dir["#{base}**/*.rb"].each do |file|
      model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')      
      # Hack to skip all xxx_related.rb files
      next if /_related/i =~ model_name

      klass = begin
        model_name.classify.constantize
      rescue LoadError<commit_msg>remove old thinking-sphinx plugin<commit_after>require 'fileutils'

namespace :dict do
  desc "Install hunspell lib"
  task :install do
    hunspell = File.join(File.dirname(__FILE__), "../hunspell")
    Dir.chdir( hunspell )
    bundle_file = File.join(hunspell, "hunspell.o")
    File.delete( bundle_file ) if File.exist?( bundle_file )
    system "ruby extconf.rb && make"
  end
  
  desc "Generate dictionaries for all models"
  task :generate => :environment do
    # Borrow from ThinkingSphinx's configure.rb
    base = "#{Rails.root}/app/models/"
    Dir["#{base}**/*.rb"].each do |file|
      default_model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1').classify      
      model_name = default_model_name.split('::').last

      klass = begin
        model_name.constantize
      rescue LoadError
{'fraction_non_alphanumeric': 0.14294516327788045, 'fraction_numerical': 0.0012322858903265558, 'mean_word_length': 3.0701754385964914, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7571223', 'n_tokens_mistral': 541, 'n_tokens_neox': 552, 'n_words': 128}