<commit_before>require 'erb'
require 'singleton'

module ThinkingSphinx
  # This class both keeps track of the configuration settings for Sphinx and
  # also generates the resulting file for Sphinx to use.
  # 
  # Here are the default settings, relative to RAILS_ROOT where relevant:
  #
  # config file::           config/#{environment}.sphinx.conf
  # searchd log file::      log/searchd.log
  # query log file::        log/searchd.query.log
  # pid file::              log/searchd.#{environment}.pid
  # searchd files::         db/sphinx/#{environment}/
  # address::               127.0.0.1
  # port::                  3312
  # allow star::            false
  # min prefix length::     1
  # min infix length::      1
  # mem limit::             64M
  # max matches::           1000
  # morphology::            nil
  # charset type::          utf-8
  # charset table::         nil
  # ignore chars::          nil
  # html strip::            false
  # html remove elements::  ''
  #
  # If you want to change these settings, create a YAML file at
  # config/sphinx.yml with settings for each environment, in a similar
  # fashion to database.yml - using the following keys: config_file,
  # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
  # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
  # max_matches, morphology, charset_type, charset_table, ignore_chars,
  # html_strip, html_remove_elements, delayed_job_priority.
  #
  # I think you've got the idea.
  # 
  # Each setting in the YAML file is optional - so only put in the ones you
  # want to change.
  #
  # Keep in mind, if for some particular reason you're using a version of
  # Sphinx older than 0.9.8 r871 (that's prior to the proper 0.9.8 release),
  # don't set allow_star to true.
  # 
  class Configuration
    include Singleton
    
    SourceOptions = %w( mysql_connect_flags sql_range_step sql_query_pre
      sql_query_post sql_ranged_throttle sql_query_post_index )
    
    IndexOptions  = %w( charset_table charset_type docinfo enable_star
      exceptions html_index_attrs html_remove_elements html_strip ignore_chars
      min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
      ngram_len phrase_boundary phrase_boundary_step preopen stopwords
      wordforms )
        
    attr_accessor :config_file, :searchd_log_file, :query_log_file,
      :pid_file, :searchd_file_path, :address, :port, :allow_star,
      :database_yml_file, :app_root, :bin_path, :model_directories,
      :delayed_job_priority
    
    attr_accessor :source_options, :index_options
    
    attr_reader :environment, :configuration
    
    # Load in the configuration settings - this will look for config/sphinx.yml
    # and parse it according to the current environment.
    # 
    def initialize(app_root = Dir.pwd)
      self.reset
    end
    
    def self.configure(&block)
      yield instance
      instance.reset(instance.app_root)
    end
    
    def reset(custom_app_root=nil)
      if custom_app_root
        self.app_root = custom_app_root
      else
        self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
        self.app_root          = Merb.root  if defined?(Merb)
        self.app_root        ||= app_root
      end
      
      @configuration = Riddle::Configuration.new
      @configuration.searchd.address    = "127.0.0.1"
      @configuration.searchd.port       = 3312
      @configuration.searchd.pid_file   = "#{self.app_root}/log/searchd.#{environment}.pid"
      @configuration.searchd.log        = "#{self.app_root}/log/searchd.log"
      @configuration.searchd.query_log  = "#{self.app_root}/log/searchd.query.log"
      
      self.database_yml_file    = "#{self.app_root}/config/database.yml"
      self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
      self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
      self.allow_star           = false
      self.bin_path             = ""
      self.model_directories    = ["#{app_root}/app/models/"] +
        Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
      self.delayed_job_priority = 0
      
      self.source_options  = {}
      self.index_options   = {
        :charset_type => "utf-8"
      }
            
      parse_config
      
      self
    end
    
    def self.environment
      @@environment ||= (
        defined?(Merb) ? Merb.environment : ENV['RAILS_ENV']
      ) || "development"
    end
    
    def environment
      self.class.environment
    end
    
    def controller
      @controller ||= Riddle::Controller.new(@configuration, self.config_file)
    end
    
    # Generate the config file for Sphinx by using all the settings defined and
    # looping through all the models with indexes to build the relevant
    # indexer and searchd configuration, and sources and indexes details.
    #
    def build(file_path=nil)
      load_models
      file_path ||= "#{self.config_file}"
      
      @configuration.indexes.clear
      
      ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
        @configuration.indexes.concat model.constantize.to_riddle(model_index)
      end
      
      open(file_path, "w") do |file|
        file.write @configuration.render
      end
    end
    
    # Make sure all models are loaded - without reloading any that
    # ActiveRecord::Base is already aware of (otherwise we start to hit some
    # messy dependencies issues).
    # 
    def load_models
      self.model_directories.each do |base|
        Dir["#{base}**/*.rb"].each do |file|
          model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
        
          next if model_name.nil?
          next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
            model.name == model_name
          }
        <commit_msg>Making sure models aren't requested twice when cache_classes is true.<commit_after>require 'erb'
require 'singleton'

module ThinkingSphinx
  # This class both keeps track of the configuration settings for Sphinx and
  # also generates the resulting file for Sphinx to use.
  # 
  # Here are the default settings, relative to RAILS_ROOT where relevant:
  #
  # config file::           config/#{environment}.sphinx.conf
  # searchd log file::      log/searchd.log
  # query log file::        log/searchd.query.log
  # pid file::              log/searchd.#{environment}.pid
  # searchd files::         db/sphinx/#{environment}/
  # address::               127.0.0.1
  # port::                  3312
  # allow star::            false
  # min prefix length::     1
  # min infix length::      1
  # mem limit::             64M
  # max matches::           1000
  # morphology::            nil
  # charset type::          utf-8
  # charset table::         nil
  # ignore chars::          nil
  # html strip::            false
  # html remove elements::  ''
  #
  # If you want to change these settings, create a YAML file at
  # config/sphinx.yml with settings for each environment, in a similar
  # fashion to database.yml - using the following keys: config_file,
  # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
  # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
  # max_matches, morphology, charset_type, charset_table, ignore_chars,
  # html_strip, html_remove_elements, delayed_job_priority.
  #
  # I think you've got the idea.
  # 
  # Each setting in the YAML file is optional - so only put in the ones you
  # want to change.
  #
  # Keep in mind, if for some particular reason you're using a version of
  # Sphinx older than 0.9.8 r871 (that's prior to the proper 0.9.8 release),
  # don't set allow_star to true.
  # 
  class Configuration
    include Singleton
    
    SourceOptions = %w( mysql_connect_flags sql_range_step sql_query_pre
      sql_query_post sql_ranged_throttle sql_query_post_index )
    
    IndexOptions  = %w( charset_table charset_type docinfo enable_star
      exceptions html_index_attrs html_remove_elements html_strip ignore_chars
      min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
      ngram_len phrase_boundary phrase_boundary_step preopen stopwords
      wordforms )
        
    attr_accessor :config_file, :searchd_log_file, :query_log_file,
      :pid_file, :searchd_file_path, :address, :port, :allow_star,
      :database_yml_file, :app_root, :bin_path, :model_directories,
      :delayed_job_priority
    
    attr_accessor :source_options, :index_options
    
    attr_reader :environment, :configuration
    
    # Load in the configuration settings - this will look for config/sphinx.yml
    # and parse it according to the current environment.
    # 
    def initialize(app_root = Dir.pwd)
      self.reset
    end
    
    def self.configure(&block)
      yield instance
      instance.reset(instance.app_root)
    end
    
    def reset(custom_app_root=nil)
      if custom_app_root
        self.app_root = custom_app_root
      else
        self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
        self.app_root          = Merb.root  if defined?(Merb)
        self.app_root        ||= app_root
      end
      
      @configuration = Riddle::Configuration.new
      @configuration.searchd.address    = "127.0.0.1"
      @configuration.searchd.port       = 3312
      @configuration.searchd.pid_file   = "#{self.app_root}/log/searchd.#{environment}.pid"
      @configuration.searchd.log        = "#{self.app_root}/log/searchd.log"
      @configuration.searchd.query_log  = "#{self.app_root}/log/searchd.query.log"
      
      self.database_yml_file    = "#{self.app_root}/config/database.yml"
      self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
      self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
      self.allow_star           = false
      self.bin_path             = ""
      self.model_directories    = ["#{app_root}/app/models/"] +
        Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
      self.delayed_job_priority = 0
      
      self.source_options  = {}
      self.index_options   = {
        :charset_type => "utf-8"
      }
            
      parse_config
      
      self
    end
    
    def self.environment
      @@environment ||= (
        defined?(Merb) ? Merb.environment : ENV['RAILS_ENV']
      ) || "development"
    end
    
    def environment
      self.class.environment
    end
    
    def controller
      @controller ||= Riddle::Controller.new(@configuration, self.config_file)
    end
    
    # Generate the config file for Sphinx by using all the settings defined and
    # looping through all the models with indexes to build the relevant
    # indexer and searchd configuration, and sources and indexes details.
    #
    def build(file_path=nil)
      load_models
      file_path ||= "#{self.config_file}"
      
      @configuration.indexes.clear
      
      ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
        @configuration.indexes.concat model.constantize.to_riddle(model_index)
      end
      
      open(file_path, "w") do |file|
        file.write @configuration.render
      end
    end
    
    # Make sure all models are loaded - without reloading any that
    # ActiveRecord::Base is already aware of (otherwise we start to hit some
    # messy dependencies issues).
    # 
    def load_models
      return if defined?(Rails) && Rails.configuration.cache_classes
      
      self.model_directories.each do |base|
        Dir["#{base}**/*.rb"].each do |file|
          model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
        
          next if model_name.nil?
          next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
            model.name == model_name
          }
        
{'fraction_non_alphanumeric': 0.09226995940460081, 'fraction_numerical': 0.006935047361299053, 'mean_word_length': 2.4345047923322682, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1670807', 'n_tokens_mistral': 3683, 'n_tokens_neox': 3584, 'n_words': 901}