<commit_before># encoding: utf-8
# acts_as_xapian/lib/acts_as_xapian.rb:
# Xapian full text search in Ruby on Rails.
#
# Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
# Email: <EMAIL>; WWW: http://www.mysociety.org/
#
# Documentation
# =============
#
# See ../README.txt foocumentation. Please update that file if you edit
# code.

# Make it so if Xapian isn't installed, the Rails app doesn't fail completely,
# just when somebody does a search.
begin
    require 'xapian'
    $acts_as_xapian_bindings_available = true
rescue LoadError
    STDERR.puts "acts_as_xapian: No Ruby bindings for Xapian installed"
    $acts_as_xapian_bindings_available = false
end

module Xapian
    class QueryParser
        def unstem(term)
            words = []

            Xapian._safelyIterate(unstem_begin(term), unstem_end(term)) do |item|
                words << item.term
            end

            words
        end
    end
end

module ActsAsXapian
    ######################################################################
    # Module level variables
    # TODO: must be some kind of cattr_accessor that can do this better
    def ActsAsXapian.bindings_available
        $acts_as_xapian_bindings_available
    end
    class NoXapianRubyBindingsError < StandardError
    end

    @@db = nil
    @@db_path = nil
    @@writable_db = nil
    @@init_values = []

    # There used to be a problem with this module being loaded more than once.
    # Keep a check here, so we can tell if the problem recurs.
    if $acts_as_xapian_class_var_init
        raise "The acts_as_xapian module has already been loaded"
    else
        $acts_as_xapian_class_var_init = true
    end

    def ActsAsXapian.db
        @@db
    end
    def ActsAsXapian.db_path=(db_path)
        @@db_path = db_path
    end
    def ActsAsXapian.db_path
        @@db_path
    end
    def ActsAsXapian.writable_db
        @@writable_db
    end
    def ActsAsXapian.stemmer
        @@stemmer
    end
    def ActsAsXapian.term_generator
        @@term_generator
    end
    def ActsAsXapian.enquire
        @@enquire
    end
    def ActsAsXapian.query_parser
        @@query_parser
    end
    def ActsAsXapian.values_by_prefix
        @@values_by_prefix
    end
    def ActsAsXapian.config
      @@config
    end

    ######################################################################
    # Initialisation
    def ActsAsXapian.init(classname = nil, options = nil)
        if not classname.nil?
            # store class and options for use later, when we open the db in readable_init
            @@init_values.push([classname,options])
        end
    end

    # Reads the config file (if any) and sets up the path to the database we'll be using
    def ActsAsXapian.prepare_environment
      return unless @@db_path.nil?

      # barf if we can't figure out the environment
      environment = (ENV['RAILS_ENV'] or Rails.env)
      raise "Set RAILS_ENV, so acts_as_xapian can find the right Xapian database" if not environment

      # check for a config file
      config_file = Rails.root.join("config","xapian.yml")
      @@config = File.exists?(config_file) ? YAML.load_file(config_file)[environment] : {}

      # figure out where the DBs should go
      if config['base_db_path']
        db_parent_path = Rails.root.join(config['base_db_path'])
      else
        db_parent_path = File.join(File.dirname(__FILE__), 'xapiandbs')
      end

      # make the directory for the xapian databases to go in
      Dir.mkdir(db_parent_path) unless File.exists?(db_parent_path)

      @@db_path = File.join(db_parent_path, environment)

      # make some things that don't depend on the db
      # TODO: this gets made once for each acts_as_xapian. Oh well.
      @@stemmer = Xapian::Stem.new('english')
    end

    # Opens / reopens the db for reading
    # TODO: we perhaps don't need to rebuild database and enquire and queryparser -
    # but db.reopen wasn't enough by itself, so just do everything it's easier.
    def ActsAsXapian.readable_init
        raise NoXapianRubyBindingsError.new("Xapian Ruby bindings not installed") unless ActsAsXapian.bindings_available
        raise "acts_as_xapian hasn't been called in any models" if @@init_values.empty?

        prepare_environment

        # We need to reopen the database each time, so Xapian gets changes to it.
        # Calling reopen() does not always pick up changes for reasons that I can
        # only speculate about at the moment. (It is easy to reproduce this by
        # changing the code below to use reopen() rather than open() followed by
        # close(), and running rake spec.)
        if !@@db.nil?
            @@db.close
        end

        # basic Xapian objects
        begin
            @@db = Xapian::Database.new(@@db_path)
            @@enquire = Xapian::Enquire.new(@@db)
        rescue IOError => e
            raise "Failed to open Xapian database #{@@db_path}: #{e.message}"
        end

        init_query_parser
    end

    # Make a new query parser
    def ActsAsXapian.init_query_parser
        # for queries
        @@query_parser = Xapian::QueryParser.new
        @@query_parser.stemmer = @@stemmer
        @@query_parser.stemming_strategy = Xapian::QueryParser::STEM_SOME
        @@query_parser.database = @@db
        @@query_parser.default_op = Xapian::Query::OP_AND
        # The set_max_wildcard_expansion method was introduced in Xapian 1.2.7,
        # so may legitimately not be available.
        #
        # Large installations of Alaveteli should consider
        # upgrading, because uncontrolled wildcard expansion
        # can crash the whole server: see http://trac.xapian.org/ticket/350
        @@query_parser.set_max_wildcard_expansion(1000) if @@query_parser.respond_to? :set_max_wildcard_expansion

        @@stopper = Xapian::SimpleStopper.new
        @@stopper.add("and")
        @@stopper.add("of")
        @@stopper.add("&")
        @@query_parser.stopper = @@stopper

        @@terms_by_capital = {}
        @@values_by_number = {}
        @@values_by_prefix = {}
        @@value_ranges_store = []

        @@init_values.each do |classname, options|
            # go through the various field types, and tell query parser about them,
            # and error check them - i.e. check for consistency between models
            @@query_parser.add_boolean_prefix("model", "M")
            @@query_parser.add_boolean_prefix("modelid", "I")
            init_terms(options[:terms]) if options[:terms]
            init_values(options[:values]) if options[:values]
        end
    end

    def ActsAsXapian.init_values(values)
        for value in options[:values]
            raise "Value index '"+value[1].to_s+"' must be an integer, is " + value[1].class.to_s if value[1].class != 1.class
            raise "Already have value index '" + value[1].to_s + "' in another model but with different prefix '" + @@values_by_number[value[1]].to_s + "'" if @@values_by_number.include?(value[1]) && @@values_by_number[value[1]] != value[2]

             # date types are special, mark them so the first model they're seen for
            if !@@values_by_number.include?(value[1])
                if value[3] == :date
                    value_range = Xapian::DateValueRangeProcessor.new(value[1])
                elsif value[3] == :string
                    value_range = Xapian::StringValueRangeProcessor.new(value[1])
                elsif value[3] == :number
                    value_range = Xapian::NumberValueRangeProcessor.new(value[1])
                else
                    raise "Unknown value type '" + value[3].to_s + "'"
                end

                @@query_parser.add_valuerangeprocessor(value_range)

                # stop it being garbage collected, as
                # add_valuerangeprocessor ref is outside Ruby's GC
                @@value_ranges_store.push(value_range)
            end

            @@values_by_number[value[1]] = value[2]
            @@values_by_prefix[value[2]] = value[1]
        end
    end

    def ActsAsXapian.init_terms(terms)
        terms.each do |method, term_code, prefix|
            raise "Use a single capital letter for term code" if not term_code.match(/^[A-Z]$/)
            raise "M and I are reserved for use as the model/id term" if term_code == "M" || term_code == "I"
            raise "model and modelid are reserved for use as the model/id prefixes" if prefix == "model" || prefix == "modelid"
            raise "Z is reserved for stemming terms" if term_code == "Z"<commit_msg>Refactor for idiomatic ruby.<commit_after># encoding: utf-8
# acts_as_xapian/lib/acts_as_xapian.rb:
# Xapian full text search in Ruby on Rails.
#
# Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
# Email: <EMAIL>; WWW: http://www.mysociety.org/
#
# Documentation
# =============
#
# See ../README.txt foocumentation. Please update that file if you edit
# code.

# Make it so if Xapian isn't installed, the Rails app doesn't fail completely,
# just when somebody does a search.
begin
    require 'xapian'
    $acts_as_xapian_bindings_available = true
rescue LoadError
    STDERR.puts "acts_as_xapian: No Ruby bindings for Xapian installed"
    $acts_as_xapian_bindings_available = false
end

module Xapian
    class QueryParser
        def unstem(term)
            words = []

            Xapian._safelyIterate(unstem_begin(term), unstem_end(term)) do |item|
                words << item.term
            end

            words
        end
    end
end

module ActsAsXapian
    ######################################################################
    # Module level variables
    # TODO: must be some kind of cattr_accessor that can do this better
    def ActsAsXapian.bindings_available
        $acts_as_xapian_bindings_available
    end
    class NoXapianRubyBindingsError < StandardError
    end

    @@db = nil
    @@db_path = nil
    @@writable_db = nil
    @@init_values = []

    # There used to be a problem with this module being loaded more than once.
    # Keep a check here, so we can tell if the problem recurs.
    if $acts_as_xapian_class_var_init
        raise "The acts_as_xapian module has already been loaded"
    else
        $acts_as_xapian_class_var_init = true
    end

    def ActsAsXapian.db
        @@db
    end
    def ActsAsXapian.db_path=(db_path)
        @@db_path = db_path
    end
    def ActsAsXapian.db_path
        @@db_path
    end
    def ActsAsXapian.writable_db
        @@writable_db
    end
    def ActsAsXapian.stemmer
        @@stemmer
    end
    def ActsAsXapian.term_generator
        @@term_generator
    end
    def ActsAsXapian.enquire
        @@enquire
    end
    def ActsAsXapian.query_parser
        @@query_parser
    end
    def ActsAsXapian.values_by_prefix
        @@values_by_prefix
    end
    def ActsAsXapian.config
      @@config
    end

    ######################################################################
    # Initialisation
    def ActsAsXapian.init(classname = nil, options = nil)
        if not classname.nil?
            # store class and options for use later, when we open the db in readable_init
            @@init_values.push([classname,options])
        end
    end

    # Reads the config file (if any) and sets up the path to the database we'll be using
    def ActsAsXapian.prepare_environment
      return unless @@db_path.nil?

      # barf if we can't figure out the environment
      environment = (ENV['RAILS_ENV'] or Rails.env)
      raise "Set RAILS_ENV, so acts_as_xapian can find the right Xapian database" if not environment

      # check for a config file
      config_file = Rails.root.join("config","xapian.yml")
      @@config = File.exists?(config_file) ? YAML.load_file(config_file)[environment] : {}

      # figure out where the DBs should go
      if config['base_db_path']
        db_parent_path = Rails.root.join(config['base_db_path'])
      else
        db_parent_path = File.join(File.dirname(__FILE__), 'xapiandbs')
      end

      # make the directory for the xapian databases to go in
      Dir.mkdir(db_parent_path) unless File.exists?(db_parent_path)

      @@db_path = File.join(db_parent_path, environment)

      # make some things that don't depend on the db
      # TODO: this gets made once for each acts_as_xapian. Oh well.
      @@stemmer = Xapian::Stem.new('english')
    end

    # Opens / reopens the db for reading
    # TODO: we perhaps don't need to rebuild database and enquire and queryparser -
    # but db.reopen wasn't enough by itself, so just do everything it's easier.
    def ActsAsXapian.readable_init
        raise NoXapianRubyBindingsError.new("Xapian Ruby bindings not installed") unless ActsAsXapian.bindings_available
        raise "acts_as_xapian hasn't been called in any models" if @@init_values.empty?

        prepare_environment

        # We need to reopen the database each time, so Xapian gets changes to it.
        # Calling reopen() does not always pick up changes for reasons that I can
        # only speculate about at the moment. (It is easy to reproduce this by
        # changing the code below to use reopen() rather than open() followed by
        # close(), and running rake spec.)
        if !@@db.nil?
            @@db.close
        end

        # basic Xapian objects
        begin
            @@db = Xapian::Database.new(@@db_path)
            @@enquire = Xapian::Enquire.new(@@db)
        rescue IOError => e
            raise "Failed to open Xapian database #{@@db_path}: #{e.message}"
        end

        init_query_parser
    end

    # Make a new query parser
    def ActsAsXapian.init_query_parser
        # for queries
        @@query_parser = Xapian::QueryParser.new
        @@query_parser.stemmer = @@stemmer
        @@query_parser.stemming_strategy = Xapian::QueryParser::STEM_SOME
        @@query_parser.database = @@db
        @@query_parser.default_op = Xapian::Query::OP_AND
        # The set_max_wildcard_expansion method was introduced in Xapian 1.2.7,
        # so may legitimately not be available.
        #
        # Large installations of Alaveteli should consider
        # upgrading, because uncontrolled wildcard expansion
        # can crash the whole server: see http://trac.xapian.org/ticket/350
        @@query_parser.set_max_wildcard_expansion(1000) if @@query_parser.respond_to? :set_max_wildcard_expansion

        @@stopper = Xapian::SimpleStopper.new
        @@stopper.add("and")
        @@stopper.add("of")
        @@stopper.add("&")
        @@query_parser.stopper = @@stopper

        @@terms_by_capital = {}
        @@values_by_number = {}
        @@values_by_prefix = {}
        @@value_ranges_store = []

        @@init_values.each do |classname, options|
            # go through the various field types, and tell query parser about them,
            # and error check them - i.e. check for consistency between models
            @@query_parser.add_boolean_prefix("model", "M")
            @@query_parser.add_boolean_prefix("modelid", "I")
            init_terms(options[:terms]) if options[:terms]
            init_values(options[:values]) if options[:values]
        end
    end

    def ActsAsXapian.init_values(values)
        values.each do |method, index, prefix, value_type|
            raise "Value index '#{index}' must be an Integer, is #{index.class}" unless index.is_a? Integer
            if @@values_by_number.include?(index) && @@values_by_number[index] != prefix
                raise "Already have value index '#{index}' in another model " \
                      "but with different prefix '#{@@values_by_number[index]}'"
            end
            # date types are special, mark them so the first model they're seen for
            unless @@values_by_number.include?(index)
                case value_type
                when :date
                    value_range = Xapian::DateValueRangeProcessor.new(index)
                when :string
                    value_range = Xapian::StringValueRangeProcessor.new(index)
                when :number
                    value_range = Xapian::NumberValueRangeProcessor.new(index)
                else
                    raise "Unknown value type '#{value_type}'"
                end

                @@query_parser.add_valuerangeprocessor(value_range)

                # stop it being garbage collected, as
                # add_valuerangeprocessor ref is outside Ruby's GC
                @@value_ranges_store.push(value_range)
            end

            @@values_by_number[index] = prefix
            @@values_by_prefix[prefix] = index
        end
    end

    def ActsAsXapian.init_terms(terms)
        terms.each do |method, term_code, prefix|
            raise "Use a single capital letter for term code" if not term_code.match(/^[A-Z]$/)
            raise "M and I are reserved for use as the model/id term" if term_code == "M" || term_code == "I"
            raise "model and modelid are reserved for use as the model/id prefixes" if prefix == "model" || prefix == "modelid"
            raise "Z is reserved for stemming terms" if term_code == "Z"
{'fraction_non_alphanumeric': 0.10193578571846307, 'fraction_numerical': 0.00298263056319083, 'mean_word_length': 2.501945525291829, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4652186', 'n_tokens_mistral': 5323, 'n_tokens_neox': 4938, 'n_words': 1484}