<commit_before>module Searchable
  extend ActiveSupport::Concern

  included do
    include Elasticsearch::Model

    settings index: { number_of_shards: 1, number_of_replicas: 0 } do
      mapping do
        indexes :name, :analyzer => 'snowball', :boost => 10
        indexes :description, :analyzer => 'snowball'
        indexes :homepage
        indexes :repository_url
        indexes :repo_name
        indexes :latest_release_number, :analyzer => 'keyword'
        indexes :keywords, :analyzer => 'keyword'
        indexes :language, :analyzer => 'keyword'
        indexes :normalized_licenses, :analyzer => 'keyword'
        indexes :platform, :analyzer => 'keyword'

        indexes :created_at, type: 'date'
        indexes :updated_at, type: 'date'
        indexes :latest_release_published_at, type: 'date'

        indexes :rank, type: 'integer'
        indexes :stars, type: 'integer'
        indexes :versions_count, type: 'integer'
        indexes :github_repository_id, type: 'integer'
      end
    end

    after_commit on: [:create] do
      __elasticsearch__.index_document
    end

    after_commit on: [:update] do
      __elasticsearch__.update_document
    end

    after_commit on: [:destroy] do
      __elasticsearch__.delete_document
    end

    def as_indexed_json(options = {})
      as_json methods: [:stars, :language, :repo_name]
    end

    def self.total
      __elasticsearch__.client.count(index: 'projects')["count"]
    end

    def self.search(query, options = {})
      facet_limit = options.fetch(:facet_limit, 30)
      query = sanitize_query(query)
      options[:filters] ||= []
      search_definition = {
        query: {
          function_score: {
            query: {
              filtered: {
                 query: {query_string: {query: query}},
                 filter:{ bool: { must: [] } }
              }
            },
            field_value_factor: {
              field: "rank",
              "modifier": "log1p"
            }
          }
        },
        facets: {
          platforms: { terms: {
            field: "platform",
            size: facet_limit},
            facet_filter: {
              bool: {
                must: filter_format(options[:filters], :platform)
              }
            }
          },
          languages: { terms: {
              field: "language",
              size: facet_limit
            },
            facet_filter: {
              bool: {
                must: filter_format(options[:filters], :language)
              }
            }
          },
          licenses: {
            terms: {
              field: "normalized_licenses",
              size: facet_limit
            },
            facet_filter: {
              bool: {
                must: filter_format(options[:filters], :normalized_licenses)
              }
            }
          }#,
          # keywords: {
          #   terms: {
          #     field: "keywords",
          #     size: facet_limit
          #   },
          #   facet_filter: {
          #     bool: {
          #       must: filter_format(options[:filters], :keywords)
          #     }
          #   }
          # }
        },
        filter: {
          bool: {
            must: []
          }
        },
        suggest: {
          did_you_mean: {
            text: query,
            term: {
              size: 3,
              field: "name"
            }
          }
        }
      }
      search_definition[:sort]  = { (options[:sort] || '_score') => (options[:order] || 'desc') }
      search_definition[:track_scores] = true
      search_definition[:filter][:bool][:must] = filter_format(options[:filters])

      __elasticsearch__.search(search_definition)
    end

    def self.filter_format(filters, except = nil)
      filters.select { |k, v| v.present? && k != except }.map do |k, v|
        {
          term: { k => v }
        }
      end
    end

    def self.sanitize_query(str)
      return '*' if str.blank?
      # Escape special characters
      # http://lucene.apache.org/core/old_versioned_docs/versions/2_9_1/queryparsersyntax.html#Escaping Special Characters
      escaped_characters = Regexp.escape('\\+-&|/!(){}[]^~?:')
      str = str.gsub(/([#{escaped_characters}])/, '\\\\\1')

      # AND, OR and NOT are used by lucene as logical operators. We need
      # to escape them
      ['AND', 'OR', 'NOT'].each do |word|
        escaped_word = word.split('').map {|char| "\\#{char}" }.join('')
        str = str.gsub(/\s*\b(#{word.upcase})\b\s*/, " #{escaped_word} ")
      end

      # Escape odd quotes
      quote_count = str.count '"'
      str = str.gsub(/(.*)"(.*)/, '\1\"\3') if quote_count % 2 == 1

      str
    end
  end
end
<commit_msg>Slightly less boost on the name<commit_after>module Searchable
  extend ActiveSupport::Concern

  included do
    include Elasticsearch::Model

    settings index: { number_of_shards: 1, number_of_replicas: 0 } do
      mapping do
        indexes :name, :analyzer => 'snowball', :boost => 5
        indexes :description, :analyzer => 'snowball'
        indexes :homepage
        indexes :repository_url
        indexes :repo_name
        indexes :latest_release_number, :analyzer => 'keyword'
        indexes :keywords, :analyzer => 'keyword'
        indexes :language, :analyzer => 'keyword'
        indexes :normalized_licenses, :analyzer => 'keyword'
        indexes :platform, :analyzer => 'keyword'

        indexes :created_at, type: 'date'
        indexes :updated_at, type: 'date'
        indexes :latest_release_published_at, type: 'date'

        indexes :rank, type: 'integer'
        indexes :stars, type: 'integer'
        indexes :versions_count, type: 'integer'
        indexes :github_repository_id, type: 'integer'
      end
    end

    after_commit on: [:create] do
      __elasticsearch__.index_document
    end

    after_commit on: [:update] do
      __elasticsearch__.update_document
    end

    after_commit on: [:destroy] do
      __elasticsearch__.delete_document
    end

    def as_indexed_json(options = {})
      as_json methods: [:stars, :language, :repo_name]
    end

    def self.total
      __elasticsearch__.client.count(index: 'projects')["count"]
    end

    def self.search(query, options = {})
      facet_limit = options.fetch(:facet_limit, 30)
      query = sanitize_query(query)
      options[:filters] ||= []
      search_definition = {
        query: {
          function_score: {
            query: {
              filtered: {
                 query: {query_string: {query: query}},
                 filter:{ bool: { must: [] } }
              }
            },
            field_value_factor: {
              field: "rank",
              "modifier": "log1p"
            }
          }
        },
        facets: {
          platforms: { terms: {
            field: "platform",
            size: facet_limit},
            facet_filter: {
              bool: {
                must: filter_format(options[:filters], :platform)
              }
            }
          },
          languages: { terms: {
              field: "language",
              size: facet_limit
            },
            facet_filter: {
              bool: {
                must: filter_format(options[:filters], :language)
              }
            }
          },
          licenses: {
            terms: {
              field: "normalized_licenses",
              size: facet_limit
            },
            facet_filter: {
              bool: {
                must: filter_format(options[:filters], :normalized_licenses)
              }
            }
          }#,
          # keywords: {
          #   terms: {
          #     field: "keywords",
          #     size: facet_limit
          #   },
          #   facet_filter: {
          #     bool: {
          #       must: filter_format(options[:filters], :keywords)
          #     }
          #   }
          # }
        },
        filter: {
          bool: {
            must: []
          }
        },
        suggest: {
          did_you_mean: {
            text: query,
            term: {
              size: 3,
              field: "name"
            }
          }
        }
      }
      search_definition[:sort]  = { (options[:sort] || '_score') => (options[:order] || 'desc') }
      search_definition[:track_scores] = true
      search_definition[:filter][:bool][:must] = filter_format(options[:filters])

      __elasticsearch__.search(search_definition)
    end

    def self.filter_format(filters, except = nil)
      filters.select { |k, v| v.present? && k != except }.map do |k, v|
        {
          term: { k => v }
        }
      end
    end

    def self.sanitize_query(str)
      return '*' if str.blank?
      # Escape special characters
      # http://lucene.apache.org/core/old_versioned_docs/versions/2_9_1/queryparsersyntax.html#Escaping Special Characters
      escaped_characters = Regexp.escape('\\+-&|/!(){}[]^~?:')
      str = str.gsub(/([#{escaped_characters}])/, '\\\\\1')

      # AND, OR and NOT are used by lucene as logical operators. We need
      # to escape them
      ['AND', 'OR', 'NOT'].each do |word|
        escaped_word = word.split('').map {|char| "\\#{char}" }.join('')
        str = str.gsub(/\s*\b(#{word.upcase})\b\s*/, " #{escaped_word} ")
      end

      # Escape odd quotes
      quote_count = str.count '"'
      str = str.gsub(/(.*)"(.*)/, '\1\"\3') if quote_count % 2 == 1

      str
    end
  end
end

{'fraction_non_alphanumeric': 0.12696736030421463, 'fraction_numerical': 0.003274532586880744, 'mean_word_length': 1.631461923290717, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1521574', 'n_tokens_mistral': 2829, 'n_tokens_neox': 2762, 'n_words': 583}