<commit_before>module Spree
  Product.class_eval do
    include Elasticsearch::Model

    index_name Spree::ElasticsearchSettings.index
    document_type 'spree_product'

    mapping _all: {'index_analyzer' => 'search_analyzer', 'search_analyzer' => 'whitespace_analyzer'} do
      # search, autocomplete, untouched & exact match
      indexes :name, type: 'multi_field' do
        indexes :name,         type: 'string', analyzer: 'search_analyzer'
        indexes :autocomplete, type: 'string', analyzer: 'ngram_analyzer'
        indexes :untouched,    type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,    type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end
      indexes :description, analyzer: 'snowball'
      indexes :sku, type: 'string', index: 'not_analyzed'
      indexes :taxon_ids, type: 'string', index: 'not_analyzed'

      indexes :available_on, type: 'date', format: 'dateOptionalTime', include_in_all: false
      indexes :published,    type: 'boolean', index: 'not_analyzed', include_in_all: false

      indexes :tracks, type: 'multi_field' do
        indexes :tracks,      type: 'string', analyzer: 'search_analyzer'
        indexes :untouched,   type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,   type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end

      indexes :track_artists, type: 'multi_field' do
        indexes :track_artists, type: 'string', analyzer: 'search_analyzer'
        indexes :untouched,     type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,     type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end

      indexes :variants, type: 'nested' do
        indexes :id, type: 'integer', index: 'not_analyzed'
        indexes :sku, type: 'string', index: 'not_analyzed'
        indexes :price, type: 'double'
        indexes :release_date, type: 'date', format: 'dateOptionalTime', include_in_all: false

        indexes :format, type: 'string', index: 'not_analyzed'
        indexes :uber_format, type: 'string', index: 'not_analyzed'

        indexes :can_preorder, type: 'boolean', index: 'not_analyzed'
        indexes :in_stock,     type: 'boolean', index: 'not_analyzed'
        indexes :published,    type: 'boolean', index: 'not_analyzed'
      end

      # genres
      indexes :genres, type: 'integer', index: 'not_analyzed'
      # artists
      indexes :artists, type: 'multi_field' do
        indexes :artists,      type: 'string', analyzer: 'search_analyzer'
        indexes :autocomplete, type: 'string', analyzer: 'ngram_analyzer'
        indexes :untouched,    type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,    type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end
      # label
      indexes :label, type: 'multi_field' do
        indexes :label,        type: 'string', analyzer: 'search_analyzer'
        indexes :autocomplete, type: 'string', analyzer: 'ngram_analyzer'
        indexes :untouched,    type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,    type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end

      indexes :created_at, type: 'date', format: 'dateOptionalTime', include_in_all: false
      indexes :deleted_at, type: 'date', format: 'dateOptionalTime', include_in_all: false
    end

    def as_indexed_json(options = {})
      result = as_json({
        methods: [:sku],
        only: [:available_on, :description, :name, :published, :created_at, :deleted_at],
      })
      result[:artists] = artists.map(&:name)
      result[:genres] = release.genres.map(&:id)
      result[:label] = label.try(:name) 

      # Store names of all tracks, uniq'd to be able to search by track name
      result[:tracks] = track_products.map(&:name).uniq
      result[:track_artists] = release.tracks.map {|t| t.artists.map(&:name) }.flatten.uniq
  
      # map variant data
      result[:variants] = variants.map do |v|
        h = v.as_json({
          only: [:id],
          methods: [:price, :format, :uber_format, :release_date]
        })
        h[:sku] = v.definitive_release_format.try(:catalogue_number)
        h[:can_preorder] = v.definitive_release_format.try(:can_pre_order) || false
        h[:published] = v.published?
        h[:in_stock] = v.suppliable?
        h
      end

      result[:taxon_ids] = taxons.map(&:id).uniq unless taxons.empty?
      result
    end

    # Inner class used to query elasticsearch. The idea is that the query is dynamically build based on the parameters.
    class Product::ElasticsearchQuery
      include ::Virtus.model

      attribute :query, String
      attribute :taxons, Array
      attribute :genres, Array
      attribute :uber_format, Array
      attribute :status, Array
      attribute :release_date, String
      attribute :category, String
      attribute :sorting, String

      # When browse_mode is enabled, the taxon filter is placed at top level. This causes the results to be limited, but facetting is done on the complete dataset.
      # When browse_mode is disabled, the taxon filter is placed inside the filtered query. This causes the facets to be limited to the resulting set.

      # Method that creates the actual query based on the current attributes.
      # The idea is to always to use the following schema and fill in the blanks.
      # {
      #   query: {
      #     filtered: {
      #       query: {
      #         query_string: { query: , fields: [] }
      #       }
      #       filter: {
      #         and: [
      #           { terms: { taxons: [] } },
      #           { terms: { properties: [] } }
      #         ]
      #       }
      #     }
      #   }
      #   filter: { range: { price: { lte: , gte: } } },
      #   sort: [],
      #   aggs:
      # }
      def to_hash
        q = { match_all: {} }

        fields, exact_fields = case category
        when 'artist'
          [
            ['artists^3', 'track_artists'],
            ['artists.lowercase^3', 'track_artists.lowercase']
          ]
        when 'release-title'
          [['name^3', 'tracks'], ['name.lowercase^3', 'tracks.lowercase']]
        when 'label'
          [['label'], ['label.lowercase^1']]
        when 'catalogue-number'
          [['sku'], ['sku']]
        else
          [
            ['artists^5', 'name^3', 'label^1', 'tracks', 'track_artists', 'sku'],
            ['artists.lowercase^5', 'name.lowercase^3', 'label.lowercase^1', 'tracks.lowercase', 'track_artists.lowercase']
          ]
          # TODO: re-enable description search
        end

        unless query.blank? # nil or empty
          q = {
            bool: {
              should: [
                { query_string: { query: query, fields: fields, default_operator: 'AND', use_dis_max: true } },
                { multi_match: { query: query, fields: exact_fields, boost: 200, type: 'best_fields'} }
              ]
            }
          }
        end
        query = q

        and_filter = []<commit_msg>Reduce the amount of boost on the exact match, so as to not eliminate non-exact matches.<commit_after>module Spree
  Product.class_eval do
    include Elasticsearch::Model

    index_name Spree::ElasticsearchSettings.index
    document_type 'spree_product'

    mapping _all: {'index_analyzer' => 'search_analyzer', 'search_analyzer' => 'whitespace_analyzer'} do
      # search, autocomplete, untouched & exact match
      indexes :name, type: 'multi_field' do
        indexes :name,         type: 'string', analyzer: 'search_analyzer'
        indexes :autocomplete, type: 'string', analyzer: 'ngram_analyzer'
        indexes :untouched,    type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,    type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end
      indexes :description, analyzer: 'snowball'
      indexes :sku, type: 'string', index: 'not_analyzed'
      indexes :taxon_ids, type: 'string', index: 'not_analyzed'

      indexes :available_on, type: 'date', format: 'dateOptionalTime', include_in_all: false
      indexes :published,    type: 'boolean', index: 'not_analyzed', include_in_all: false

      indexes :tracks, type: 'multi_field' do
        indexes :tracks,      type: 'string', analyzer: 'search_analyzer'
        indexes :untouched,   type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,   type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end

      indexes :track_artists, type: 'multi_field' do
        indexes :track_artists, type: 'string', analyzer: 'search_analyzer'
        indexes :untouched,     type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,     type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end

      indexes :variants, type: 'nested' do
        indexes :id, type: 'integer', index: 'not_analyzed'
        indexes :sku, type: 'string', index: 'not_analyzed'
        indexes :price, type: 'double'
        indexes :release_date, type: 'date', format: 'dateOptionalTime', include_in_all: false

        indexes :format, type: 'string', index: 'not_analyzed'
        indexes :uber_format, type: 'string', index: 'not_analyzed'

        indexes :can_preorder, type: 'boolean', index: 'not_analyzed'
        indexes :in_stock,     type: 'boolean', index: 'not_analyzed'
        indexes :published,    type: 'boolean', index: 'not_analyzed'
      end

      # genres
      indexes :genres, type: 'integer', index: 'not_analyzed'
      # artists
      indexes :artists, type: 'multi_field' do
        indexes :artists,      type: 'string', analyzer: 'search_analyzer'
        indexes :autocomplete, type: 'string', analyzer: 'ngram_analyzer'
        indexes :untouched,    type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,    type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end
      # label
      indexes :label, type: 'multi_field' do
        indexes :label,        type: 'string', analyzer: 'search_analyzer'
        indexes :autocomplete, type: 'string', analyzer: 'ngram_analyzer'
        indexes :untouched,    type: 'string', include_in_all: false, index: 'not_analyzed'
        indexes :lowercase,    type: 'string', analyzer: 'lowercase_analyzer', include_in_all: false
      end

      indexes :created_at, type: 'date', format: 'dateOptionalTime', include_in_all: false
      indexes :deleted_at, type: 'date', format: 'dateOptionalTime', include_in_all: false
    end

    def as_indexed_json(options = {})
      result = as_json({
        methods: [:sku],
        only: [:available_on, :description, :name, :published, :created_at, :deleted_at],
      })
      result[:artists] = artists.map(&:name)
      result[:genres] = release.genres.map(&:id)
      result[:label] = label.try(:name) 

      # Store names of all tracks, uniq'd to be able to search by track name
      result[:tracks] = track_products.map(&:name).uniq
      result[:track_artists] = release.tracks.map {|t| t.artists.map(&:name) }.flatten.uniq
  
      # map variant data
      result[:variants] = variants.map do |v|
        h = v.as_json({
          only: [:id],
          methods: [:price, :format, :uber_format, :release_date]
        })
        h[:sku] = v.definitive_release_format.try(:catalogue_number)
        h[:can_preorder] = v.definitive_release_format.try(:can_pre_order) || false
        h[:published] = v.published?
        h[:in_stock] = v.suppliable?
        h
      end

      result[:taxon_ids] = taxons.map(&:id).uniq unless taxons.empty?
      result
    end

    # Inner class used to query elasticsearch. The idea is that the query is dynamically build based on the parameters.
    class Product::ElasticsearchQuery
      include ::Virtus.model

      attribute :query, String
      attribute :taxons, Array
      attribute :genres, Array
      attribute :uber_format, Array
      attribute :status, Array
      attribute :release_date, String
      attribute :category, String
      attribute :sorting, String

      # When browse_mode is enabled, the taxon filter is placed at top level. This causes the results to be limited, but facetting is done on the complete dataset.
      # When browse_mode is disabled, the taxon filter is placed inside the filtered query. This causes the facets to be limited to the resulting set.

      # Method that creates the actual query based on the current attributes.
      # The idea is to always to use the following schema and fill in the blanks.
      # {
      #   query: {
      #     filtered: {
      #       query: {
      #         query_string: { query: , fields: [] }
      #       }
      #       filter: {
      #         and: [
      #           { terms: { taxons: [] } },
      #           { terms: { properties: [] } }
      #         ]
      #       }
      #     }
      #   }
      #   filter: { range: { price: { lte: , gte: } } },
      #   sort: [],
      #   aggs:
      # }
      def to_hash
        q = { match_all: {} }

        fields, exact_fields = case category
        when 'artist'
          [
            ['artists^3', 'track_artists'],
            ['artists.lowercase^3', 'track_artists.lowercase']
          ]
        when 'release-title'
          [['name^3', 'tracks'], ['name.lowercase^3', 'tracks.lowercase']]
        when 'label'
          [['label'], ['label.lowercase^1']]
        when 'catalogue-number'
          [['sku'], ['sku']]
        else
          [
            ['artists^5', 'name^3', 'label^1', 'tracks', 'track_artists', 'sku'],
            ['artists.lowercase^5', 'name.lowercase^3', 'label.lowercase^1', 'tracks.lowercase', 'track_artists.lowercase']
          ]
          # TODO: re-enable description search
        end

        unless query.blank? # nil or empty
          q = {
            bool: {
              should: [
                { query_string: { query: query, fields: fields, default_operator: 'AND', use_dis_max: true } },
                { multi_match: { query: query, fields: exact_fields, boost: 1, type: 'best_fields'} }
              ]
            }
          }
        end
        query = q

        and_filter = []
{'fraction_non_alphanumeric': 0.11944965603502189, 'fraction_numerical': 0.0018066847335140017, 'mean_word_length': 2.5623762376237624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '393982', 'n_tokens_mistral': 4475, 'n_tokens_neox': 4351, 'n_words': 1193}