<commit_before>module Searchkick
  module IndexOptions
    def index_options
      options = @options
      language = options[:language]
      language = language.call if language.respond_to?(:call)
      index_type = options[:_type]
      index_type = index_type.call if index_type.respond_to?(:call)

      if options[:mappings] && !options[:merge_mappings]
        settings = options[:settings] || {}
        mappings = options[:mappings]
      else
        below60 = Searchkick.server_below?("6.0.0")
        below62 = Searchkick.server_below?("6.2.0")

        default_type = "text"
        default_analyzer = :searchkick_index
        keyword_mapping = {type: "keyword"}

        all = options.key?(:_all) ? options[:_all] : false
        index_true_value = true
        index_false_value = false

        keyword_mapping[:ignore_above] = options[:ignore_above] || 30000

        settings = {
          analysis: {
            analyzer: {
              searchkick_keyword: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
              },
              default_analyzer => {
                type: "custom",
                # character filters -> tokenizer -> token filters
                # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
                char_filter: ["ampersand"],
                tokenizer: "standard",
                # synonym should come last, after stemming and shingle
                # shingle must come before searchkick_stemmer
                filter: ["lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
              },
              searchkick_search: {
                type: "custom",
                char_filter: ["ampersand"],
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
              },
              searchkick_search2: {
                type: "custom",
                char_filter: ["ampersand"],
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_stemmer"]
              },
              # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
              searchkick_autocomplete_search: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding"]
              },
              searchkick_word_search: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding"]
              },
              searchkick_suggest_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
              },
              searchkick_text_start_index: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
              },
              searchkick_text_middle_index: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
              },
              searchkick_text_end_index: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
              },
              searchkick_word_start_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
              },
              searchkick_word_middle_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
              },
              searchkick_word_end_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
              }
            },
            filter: {
              searchkick_index_shingle: {
                type: "shingle",
                token_separator: ""
              },
              # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
              searchkick_search_shingle: {
                type: "shingle",
                token_separator: "",
                output_unigrams: false,
                output_unigrams_if_no_shingles: true
              },
              searchkick_suggest_shingle: {
                type: "shingle",
                max_shingle_size: 5
              },
              searchkick_edge_ngram: {
                type: "edgeNGram",
                min_gram: 1,
                max_gram: 50
              },
              searchkick_ngram: {
                type: "nGram",
                min_gram: 1,
                max_gram: 50
              },
              searchkick_stemmer: {
                # use stemmer if language is lowercase, snowball otherwise
                type: language == language.to_s.downcase ? "stemmer" : "snowball",
                language: language || "English"
              }
            },
            char_filter: {
              # https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html
              # &_to_and
              ampersand: {
                type: "mapping",
                mappings: ["&=> and "]
              }
            }
          }
        }

        if below60
          # ES docs say standard token filter does nothing in ES 5
          # (and therefore isn't needed at at), but tests say otherwise
          # https://www.elastic.co/guide/en/elasticsearch/reference/5.0/analysis-standard-tokenfilter.html
          [default_analyzer, :searchkick_search, :searchkick_search2].each do |analyzer|
            settings[:analysis][:analyzer][analyzer][:filter].unshift("standard")
          end
        end

        case language
        when "chinese"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "ik_smart"
            },
            searchkick_search: {
              type: "ik_smart"
            },
            searchkick_search2: {
              type: "ik_max_word"
            }
          )

          settings[:analysis][:filter].delete(:searchkick_stemmer)
        when "japanese"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "kuromoji"
            },
            searchkick_search: {
              type: "kuromoji"
            },
            searchkick_search2: {
              type: "kuromoji"
            }
          )
        when "korean"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "openkoreantext-analyzer"
            },
            searchkick_search: {
              type: "openkoreantext-analyzer"
            },
            searchkick_search2: {
              type: "openkoreantext-analyzer"
            }
          )
        when "vietnamese"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "vi_analyzer"
            },
            searchkick_search: {
              type: "vi_analyzer"
            },
            searchkick_search2: {
              type: "vi_analyzer"
            }
          )<commit_msg>Remove Japanese stemmer that does not exist (#1216)<commit_after>module Searchkick
  module IndexOptions
    def index_options
      options = @options
      language = options[:language]
      language = language.call if language.respond_to?(:call)
      index_type = options[:_type]
      index_type = index_type.call if index_type.respond_to?(:call)

      if options[:mappings] && !options[:merge_mappings]
        settings = options[:settings] || {}
        mappings = options[:mappings]
      else
        below60 = Searchkick.server_below?("6.0.0")
        below62 = Searchkick.server_below?("6.2.0")

        default_type = "text"
        default_analyzer = :searchkick_index
        keyword_mapping = {type: "keyword"}

        all = options.key?(:_all) ? options[:_all] : false
        index_true_value = true
        index_false_value = false

        keyword_mapping[:ignore_above] = options[:ignore_above] || 30000

        settings = {
          analysis: {
            analyzer: {
              searchkick_keyword: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
              },
              default_analyzer => {
                type: "custom",
                # character filters -> tokenizer -> token filters
                # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
                char_filter: ["ampersand"],
                tokenizer: "standard",
                # synonym should come last, after stemming and shingle
                # shingle must come before searchkick_stemmer
                filter: ["lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
              },
              searchkick_search: {
                type: "custom",
                char_filter: ["ampersand"],
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
              },
              searchkick_search2: {
                type: "custom",
                char_filter: ["ampersand"],
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_stemmer"]
              },
              # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
              searchkick_autocomplete_search: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding"]
              },
              searchkick_word_search: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding"]
              },
              searchkick_suggest_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
              },
              searchkick_text_start_index: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
              },
              searchkick_text_middle_index: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
              },
              searchkick_text_end_index: {
                type: "custom",
                tokenizer: "keyword",
                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
              },
              searchkick_word_start_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
              },
              searchkick_word_middle_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
              },
              searchkick_word_end_index: {
                type: "custom",
                tokenizer: "standard",
                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
              }
            },
            filter: {
              searchkick_index_shingle: {
                type: "shingle",
                token_separator: ""
              },
              # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
              searchkick_search_shingle: {
                type: "shingle",
                token_separator: "",
                output_unigrams: false,
                output_unigrams_if_no_shingles: true
              },
              searchkick_suggest_shingle: {
                type: "shingle",
                max_shingle_size: 5
              },
              searchkick_edge_ngram: {
                type: "edgeNGram",
                min_gram: 1,
                max_gram: 50
              },
              searchkick_ngram: {
                type: "nGram",
                min_gram: 1,
                max_gram: 50
              },
              searchkick_stemmer: {
                # use stemmer if language is lowercase, snowball otherwise
                type: language == language.to_s.downcase ? "stemmer" : "snowball",
                language: language || "English"
              }
            },
            char_filter: {
              # https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html
              # &_to_and
              ampersand: {
                type: "mapping",
                mappings: ["&=> and "]
              }
            }
          }
        }

        if below60
          # ES docs say standard token filter does nothing in ES 5
          # (and therefore isn't needed at at), but tests say otherwise
          # https://www.elastic.co/guide/en/elasticsearch/reference/5.0/analysis-standard-tokenfilter.html
          [default_analyzer, :searchkick_search, :searchkick_search2].each do |analyzer|
            settings[:analysis][:analyzer][analyzer][:filter].unshift("standard")
          end
        end

        case language
        when "chinese"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "ik_smart"
            },
            searchkick_search: {
              type: "ik_smart"
            },
            searchkick_search2: {
              type: "ik_max_word"
            }
          )

          settings[:analysis][:filter].delete(:searchkick_stemmer)
        when "japanese"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "kuromoji"
            },
            searchkick_search: {
              type: "kuromoji"
            },
            searchkick_search2: {
              type: "kuromoji"
            }
          )

          settings[:analysis][:filter].delete(:searchkick_stemmer)
        when "korean"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "openkoreantext-analyzer"
            },
            searchkick_search: {
              type: "openkoreantext-analyzer"
            },
            searchkick_search2: {
              type: "openkoreantext-analyzer"
            }
          )
        when "vietnamese"
          settings[:analysis][:analyzer].merge!(
            default_analyzer => {
              type: "vi_analyzer"
            },
            searchkick_search: {
              type: "vi_analyzer"
            },
            searchkick_search2: {
              type: "vi_analyzer"
            }
          )
{'fraction_non_alphanumeric': 0.10461038961038961, 'fraction_numerical': 0.005194805194805195, 'mean_word_length': 1.4543426294820716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 8, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2739460', 'n_tokens_mistral': 4267, 'n_tokens_neox': 3940, 'n_words': 694}