<commit_before>package com.sksamuel.elastic4s

import org.elasticsearch.common.xcontent.XContentBuilder

trait TokenFilter extends AnalyzerFilter

sealed trait TokenFilterDefinition extends TokenFilter with AnalyzerFilterDefinition

case object ReverseTokenFilter extends TokenFilter {
  val name = "reverse"
}

case object TrimTokenFilter extends TokenFilter {
  val name = "trim"
}

case object StandardTokenFilter extends TokenFilter {
  val name = "standard"
}

case object AsciiFoldingTokenFilter extends TokenFilter {
  val name = "asciifolding"
}

case object LowercaseTokenFilter extends TokenFilter {
  val name = "lowercase"
}

case object KStemTokenFilter extends TokenFilter {
  val name = "kstem"
}

case object PorterStemTokenFilter extends TokenFilter {
  val name = "porterStem"
}

case object UniqueTokenFilter extends TokenFilter {
  val name = "unique"
}

case class TruncateTokenFilter(name: String, length: Int = 10)
    extends TokenFilterDefinition {

  val filterType = "truncate"

  override def build(source: XContentBuilder): Unit = {
    source.field("length", length)
  }
}

case class LengthTokenFilter(name: String, min: Int = 0, max: Int = Integer.MAX_VALUE)
    extends TokenFilterDefinition {

  val filterType = "length"

  override def build(source: XContentBuilder): Unit = {
    if (min > 0) source.field("min", min)
    if (max < Integer.MAX_VALUE) source.field("max", max)<commit_msg>Add a Synonym Token Filter<commit_after>package com.sksamuel.elastic4s

import org.elasticsearch.common.xcontent.XContentBuilder

trait TokenFilter extends AnalyzerFilter

sealed trait TokenFilterDefinition extends TokenFilter with AnalyzerFilterDefinition

case object ReverseTokenFilter extends TokenFilter {
  val name = "reverse"
}

case object TrimTokenFilter extends TokenFilter {
  val name = "trim"
}

case object StandardTokenFilter extends TokenFilter {
  val name = "standard"
}

case object AsciiFoldingTokenFilter extends TokenFilter {
  val name = "asciifolding"
}

case object LowercaseTokenFilter extends TokenFilter {
  val name = "lowercase"
}

case object KStemTokenFilter extends TokenFilter {
  val name = "kstem"
}

case object PorterStemTokenFilter extends TokenFilter {
  val name = "porterStem"
}

case object UniqueTokenFilter extends TokenFilter {
  val name = "unique"
}

case class SynonymTokenFilter(name: String, path: String, ignoreCase: Boolean = false, expand: Boolean = true)
    extends TokenFilterDefinition {

  val filterType = "synonym"

  override def build(source: XContentBuilder): Unit = {
    source.field("synonyms_path", path)
    source.field("ignore_case", ignoreCase)
    source.field("expand", expand)
  }
}

case class TruncateTokenFilter(name: String, length: Int = 10)
    extends TokenFilterDefinition {

  val filterType = "truncate"

  override def build(source: XContentBuilder): Unit = {
    source.field("length", length)
  }
}

case class LengthTokenFilter(name: String, min: Int = 0, max: Int = Integer.MAX_VALUE)
    extends TokenFilterDefinition {

  val filterType = "length"

  override def build(source: XContentBuilder): Unit = {
    if (min > 0) source.field("min", min)
    if (max < Integer.MAX_VALUE) source.field("max", max)
{'fraction_non_alphanumeric': 0.08227258615336852, 'fraction_numerical': 0.003104625892579944, 'mean_word_length': 4.868852459016393, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6268339', 'n_tokens_mistral': 950, 'n_tokens_neox': 927, 'n_words': 287}