<commit_before>package elasticsearch

import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s._
import com.sksamuel.elastic4s.mappings._
import helpers._
import models._
import models.cassandra.Indexable
import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.delete.DeleteResponse
import org.elasticsearch.action.index.IndexResponse
import org.elasticsearch.action.search.MultiSearchResponse
import org.elasticsearch.action.update.UpdateResponse
import org.elasticsearch.common.settings.ImmutableSettings
import org.elasticsearch.common.xcontent.XContentBuilder
import play.api.libs.json.JsValue

import scala.concurrent._

/**
 * @author <EMAIL>
 */
class ElasticSearch(
  val basicPlayApi: BasicPlayApi
)
  extends AppConfigComponents
  with BasicPlayComponents
  with DefaultPlayExecutor
  with BootingProcess
  with Logging {

  applicationLifecycle.addStopHook { () =>
    Logger.info(s"Shutdown Elastic Search Client")
    Future.successful(Client.close())
  }

  import ESyntax._

  implicit lazy val Client = (
    for {
      settings <- config.getString("cluster.name").map { name =>
        ImmutableSettings
          .settingsBuilder()
          .put("cluster.name", name)
          .build()
      }
      uri <- config.getString("client.uri").map { uri =>
        ElasticsearchClientUri(uri)
      }
    } yield {
      ElasticClient.remote(settings, uri)
    })
    .getOrElse {
      Logger.warn("client.uri / cluster.name is not configured yet")
      ElasticClient.local
    }

  implicit val indexName = domain

  // create index with customized analyzers
  onStart(
    Client.execute {
      create index indexName analysis(
        ElasticSearch.analyzers.defaultDef,
        ElasticSearch.analyzers.emailDef)
    }.recover {
      // if index already exists
      case e: Throwable => Logger.debug(e.getMessage)
    }
  )

  def PutMapping[T <: Indexable[_]](
    mapping: T => Iterable[TypedFieldDefinition]
  )(implicit t: T) = Client.execute {
    put mapping indexName / t.basicName as mapping(t)
  }

  def Index[T <: HasID[_]](r: T) = new IndexAction(r)

  def Delete[ID](r: ID) = new DeleteAction(r)

  def Delete = new DeleteMappingAction

  def Update[T <: HasID[_]](r: T) = new UpdateAction(r)

  def BulkIndex[T <: HasID[_]](rs: Seq[T]) = new BulkIndexAction(rs)

  def Search(q: Option[String], p: Pager, sort: Seq[SortField]) =
    new SearchAction(q, p, sort)

  def Multi(defs: (ElasticSearch => ReadySearchDefinition)*) =
    new ReadyMultiSearchDefinition(Client)(
      defs.map(_ (this).definition)
    )

}

object ElasticSearch extends CanonicalNamed {

  override val basicName: String = "elastic_search"

  object analyzers {
    val emailDef = CustomAnalyzerDefinition("email", UaxUrlEmailTokenizer)
    val email    = CustomAnalyzer("email")

    val defaultDef = new AnalyzerDefinition("default") {
      def build(source: XContentBuilder): Unit = {
        source.field("type", "smartcn")
      }
    }
  }

}
<commit_msg>Added a few Type Param in ES<commit_after>package elasticsearch

import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s._
import com.sksamuel.elastic4s.mappings._
import helpers._
import models._
import models.cassandra.Indexable
import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.delete.DeleteResponse
import org.elasticsearch.action.index.IndexResponse
import org.elasticsearch.action.search.MultiSearchResponse
import org.elasticsearch.action.update.UpdateResponse
import org.elasticsearch.common.settings.ImmutableSettings
import org.elasticsearch.common.xcontent.XContentBuilder
import play.api.libs.json.JsValue

import scala.concurrent._

/**
 * @author <EMAIL>
 */
class ElasticSearch(
  val basicPlayApi: BasicPlayApi
)
  extends AppConfigComponents
  with BasicPlayComponents
  with DefaultPlayExecutor
  with BootingProcess
  with Logging {

  applicationLifecycle.addStopHook { () =>
    Logger.info(s"Shutdown Elastic Search Client")
    Future.successful(Client.close())
  }

  import ESyntax._

  implicit lazy val Client = (
    for {
      settings <- config.getString("cluster.name").map { name =>
        ImmutableSettings
          .settingsBuilder()
          .put("cluster.name", name)
          .build()
      }
      uri <- config.getString("client.uri").map { uri =>
        ElasticsearchClientUri(uri)
      }
    } yield {
      ElasticClient.remote(settings, uri)
    })
    .getOrElse {
      Logger.warn("client.uri / cluster.name is not configured yet")
      ElasticClient.local
    }

  implicit val indexName = domain

  // create index with customized analyzers
  onStart(
    Client.execute {
      create index indexName analysis(
        ElasticSearch.analyzers.defaultDef,
        ElasticSearch.analyzers.emailDef)
    }.recover {
      // if index already exists
      case e: Throwable => Logger.debug(e.getMessage)
    }
  )

  def PutMapping[T <: Indexable[_]](
    mapping: T => Iterable[TypedFieldDefinition]
  )(implicit t: T) = Client.execute {
    put mapping indexName / t.basicName as mapping(t)
  }

  def Index[T <: HasID[_]](r: T) = new IndexAction[T](r)

  def Delete[ID](r: ID) = new DeleteAction[ID](r)

  def Delete = new DeleteMappingAction

  def Update[T <: HasID[_]](r: T) = new UpdateAction[T](r)

  def BulkIndex[T <: HasID[_]](rs: Seq[T]) = new BulkIndexAction[T](rs)

  def Search(q: Option[String], p: Pager, sort: Seq[SortField]) =
    new SearchAction(q, p, sort)

  def Multi(defs: (ElasticSearch => ReadySearchDefinition)*) =
    new ReadyMultiSearchDefinition(Client)(
      defs.map(_ (this).definition)
    )

}

object ElasticSearch extends CanonicalNamed {

  override val basicName: String = "elastic_search"

  object analyzers {
    val emailDef = CustomAnalyzerDefinition("email", UaxUrlEmailTokenizer)
    val email    = CustomAnalyzer("email")

    val defaultDef = new AnalyzerDefinition("default") {
      def build(source: XContentBuilder): Unit = {
        source.field("type", "smartcn")
      }
    }
  }

}

{'fraction_non_alphanumeric': 0.10729200128493414, 'fraction_numerical': 0.0009637006103437199, 'mean_word_length': 3.5955719557195573, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3517713', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1887, 'n_words': 412}