<commit_before>package lila.gameSearch

import akka.actor._
import akka.pattern.pipe
import com.sksamuel.elastic4s.ElasticClient
import com.sksamuel.elastic4s.ElasticDsl.{ RichFuture => _, _ }
import com.sksamuel.elastic4s.mappings.FieldType._

import lila.game.actorApi.{ InsertGame, FinishGame }
import lila.game.GameRepo
import lila.search.actorApi._
import lila.search.ElasticSearch

private[gameSearch] final class Indexer(
    client: ElasticClient,
    indexName: String,
    typeName: String) extends Actor {

  context.system.lilaBus.subscribe(self, 'finishGame)

  def receive = {

    case Search(definition)     => client execute definition pipeTo sender
    case Count(definition)      => client execute definition pipeTo sender

    case FinishGame(game, _, _) => self ! InsertGame(game)

    case InsertGame(game) => if (storable(game)) {
      GameRepo isAnalysed game.id foreach { analysed =>
        client execute store(indexName, game, analysed)
      }
    }

    case Reset =>
      val replyTo = sender
      val tempIndexName = "lila_" + ornicar.scalalib.Random.nextString(4)
      ElasticSearch.createType(client, tempIndexName, typeName)
      import Fields._
      client.execute {
        put mapping tempIndexName / typeName as Seq(
          status typed ShortType,
          turns typed ShortType,
          rated typed BooleanType,
          variant typed ShortType,
          uids typed StringType,
          winner typed StringType,
          winnerColor typed ShortType,
          averageRating typed ShortType,
          ai typed ShortType,
          opening typed StringType,
          date typed DateType format ElasticSearch.Date.format,
          duration typed IntegerType,
          analysed typed BooleanType,
          whiteUser typed StringType,
          blackUser typed StringType
        ).map(_ index "not_analyzed")
      }.await
      import scala.concurrent.duration._
      import play.api.libs.json.Json
      import lila.db.api._
      import lila.game.tube.gameTube
      loginfo("[game search] counting games...")
      val size = SprayPimpedFuture($count($select.all)).await
      val batchSize = 1000
      var nb = 0
      var nbSkipped = 0
      var started = nowMillis
      $enumerate.bulk[Option[lila.game.Game]]($query.all, batchSize, limit = 500000) { gameOptions =>
        val games = gameOptions.flatten filter storable
        val nbGames = games.size
        (GameRepo filterAnalysed games.map(_.id).toSeq flatMap { analysedIds =><commit_msg>remove game indexer limit<commit_after>package lila.gameSearch

import akka.actor._
import akka.pattern.pipe
import com.sksamuel.elastic4s.ElasticClient
import com.sksamuel.elastic4s.ElasticDsl.{ RichFuture => _, _ }
import com.sksamuel.elastic4s.mappings.FieldType._

import lila.game.actorApi.{ InsertGame, FinishGame }
import lila.game.GameRepo
import lila.search.actorApi._
import lila.search.ElasticSearch

private[gameSearch] final class Indexer(
    client: ElasticClient,
    indexName: String,
    typeName: String) extends Actor {

  context.system.lilaBus.subscribe(self, 'finishGame)

  def receive = {

    case Search(definition)     => client execute definition pipeTo sender
    case Count(definition)      => client execute definition pipeTo sender

    case FinishGame(game, _, _) => self ! InsertGame(game)

    case InsertGame(game) => if (storable(game)) {
      GameRepo isAnalysed game.id foreach { analysed =>
        client execute store(indexName, game, analysed)
      }
    }

    case Reset =>
      val replyTo = sender
      val tempIndexName = "lila_" + ornicar.scalalib.Random.nextString(4)
      ElasticSearch.createType(client, tempIndexName, typeName)
      import Fields._
      client.execute {
        put mapping tempIndexName / typeName as Seq(
          status typed ShortType,
          turns typed ShortType,
          rated typed BooleanType,
          variant typed ShortType,
          uids typed StringType,
          winner typed StringType,
          winnerColor typed ShortType,
          averageRating typed ShortType,
          ai typed ShortType,
          opening typed StringType,
          date typed DateType format ElasticSearch.Date.format,
          duration typed IntegerType,
          analysed typed BooleanType,
          whiteUser typed StringType,
          blackUser typed StringType
        ).map(_ index "not_analyzed")
      }.await
      import scala.concurrent.duration._
      import play.api.libs.json.Json
      import lila.db.api._
      import lila.game.tube.gameTube
      loginfo("[game search] counting games...")
      val size = SprayPimpedFuture($count($select.all)).await
      val batchSize = 1000
      var nb = 0
      var nbSkipped = 0
      var started = nowMillis
      $enumerate.bulk[Option[lila.game.Game]]($query.all, batchSize) { gameOptions =>
        val games = gameOptions.flatten filter storable
        val nbGames = games.size
        (GameRepo filterAnalysed games.map(_.id).toSeq flatMap { analysedIds =>
{'fraction_non_alphanumeric': 0.08124253285543608, 'fraction_numerical': 0.005177220230983672, 'mean_word_length': 2.779533483822423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4614326', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1433, 'n_words': 377}