<commit_before>package lila.fishnet

import org.joda.time.DateTime
import reactivemongo.bson._
import scala.concurrent.duration._

import lila.db.Implicits._

private final class Monitor(
    moveDb: MoveDB,
    repo: FishnetRepo,
    sequencer: lila.hub.FutureSequencer,
    scheduler: lila.common.Scheduler) {

  private[fishnet] def acquire(client: Client) =
    lila.mon.fishnet.acquire.count(client.userId.value)()

  private case class AnalysisMeta(time: Int, nodes: Int, nps: Int, depth: Int, pvSize: Int)

  private def sumOf[A](ints: List[A])(f: A => Option[Int]) = ints.foldLeft(0) {
    case (acc, a) => acc + f(a).getOrElse(0)
  }

  private[fishnet] def analysis(work: Work.Analysis, client: Client, result: JsonApi.Request.PostAnalysis) = {
    success(work, client)

    val monitor = lila.mon.fishnet.analysis by client.userId.value

    result.engine.options.hashInt foreach { monitor.hash(_) }
    result.engine.options.threadsInt foreach { monitor.threads(_) }

    monitor.totalSecond(sumOf(result.analysis)(_.time) / 1000)
    monitor.totalMeganode(sumOf(result.analysis) { eval =>
      eval.nodes ifFalse eval.mateFound
    } / 1000000)
    monitor.totalPosition(result.analysis.size)

    val metaMovesSample = result.analysis.drop(6).filterNot(_.mateFound).take(100)
    def avgOf(f: JsonApi.Request.Evaluation => Option[Int]): Option[Int] = {
      val (sum, nb) = metaMovesSample.foldLeft(0 -> 0) {
        case ((sum, nb), move) => f(move).fold(sum -> nb) { v =>
          (sum + v, nb + 1)
        }
      }
      (nb > 0) option (sum / nb)
    }
    avgOf(_.time) foreach { monitor.movetime(_) }
    avgOf(_.nodes) foreach { monitor.node(_) }
    avgOf(_.cappedNps) foreach { monitor.nps(_) }
    avgOf(_.depth) foreach { monitor.depth(_) }
    avgOf(_.pvList.size.some) foreach { monitor.pvSize(_) }
  }

  private[fishnet] def move(work: Work.Move, client: Client) = {
    success(work, client)
    if (work.level == 8) work.acquiredAt foreach { acquiredAt =>
      lila.mon.fishnet.move.time(client.userId.value)(nowMillis - acquiredAt.getMillis)
    }
  }

  private def sample[A](elems: List[A], n: Int) = scala.util.Random shuffle elems take n

  private def success(work: Work, client: Client) = {

    lila.mon.fishnet.client.result(client.userId.value, work.skill.key).success()

    work.acquiredAt foreach { acquiredAt =>
      lila.mon.fishnet.queue.db(work.skill.key) {
        acquiredAt.getMillis - work.createdAt.getMillis
      }
    }
  }<commit_msg>better sampling of fishnet evaluations<commit_after>package lila.fishnet

import org.joda.time.DateTime
import reactivemongo.bson._
import scala.concurrent.duration._

import lila.db.Implicits._

private final class Monitor(
    moveDb: MoveDB,
    repo: FishnetRepo,
    sequencer: lila.hub.FutureSequencer,
    scheduler: lila.common.Scheduler) {

  private[fishnet] def acquire(client: Client) =
    lila.mon.fishnet.acquire.count(client.userId.value)()

  private case class AnalysisMeta(time: Int, nodes: Int, nps: Int, depth: Int, pvSize: Int)

  private def sumOf[A](ints: List[A])(f: A => Option[Int]) = ints.foldLeft(0) {
    case (acc, a) => acc + f(a).getOrElse(0)
  }

  private[fishnet] def analysis(work: Work.Analysis, client: Client, result: JsonApi.Request.PostAnalysis) = {
    success(work, client)

    val monitor = lila.mon.fishnet.analysis by client.userId.value

    result.engine.options.hashInt foreach { monitor.hash(_) }
    result.engine.options.threadsInt foreach { monitor.threads(_) }

    monitor.totalSecond(sumOf(result.analysis)(_.time) / 1000)
    monitor.totalMeganode(sumOf(result.analysis) { eval =>
      eval.nodes ifFalse eval.mateFound
    } / 1000000)
    monitor.totalPosition(result.analysis.size)

    val metaMovesSample = sample(result.analysis.drop(6).filterNot(_.mateFound), 100)
    def avgOf(f: JsonApi.Request.Evaluation => Option[Int]): Option[Int] = {
      val (sum, nb) = metaMovesSample.foldLeft(0 -> 0) {
        case ((sum, nb), move) => f(move).fold(sum -> nb) { v =>
          (sum + v, nb + 1)
        }
      }
      (nb > 0) option (sum / nb)
    }
    avgOf(_.time) foreach { monitor.movetime(_) }
    avgOf(_.nodes) foreach { monitor.node(_) }
    avgOf(_.cappedNps) foreach { monitor.nps(_) }
    avgOf(_.depth) foreach { monitor.depth(_) }
    avgOf(_.pvList.size.some) foreach { monitor.pvSize(_) }
  }

  private[fishnet] def move(work: Work.Move, client: Client) = {
    success(work, client)
    if (work.level == 8) work.acquiredAt foreach { acquiredAt =>
      lila.mon.fishnet.move.time(client.userId.value)(nowMillis - acquiredAt.getMillis)
    }
  }

  private def sample[A](elems: List[A], n: Int) =
    if (elems.size <= n) elems else scala.util.Random shuffle elems take n

  private def success(work: Work, client: Client) = {

    lila.mon.fishnet.client.result(client.userId.value, work.skill.key).success()

    work.acquiredAt foreach { acquiredAt =>
      lila.mon.fishnet.queue.db(work.skill.key) {
        acquiredAt.getMillis - work.createdAt.getMillis
      }
    }
  }
{'fraction_non_alphanumeric': 0.1409090909090909, 'fraction_numerical': 0.008695652173913044, 'mean_word_length': 3.3932291666666665, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3938311', 'n_tokens_mistral': 1801, 'n_tokens_neox': 1732, 'n_words': 344}