<commit_before>package lila.insight

import reactivemongo.api.collections.bson.BSONBatchCommands.AggregationFramework._
import reactivemongo.bson._
import scalaz.NonEmptyList

import lila.db.dsl._

private final class AggregationPipeline {

  import lila.insight.{ Dimension => D, Metric => M }
  import Storage._
  import Entry.{ BSONFields => F }

  private lazy val movetimeIdDispatcher =
    MovetimeRange.reversedNoInf.foldLeft[BSONValue](BSONInteger(MovetimeRange.MTRInf.id)) {
      case (acc, mtr) => $doc(
        "$cond" -> $arr(
          $doc("$lte" -> $arr("$" + F.moves("t"), mtr.tenths.last)),
          mtr.id,
          acc))
    }
  private lazy val materialIdDispatcher = $doc(
    "$cond" -> $arr(
      $doc("$eq" -> $arr("$" + F.moves("i"), 0)),
      MaterialRange.Equal.id,
      MaterialRange.reversedButEqualAndLast.foldLeft[BSONValue](BSONInteger(MaterialRange.Up4.id)) {
        case (acc, mat) => $doc(
          "$cond" -> $arr(
            $doc(mat.negative.fold("$lt", "$lte") -> $arr("$" + F.moves("i"), mat.imbalance)),
            mat.id,
            acc))
      }))
  private def dimensionGroupId(dim: Dimension[_]): BSONValue = dim match {
    case Dimension.MovetimeRange => movetimeIdDispatcher
    case Dimension.MaterialRange => materialIdDispatcher
    case d                       => BSONString("$" + d.dbKey)
  }

  private val sampleGames = Sample(10 * 1000)
  private val sortDate = Sort(Descending(F.date))
  private val sampleMoves = Sample(200 * 1000).some
  private val unwindMoves = UnwindField(F.moves).some
  private val sortNb = Sort(Descending("nb")).some
  private def limit(nb: Int) = Limit(nb).some
  private def group(d: Dimension[_], f: GroupFunction) = Group(dimensionGroupId(d))(
    "v" -> f,
    "nb" -> SumValue(1),
    "ids" -> AddFieldToSet("_id")
  ).some
  private def groupMulti(d: Dimension[_], metricDbKey: String) = Group($doc(
    "dimension" -> dimensionGroupId(d),
    "metric" -> ("$" + metricDbKey)))(
    "v" -> SumValue(1),
    "ids" -> AddFieldToSet("_id")
  ).some
  private val regroupStacked = GroupField("_id.dimension")(
    "nb" -> SumField("v"),
    "ids" -> FirstField("ids"),
    "stack" -> Push(BSONDocument(
      "metric" -> "_id.metric",
      "v" -> "v"))).some
  private val sliceIds = Project($doc(
    "_id" -> true,
    "v" -> true,
    "nb" -> true,
    "ids" -> $doc("$slice" -> $arr("$ids", 4))
  )).some
  private val sliceStackedIds = Project($doc(
    "_id" -> true,
    "nb" -> true,
    "stack" -> true,
    "ids" -> $doc("$slice" -> $arr("$ids", 4))
  )).some

  def apply(question: Question[_], userId: String): NonEmptyList[PipelineOperator] = {
    import question.{ dimension, metric, filters }
    val gameMatcher = combineDocs(question.filters.collect {
      case f if f.dimension.isInGame => f.matcher
    })
    def matchMoves(extraMatcher: Bdoc = $empty) =
      combineDocs(extraMatcher :: question.filters.collect {
        case f if f.dimension.isInMove => f.matcher
      }).some.filterNot(_.isEmpty) map Match
    def projectForMove = Project(BSONDocument({
      metric.dbKey :: dimension.dbKey :: filters.collect {
        case Filter(d, _) if d.isInMove => d.dbKey<commit_msg>fix reactivemongo BC break in insight pipeline<commit_after>package lila.insight

import reactivemongo.api.collections.bson.BSONBatchCommands.AggregationFramework._
import reactivemongo.bson._
import scalaz.NonEmptyList

import lila.db.dsl._

private final class AggregationPipeline {

  import lila.insight.{ Dimension => D, Metric => M }
  import Storage._
  import Entry.{ BSONFields => F }

  private lazy val movetimeIdDispatcher =
    MovetimeRange.reversedNoInf.foldLeft[BSONValue](BSONInteger(MovetimeRange.MTRInf.id)) {
      case (acc, mtr) => $doc(
        "$cond" -> $arr(
          $doc("$lte" -> $arr("$" + F.moves("t"), mtr.tenths.last)),
          mtr.id,
          acc))
    }
  private lazy val materialIdDispatcher = $doc(
    "$cond" -> $arr(
      $doc("$eq" -> $arr("$" + F.moves("i"), 0)),
      MaterialRange.Equal.id,
      MaterialRange.reversedButEqualAndLast.foldLeft[BSONValue](BSONInteger(MaterialRange.Up4.id)) {
        case (acc, mat) => $doc(
          "$cond" -> $arr(
            $doc(mat.negative.fold("$lt", "$lte") -> $arr("$" + F.moves("i"), mat.imbalance)),
            mat.id,
            acc))
      }))
  private def dimensionGroupId(dim: Dimension[_]): BSONValue = dim match {
    case Dimension.MovetimeRange => movetimeIdDispatcher
    case Dimension.MaterialRange => materialIdDispatcher
    case d                       => BSONString("$" + d.dbKey)
  }

  private val sampleGames = Sample(10 * 1000)
  private val sortDate = Sort(Descending(F.date))
  private val sampleMoves = Sample(200 * 1000).some
  private val unwindMoves = UnwindField(F.moves).some
  private val sortNb = Sort(Descending("nb")).some
  private def limit(nb: Int) = Limit(nb).some
  private def group(d: Dimension[_], f: GroupFunction) = Group(dimensionGroupId(d))(
    "v" -> f,
    "nb" -> SumValue(1),
    "ids" -> AddFieldToSet("_id")
  ).some
  private def groupMulti(d: Dimension[_], metricDbKey: String) = Group($doc(
    "dimension" -> dimensionGroupId(d),
    "metric" -> ("$" + metricDbKey)))(
    "v" -> SumValue(1),
    "ids" -> AddFieldToSet("_id")
  ).some
  private val regroupStacked = GroupField("_id.dimension")(
    "nb" -> SumField("v"),
    "ids" -> FirstField("ids"),
    "stack" -> Push($doc(
      "metric" -> "$_id.metric",
      "v" -> "$v"))).some
  private val sliceIds = Project($doc(
    "_id" -> true,
    "v" -> true,
    "nb" -> true,
    "ids" -> $doc("$slice" -> $arr("$ids", 4))
  )).some
  private val sliceStackedIds = Project($doc(
    "_id" -> true,
    "nb" -> true,
    "stack" -> true,
    "ids" -> $doc("$slice" -> $arr("$ids", 4))
  )).some

  def apply(question: Question[_], userId: String): NonEmptyList[PipelineOperator] = {
    import question.{ dimension, metric, filters }
    val gameMatcher = combineDocs(question.filters.collect {
      case f if f.dimension.isInGame => f.matcher
    })
    def matchMoves(extraMatcher: Bdoc = $empty) =
      combineDocs(extraMatcher :: question.filters.collect {
        case f if f.dimension.isInMove => f.matcher
      }).some.filterNot(_.isEmpty) map Match
    def projectForMove = Project(BSONDocument({
      metric.dbKey :: dimension.dbKey :: filters.collect {
        case Filter(d, _) if d.isInMove => d.dbKey
{'fraction_non_alphanumeric': 0.16529311149121442, 'fraction_numerical': 0.005908878867983206, 'mean_word_length': 3.104658583280153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7481190', 'n_tokens_mistral': 2221, 'n_tokens_neox': 2155, 'n_words': 467}