<commit_before>package lila.analyse

import com.github.nscala_time.time.Imports._
import org.joda.time.DateTime
import play.api.libs.json.Json
import play.modules.reactivemongo.json.ImplicitBSONHandlers.JsObjectWriter

import lila.db.api._
import lila.db.Implicits._
import tube.analysisTube

object AnalysisRepo {

  type ID = String

  def done(id: ID, a: Analysis) = $update(
    $select(id),
    $set(Json.obj(
      "done" -> true,
      "data" -> Info.encodeList(a.infos)
    ))
  )

  def progress(id: ID, userId: ID) = $update(
    $select(id),
    $set(Json.obj(
      "uid" -> userId,
      "done" -> false,
      "date" -> $date(DateTime.now)
    )) ++ $unset("old", "data"),
    upsert = true)

  def doneById(id: ID): Fu[Option[Analysis]] =
    $find.one($select(id) ++ Json.obj("done" -> true))

  def doneByIds(ids: Seq[ID]): Fu[Seq[Option[Analysis]]] = $find byOrderedIds ids map { as =>
    ids.map { id =>
      as find { a => a.id == id && a.done }
    }
  }

  def doneByIdNotOld(id: ID): Fu[Option[Analysis]] =
    $find.one($select(id) ++ Json.obj("done" -> true, "old" -> $exists(false)))

  def isDone(id: ID): Fu[Boolean] =
    $count.exists($select(id) ++ Json.obj("done" -> true))

  def userInProgress(uid: ID): Fu[Option[String]] = $primitive.one(
    Json.obj(
      "uid" -> uid,
      "done" -> false,
      "date" -> $gt($date(DateTime.now - 20.minutes))),
    "_id")(_.asOpt[String])

  def getOrRemoveStaled(id: ID): Fu[Option[Analysis]] =
    $find.one($select(id) ++ Json.obj(
      "done" -> false,
      "date" -> $lt($date(DateTime.now - 20.minutes)))) flatMap {
      _.fold($find byId id) { staled => $remove byId id inject none }
    }

  def recent(nb: Int): Fu[List[Analysis]] =
    $find($query(Json.obj("done" -> true)) sort $sort.desc("date"), nb)

  def skipping(skip: Int, nb: Int): Fu[List[Analysis]] =
    $find($query(Json.obj("done" -> true)) skip skip, nb)
<commit_msg>fix detection of stalled analysis<commit_after>package lila.analyse

import com.github.nscala_time.time.Imports._
import org.joda.time.DateTime
import play.api.libs.json.Json
import play.modules.reactivemongo.json.ImplicitBSONHandlers.JsObjectWriter

import lila.db.api._
import lila.db.Implicits._
import tube.analysisTube

object AnalysisRepo {

  type ID = String

  def done(id: ID, a: Analysis) = $update(
    $select(id),
    $set(Json.obj(
      "done" -> true,
      "data" -> Info.encodeList(a.infos)
    ))
  )

  def progress(id: ID, userId: ID) = $update(
    $select(id),
    $set(Json.obj(
      "uid" -> userId,
      "done" -> false,
      "date" -> $date(DateTime.now)
    )) ++ $unset("old", "data"),
    upsert = true)

  def doneById(id: ID): Fu[Option[Analysis]] =
    $find.one($select(id) ++ Json.obj("done" -> true))

  def doneByIds(ids: Seq[ID]): Fu[Seq[Option[Analysis]]] = $find byOrderedIds ids map { as =>
    ids.map { id =>
      as find { a => a.id == id && a.done }
    }
  }

  def doneByIdNotOld(id: ID): Fu[Option[Analysis]] =
    $find.one($select(id) ++ Json.obj("done" -> true, "old" -> $exists(false)))

  def isDone(id: ID): Fu[Boolean] =
    $count.exists($select(id) ++ Json.obj("done" -> true))

  def userInProgress(uid: ID): Fu[Option[String]] = $primitive.one(
    Json.obj(
      "uid" -> uid,
      "done" -> false,
      "date" -> $gt($date(DateTime.now - 20.minutes))),
    "_id")(_.asOpt[String])

  def getOrRemoveStaled(id: ID): Fu[Option[Analysis]] =
    $find.one(
      $select(id) ++
        Json.obj("date" -> $lt($date(DateTime.now - 20.minutes))) ++
        $or(List(Json.obj("done" -> false), Json.obj("data" -> "")))
    ) flatMap {
        case None          => $find byId id
        case Some(stalled) => $remove byId id inject none
      }

  def recent(nb: Int): Fu[List[Analysis]] =
    $find($query(Json.obj("done" -> true)) sort $sort.desc("date"), nb)

  def skipping(skip: Int, nb: Int): Fu[List[Analysis]] =
    $find($query(Json.obj("done" -> true)) skip skip, nb)

{'fraction_non_alphanumeric': 0.19424823410696265, 'fraction_numerical': 0.0020181634712411706, 'mean_word_length': 3.1345151199165797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5926135', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1444, 'n_words': 300}