<commit_before>package com.lorandszakacs.sg.model.impl

import com.lorandszakacs.sg.model._
import reactivemongo.api.DB
import reactivemongo.bson._

import scala.concurrent.{ExecutionContext, Future}

/**
  *
  * @author <NAME>, <EMAIL>, <EMAIL>
  * @since 04 Jul 2016
  *
  */
final private[model] class IndexDao(val db: DB)(implicit val ec: ExecutionContext) extends MongoDAO {
  override protected val collectionName: String = "models_index"

  private val SGIndexId = "suicide-girls-index"
  private val HopefulIndexId = "hopefuls-index"
  private val LastProcessedId = "last-processed"

  private val Names = "names"
  private val Number = "number"
  private val NeedsReindexing = "needsReindexing"

  def suicideGirlsIndex: Future[SuicideGirlIndex] = {
    val q = BSONDocument(_id -> SGIndexId)
    collection.find(q).one[SuicideGirlIndex] map {
      _.getOrElse(SuicideGirlIndex(Nil, Nil, 0))
    }
  }

  def rewriteSGIndex(names: List[ModelName]): Future[Unit] = {
    val sanitized = names.distinct.sorted
    val d = BSONDocument(
      _id -> SGIndexId,
      Names -> sanitized,
      NeedsReindexing -> sanitized,
      Number -> sanitized.length
    )
    val q = BSONDocument(_id -> SGIndexId)

    collection.update(selector = q, update = d, upsert = true) map { _ => () }
  }

  def rewriteSGIndex(i: SuicideGirlIndex): Future[SuicideGirlIndex] = {
    val sanitizedNames = i.names.distinct.sorted
    val sanitizedIndex = i.copy(
      names = sanitizedNames,
      needsReindexing = i.needsReindexing.distinct.sorted,
      number = sanitizedNames.length
    )
    val q = BSONDocument(_id -> SGIndexId)

    collection.update(selector = q, update = sanitizedIndex, upsert = true) map { _ => sanitizedIndex }
  }

  def markSGAsIndexed(name: ModelName): Future[Unit] = {
    val q = BSONDocument(_id -> SGIndexId)
    val u = BSONDocument(
      "$pull" -> BSONDocument(
        NeedsReindexing -> name
      )
    )
    collection.update(q, u, upsert = false) map (_ => ())
  }


  def hopefulIndex: Future[HopefulIndex] = {
    val q = BSONDocument(_id -> HopefulIndexId)
    collection.find(q).one[HopefulIndex] map {
      _.getOrElse(HopefulIndex(Nil, Nil, 0))
    }
  }


  def rewriteHopefulsIndex(names: List[ModelName]): Future[Unit] = {
    val sanitized = names.distinct.sorted
    val d = BSONDocument(
      _id -> HopefulIndexId,
      Names -> sanitized,
      NeedsReindexing -> sanitized,
      Number -> sanitized.length
    )

    collection.update(selector = BSONDocument(_id -> HopefulIndexId), update = d, upsert = true) map { _ => () }
  }

  def rewriteHopefulsIndex(i: HopefulIndex): Future[HopefulIndex] = {
    val sanitizedNames = i.names.distinct.sorted
    val sanitizedIndex = i.copy(
      names = sanitizedNames,
      needsReindexing = i.needsReindexing.distinct.sorted,
      number = sanitizedNames.length
    )
    val q = BSONDocument(_id -> HopefulIndexId)

    collection.update(selector = q, update = sanitizedIndex, upsert = true) map { _ => sanitizedIndex }
  }

  def markHopefulAsIndexed(name: ModelName): Future[Unit] = {
    val q = BSONDocument(_id -> HopefulIndexId)
    val u = BSONDocument(
      "$pull" -> BSONDocument(
        NeedsReindexing -> name
      )
    )
    collection.update(q, u, upsert = false) map (_ => ())
  }


  def createOrUpdateLastProcessedStatus(status: LastProcessedMarker): Future[Unit] = {
    val temp: BSONDocument = status match {
      case h: LastProcessedHopeful => lastProcessedHopefulBSON.write(h)
      case sg: LastProcessedSG => lastProcessedSuicideGirlBSON.write(sg)
    }
    val d = temp ++ (_id -> LastProcessedId)

    val s: BSONDocument = BSONDocument(_id -> LastProcessedId)

    collection.update(selector = s, update = d, upsert = true) map { _ => () }
  }

  def lastProcessedStatus: Future[Option[LastProcessedMarker]] = {
    val q: BSONDocument = BSONDocument(_id -> LastProcessedId)
    val b = collection.find(q).one[BSONDocument]
    val r: Future[Option[LastProcessedMarker]] = b map {
      case None => None
      case Some(bson) =>
        bson.getAs[String]("className") match {
          case None => throw new AssertionError("lastProcessedMarker.className field should exist")
          case Some(x) => x match {
            case "LastProcessedHopeful" => Option(lastProcessedHopefulBSON.read(bson))
            case "LastProcessedSG" => Option(lastProcessedHopefulBSON.read(bson))
          }
        }
    }
    r
  }
}
<commit_msg>Fix MongoDB write error<commit_after>package com.lorandszakacs.sg.model.impl

import com.lorandszakacs.sg.model._
import reactivemongo.api.DB
import reactivemongo.bson._

import scala.concurrent.{ExecutionContext, Future}

/**
  *
  * @author <NAME>, <EMAIL>, <EMAIL>
  * @since 04 Jul 2016
  *
  */
final private[model] class IndexDao(val db: DB)(implicit val ec: ExecutionContext) extends MongoDAO {
  override protected val collectionName: String = "models_index"

  private val SGIndexId = "suicide-girls-index"
  private val HopefulIndexId = "hopefuls-index"
  private val LastProcessedId = "last-processed"

  private val Names = "names"
  private val Number = "number"
  private val NeedsReindexing = "needsReindexing"

  def suicideGirlsIndex: Future[SuicideGirlIndex] = {
    val q = BSONDocument(_id -> SGIndexId)
    collection.find(q).one[SuicideGirlIndex] map {
      _.getOrElse(SuicideGirlIndex(Nil, Nil, 0))
    }
  }

  def rewriteSGIndex(names: List[ModelName]): Future[Unit] = {
    val sanitized = names.distinct.sorted
    val d = BSONDocument(
      _id -> SGIndexId,
      Names -> sanitized,
      NeedsReindexing -> sanitized,
      Number -> sanitized.length
    )
    val q = BSONDocument(_id -> SGIndexId)

    collection.update(selector = q, update = d, upsert = true) map { _ => () }
  }

  def rewriteSGIndex(i: SuicideGirlIndex): Future[SuicideGirlIndex] = {
    val sanitizedNames = i.names.distinct.sorted
    val sanitizedIndex = i.copy(
      names = sanitizedNames,
      needsReindexing = i.needsReindexing.distinct.sorted,
      number = sanitizedNames.length
    )
    val q = BSONDocument(_id -> SGIndexId)

    collection.update(selector = q, update = sanitizedIndex, upsert = true) map { _ => sanitizedIndex }
  }

  def markSGAsIndexed(name: ModelName): Future[Unit] = {
    val q = BSONDocument(_id -> SGIndexId)
    val u = BSONDocument(
      "$pull" -> BSONDocument(
        NeedsReindexing -> name
      )
    )
    collection.update(q, u, upsert = false) map (_ => ())
  }


  def hopefulIndex: Future[HopefulIndex] = {
    val q = BSONDocument(_id -> HopefulIndexId)
    collection.find(q).one[HopefulIndex] map {
      _.getOrElse(HopefulIndex(Nil, Nil, 0))
    }
  }


  def rewriteHopefulsIndex(names: List[ModelName]): Future[Unit] = {
    val sanitized = names.distinct.sorted
    val d = BSONDocument(
      _id -> HopefulIndexId,
      Names -> sanitized,
      NeedsReindexing -> sanitized,
      Number -> sanitized.length
    )

    collection.update(selector = BSONDocument(_id -> HopefulIndexId), update = d, upsert = true) map { _ => () }
  }

  def rewriteHopefulsIndex(i: HopefulIndex): Future[HopefulIndex] = {
    val sanitizedNames = i.names.distinct.sorted
    val sanitizedIndex = i.copy(
      names = sanitizedNames,
      needsReindexing = i.needsReindexing.distinct.sorted,
      number = sanitizedNames.length
    )
    val q = BSONDocument(_id -> HopefulIndexId)

    collection.update(selector = q, update = sanitizedIndex, upsert = true) map { _ => sanitizedIndex }
  }

  def markHopefulAsIndexed(name: ModelName): Future[Unit] = {
    val q = BSONDocument(_id -> HopefulIndexId)
    val u = BSONDocument(
      "$pull" -> BSONDocument(
        NeedsReindexing -> name
      )
    )
    collection.update(q, u, upsert = false) map (_ => ())
  }


  def createOrUpdateLastProcessedStatus(status: LastProcessedMarker): Future[Unit] = {
    val temp: BSONDocument = status match {
      case h: LastProcessedHopeful => lastProcessedHopefulBSON.write(h)
      case sg: LastProcessedSG => lastProcessedSuicideGirlBSON.write(sg)
    }
    val d = temp ++ (_id -> LastProcessedId)

    val s: BSONDocument = BSONDocument(_id -> LastProcessedId)

    collection.update(selector = s, update = d, upsert = true) map { _ => () }
  }

  def lastProcessedStatus: Future[Option[LastProcessedMarker]] = {
    val q: BSONDocument = BSONDocument(_id -> LastProcessedId)
    val b = collection.find(q).one[BSONDocument]
    val r: Future[Option[LastProcessedMarker]] = b map {
      case None => None
      case Some(bson) =>
        bson.getAs[String]("className") match {
          case None => throw new AssertionError("lastProcessedMarker.className field should exist")
          case Some(x) => x match {
            case "LastProcessedHopeful" => Option(lastProcessedHopefulBSON.read(bson))
            case "LastProcessedSG" => Option(lastProcessedSuicideGirlBSON.read(bson))
          }
        }
    }
    r
  }
}

{'fraction_non_alphanumeric': 0.1141134435015632, 'fraction_numerical': 0.001786511835640911, 'mean_word_length': 3.221017907634307, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7394120', 'n_tokens_mistral': 3051, 'n_tokens_neox': 2869, 'n_words': 625}