<commit_before>package org.intracer.wmua

import com.typesafe.config.ConfigFactory
import controllers.Global.commons
import controllers.GlobalRefactor
import db.scalikejdbc._
import org.intracer.wmua.cmd.{DistributeImages, ImageWithRatingSeqFilter}
import org.scalawiki.MwBot
import org.scalawiki.dto.{Image, Namespace}
import org.scalawiki.wlx.{ImageDB, ListFiller, MonumentDB}
import org.scalawiki.wlx.dto.{Contest, Monument}
import org.scalawiki.wlx.query.MonumentQuery
import play.api.Play
import scalikejdbc.{ConnectionPool, GlobalSettings, LoggingSQLAndTimeSettings}
import spray.util.pimpFuture

import scala.concurrent.Await
import scala.concurrent.ExecutionContext.Implicits.global
import scala.io.Source

object Tools {

  def main(args: Array[String]) {
    Class.forName("com.mysql.jdbc.Driver")
    val url: String = "jdbc:mysql://localhost/wlxjury?autoReconnect=true&autoReconnectForPools=true&useUnicode=true&characterEncoding=UTF-8"
    //"jdbc:mysql://jury.wikilovesearth.org.ua/wlxjury"
    println(s"URL:" + url)

    val config = ConfigFactory.load()
    val user = config.getString("db.default.username")
    val password = config.getString("db.default.password")

    ConnectionPool.singleton(url, user, password)

    GlobalSettings.loggingSQLAndTime = LoggingSQLAndTimeSettings(
      enabled = true,
      singleLineMode = false,
      printUnprocessedStackTrace = false,
      stackTraceDepth = 15,
      logLevel = 'info,
      warningEnabled = false,
      warningThresholdMillis = 3000L,
      warningLogLevel = 'warn
    )

//    addMonuments()
//    newlyPictured()
    //    addCriteria()
    //    fetchMonumentDb()
    byCity()
    //fillLists()
  }

  def fetchMonumentDb() = {
    val commons = MwBot.fromHost(controllers.Global.COMMONS_WIKIMEDIA_ORG)

    val contest = Contest.WLMUkraine(2017)
    new GlobalRefactor(commons).updateLists(contest)
  }

  def distributeImages(round: Round,
                       jurors: Seq[User],
                       prevRound: Option[Round]): Unit = {
    val images = getFilteredImages(round, jurors, prevRound)

    distributeImages(round, jurors, images)
  }

  def getFilteredImages(round: Round, jurors: Seq[User], prevRound: Option[Round]): Seq[Image] = {
    getFilteredImages(round, jurors, prevRound, selectedAtLeast = round.prevSelectedBy,
      selectMinAvgRating = round.prevMinAvgRate,
      sourceCategory = round.category,
      includeCategory = round.categoryClause.map(_ > 0),
      includeRegionIds = round.regionIds.toSet,
      includeMonumentIds = round.monumentIds.toSet
    )
  }

  def distributeImages(round: Round, jurors: Seq[User], images: Seq[Image]): Unit = {
    DistributeImages(round, images, jurors).apply()
  }

  def getFilteredImages(
                         round: Round,
                         jurors: Seq[User],
                         prevRound: Option[Round],
                         includeRegionIds: Set[String] = Set.empty,
                         excludeRegionIds: Set[String] = Set.empty,
                         includeMonumentIds: Set[String] = Set.empty,
                         includePageIds: Set[Long] = Set.empty,
                         excludePageIds: Set[Long] = Set.empty,
                         includeTitles: Set[String] = Set.empty,
                         excludeTitles: Set[String] = Set.empty,
                         selectMinAvgRating: Option[Int] = None,
                         selectTopByRating: Option[Int] = None,
                         selectedAtLeast: Option[Int] = None,
                         includeJurorId: Set[Long] = Set.empty,
                         excludeJurorId: Set[Long] = Set.empty,
                         sourceCategory: Option[String] = None,
                         includeCategory: Option[Boolean] = None
                       ): Seq[Image] = {

    val catIds = sourceCategory.map { category =>
      val pages = commons.page(category).imageInfoByGenerator("categorymembers", "cm", Set(Namespace.FILE)).await
      pages.flatMap(_.id)
    }

    val (includeFromCats, excludeFromCats) = (
      for (ids <- catIds;
           include <- includeCategory)
        yield
          if (include)
            (ids, Seq.empty)
          else (Seq.empty, ids)
      ).getOrElse(Seq.empty, Seq.empty)


    val currentSelection = ImageJdbc.byRoundMerged(round.id.get).filter(iwr => iwr.selection.nonEmpty).toSet
    val existingImageIds = currentSelection.map(_.pageId)
    val existingJurorIds = currentSelection.flatMap(_.jurors)
    val mpxAtLeast = round.minMpx
    val sizeAtLeast = round.minImageSize.map(_ * 1024 * 1024)

    val contest = ContestJuryJdbc.findById(round.contest).get
    val imagesAll = prevRound.fold[Seq[ImageWithRating]](
      ImageJdbc.findByContest(contest).map(i =>
        new ImageWithRating(i, Seq.empty)
      )
    )(r =>
      ImageJdbc.byRoundMerged(r.id.get)
    )
    println("Total images: " + imagesAll.size)

    val funGens = ImageWithRatingSeqFilter.funGenerators(prevRound,
      includeRegionIds = includeRegionIds,
      excludeRegionIds = excludeRegionIds,
      includeMonumentIds = includeMonumentIds,
      includePageIds = includePageIds ++ includeFromCats.toSet,
      excludePageIds = excludePageIds ++ existingImageIds ++ excludeFromCats.toSet,
      includeTitles = includeTitles,
      excludeTitles = excludeTitles,
      includeJurorId = includeJurorId,
      excludeJurorId = excludeJurorId /*++ existingJurorIds*/ ,<commit_msg>fix ImageDistributorSpec<commit_after>package org.intracer.wmua

import com.typesafe.config.ConfigFactory
import controllers.Global.commons
import controllers.GlobalRefactor
import db.scalikejdbc._
import org.intracer.wmua.cmd.{DistributeImages, ImageWithRatingSeqFilter}
import org.scalawiki.MwBot
import org.scalawiki.dto.{Image, Namespace}
import org.scalawiki.wlx.{ImageDB, ListFiller, MonumentDB}
import org.scalawiki.wlx.dto.{Contest, Monument}
import org.scalawiki.wlx.query.MonumentQuery
import play.api.Play
import scalikejdbc.{ConnectionPool, GlobalSettings, LoggingSQLAndTimeSettings}
import spray.util.pimpFuture

import scala.concurrent.Await
import scala.concurrent.ExecutionContext.Implicits.global
import scala.io.Source

object Tools {

  def main(args: Array[String]) {
    Class.forName("com.mysql.jdbc.Driver")
    val url: String = "jdbc:mysql://localhost/wlxjury?autoReconnect=true&autoReconnectForPools=true&useUnicode=true&characterEncoding=UTF-8"
    //"jdbc:mysql://jury.wikilovesearth.org.ua/wlxjury"
    println(s"URL:" + url)

    val config = ConfigFactory.load()
    val user = config.getString("db.default.username")
    val password = config.getString("db.default.password")

    ConnectionPool.singleton(url, user, password)

    GlobalSettings.loggingSQLAndTime = LoggingSQLAndTimeSettings(
      enabled = true,
      singleLineMode = false,
      printUnprocessedStackTrace = false,
      stackTraceDepth = 15,
      logLevel = 'info,
      warningEnabled = false,
      warningThresholdMillis = 3000L,
      warningLogLevel = 'warn
    )

//    addMonuments()
//    newlyPictured()
    //    addCriteria()
    //    fetchMonumentDb()
    byCity()
    //fillLists()
  }

  def fetchMonumentDb() = {
    val commons = MwBot.fromHost(controllers.Global.COMMONS_WIKIMEDIA_ORG)

    val contest = Contest.WLMUkraine(2017)
    new GlobalRefactor(commons).updateLists(contest)
  }

  def distributeImages(round: Round,
                       jurors: Seq[User],
                       prevRound: Option[Round]): Unit = {
    val images = getFilteredImages(round, jurors, prevRound)

    distributeImages(round, jurors, images)
  }

  def getFilteredImages(round: Round, jurors: Seq[User], prevRound: Option[Round]): Seq[Image] = {
    getFilteredImages(round, jurors, prevRound, selectedAtLeast = round.prevSelectedBy,
      selectMinAvgRating = round.prevMinAvgRate,
      sourceCategory = round.category,
      includeCategory = round.categoryClause.map(_ > 0),
      includeRegionIds = round.regionIds.toSet,
      includeMonumentIds = round.monumentIds.toSet
    )
  }

  def distributeImages(round: Round, jurors: Seq[User], images: Seq[Image]): Unit = {
    DistributeImages(round, images, jurors).apply()
  }

  def getFilteredImages(
                         round: Round,
                         jurors: Seq[User],
                         prevRound: Option[Round],
                         includeRegionIds: Set[String] = Set.empty,
                         excludeRegionIds: Set[String] = Set.empty,
                         includeMonumentIds: Set[String] = Set.empty,
                         includePageIds: Set[Long] = Set.empty,
                         excludePageIds: Set[Long] = Set.empty,
                         includeTitles: Set[String] = Set.empty,
                         excludeTitles: Set[String] = Set.empty,
                         selectMinAvgRating: Option[Int] = None,
                         selectTopByRating: Option[Int] = None,
                         selectedAtLeast: Option[Int] = None,
                         includeJurorId: Set[Long] = Set.empty,
                         excludeJurorId: Set[Long] = Set.empty,
                         sourceCategory: Option[String] = None,
                         includeCategory: Option[Boolean] = None
                       ): Seq[Image] = {

    val catIds = sourceCategory.map { category =>
      val pages = commons.page(category).imageInfoByGenerator("categorymembers", "cm", Set(Namespace.FILE)).await
      pages.flatMap(_.id)
    }

    val (includeFromCats, excludeFromCats) = (
      for (ids <- catIds;
           include <- includeCategory)
        yield
          if (include)
            (ids, Seq.empty)
          else (Seq.empty, ids)
      ).getOrElse(Seq.empty, Seq.empty)


    val currentSelection = ImageJdbc.byRoundMerged(round.id.get).filter(iwr => iwr.selection.nonEmpty).toSet
    val existingImageIds = currentSelection.map(_.pageId)
    val existingJurorIds = currentSelection.flatMap(_.jurors)
    val mpxAtLeast = round.minMpx
    val sizeAtLeast = round.minImageSize.map(_ * 1024 * 1024)

    val contest = ContestJuryJdbc.findById(round.contest).get
    val imagesAll = prevRound.fold[Seq[ImageWithRating]](
      ImageJdbc.findByContest(contest).map(i =>
        new ImageWithRating(i, Seq.empty)
      )
    )(r =>
      ImageJdbc.byRoundMerged(r.id.get, rated = selectedAtLeast.map(_ => true))
    )
    println("Total images: " + imagesAll.size)

    val funGens = ImageWithRatingSeqFilter.funGenerators(prevRound,
      includeRegionIds = includeRegionIds,
      excludeRegionIds = excludeRegionIds,
      includeMonumentIds = includeMonumentIds,
      includePageIds = includePageIds ++ includeFromCats.toSet,
      excludePageIds = excludePageIds ++ existingImageIds ++ excludeFromCats.toSet,
      includeTitles = includeTitles,
      excludeTitles = excludeTitles,
      includeJurorId = includeJurorId,
      excludeJurorId = excludeJurorId /*++ existingJurorIds*/ ,
{'fraction_non_alphanumeric': 0.10177353342428376, 'fraction_numerical': 0.0036380172805820826, 'mean_word_length': 2.752901023890785, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3376832', 'n_tokens_mistral': 3394, 'n_tokens_neox': 3204, 'n_words': 580}