<commit_before>package io.prediction.engines.itemrank

import io.prediction.{ Evaluator, BaseEvaluationResults }
import io.prediction.{ EvaluatorFactory }
import io.prediction.core.AbstractEvaluator
import io.prediction.storage.Config
import io.prediction.storage.{ Item, U2IAction, User, ItemSet }

import scala.collection.mutable.ArrayBuffer
import com.github.nscala_time.time.Imports._
import scala.math.BigDecimal

object ItemRankEvaluator extends EvaluatorFactory {

  val config = new Config
  val usersDb = config.getAppdataUsers
  val itemsDb = config.getAppdataItems
  val u2iDb = config.getAppdataU2IActions
  val itemSetsDb = config.getAppdataItemSets

  override def apply(): AbstractEvaluator = {
    new ItemRankEvaluator
  }
}

class ItemRankEvaluator
  extends Evaluator[EvalParams, TrainDataPrepParams, EvalDataPrepParams,
      TrainigData, Feature, Prediction, Actual, EvalUnit, EvalResults] {

  final val CONFLICT_LATEST: String = "latest"
  final val CONFLICT_HIGHEST: String = "highest"
  final val CONFLICT_LOWEST: String = "lowest"

  val usersDb = ItemRankEvaluator.usersDb
  val itemsDb = ItemRankEvaluator.itemsDb
  val u2iDb = ItemRankEvaluator.u2iDb
  val itemSetsDb = ItemRankEvaluator.itemSetsDb

  // Data generation
  override def getParamsSet(params: EvalParams): Seq[(TrainDataPrepParams,
    EvalDataPrepParams)] = {

    var testStart = params.testStart
    val testStartSeq = ArrayBuffer[DateTime]()
    val period = Period.hours(params.hours)
    while (testStart < params.testUntil) {
      testStartSeq += testStart
      testStart = testStart + period
    }

    val paramSeq = testStartSeq.toList.map { ts =>
      val trainingP = new TrainDataPrepParams(
        appid = params.appid,
        itypes = params.itypes,
        actions = params.actions,
        conflict = params.conflict,
        seenActions = params.seenActions,
        startUntil = Some((params.trainStart, ts))
      )
      val evalP = new EvalDataPrepParams(
        appid = params.appid,
        itypes = params.itypes,
        startUntil = (ts, ts + period),
        goal = params.goal
      )
      (trainingP, evalP)
    }
    paramSeq
  }


  override def prepareTraining(params: TrainDataPrepParams): TrainigData = {
    val usersMap: Map[String, Int] = usersDb.getByAppid(params.appid)
      .map(_.id).zipWithIndex
      .map { case (uid, index) => (uid, index + 1) }.toMap

    val itemsMap: Map[String, (ItemTD, Int)] = params.itypes.map { itypes =>
      itemsDb.getByAppidAndItypes(params.appid, itypes.toSeq)
    }.getOrElse {
      itemsDb.getByAppid(params.appid)
    }.zipWithIndex.map {
      case (item, index) =>
        val itemTD = new ItemTD(
          iid = item.id,
          itypes = item.itypes,
          starttime = item.starttime.map[Long](_.getMillis()),
          endtime = item.endtime.map[Long](_.getMillis()),
          inactive = item.inactive.getOrElse(false)
        )
        (item.id -> (itemTD, index + 1))
    }.toMap

    // NOTE: only contain valid items (eg. valid starttime and endtime,
    // inactive=false)
    /*
    val possibleItems: Set[Int] = itemsMap.filter {
      case (iid, (itemTD, iindex)) =>
        val validTime = itemTimeFilter(true,
          itemTD.starttime, itemTD.endtime,
          params.recommendationTime)

        validTime && (!itemTD.inactive)
    }.map {
      case (iid, (itemTD, iindex)) => iindex
    }.toSet
    */

    val u2iActions = params.startUntil.map{ startUntil =>
      u2iDb.getByAppidAndTime(params.appid, startUntil._1,
        startUntil._2).toSeq
    }.getOrElse{
      u2iDb.getAllByAppid(params.appid).toSeq
    }

    val u2iRatings = u2iActions
      .filter { u2i =>
        val validAction = params.actions.contains(u2i.action)
        val validUser = usersMap.contains(u2i.uid)
        val validItem = itemsMap.contains(u2i.iid)
        (validAction && validUser && validItem)
      }.map { u2i =>
        val rating = params.actions(u2i.action).getOrElse(u2i.v.getOrElse(0))

        new RatingTD(
          uindex = usersMap(u2i.uid), // map to index
          iindex = itemsMap(u2i.iid)._2,
          rating = rating,
          t = u2i.t.getMillis
        )
      }

    val ratingReduced = u2iRatings.groupBy(x => (x.iindex, x.uindex))
      .mapValues { v =>
        v.reduce { (a, b) =>
          resolveConflict(a, b, params.conflict)
        }
      }.values
      .toSeq

    /* write u2i seen */
    val u2iSeen = u2iActions
      .filter { u2i =>
        val validAction = params.seenActions.map(seenActions =>
          seenActions.contains(u2i.action)).getOrElse(
          // same as training actions if seenActions is not defined
          params.actions.contains(u2i.action))

        val validUser = usersMap.contains(u2i.uid)
        val validItem = itemsMap.contains(u2i.iid)
        (validAction && validUser && validItem)
      }
      // convert to index
      .map { u2i => (usersMap(u2i.uid), itemsMap(u2i.iid)._2) }
      .toSet

    new TrainigData(
      users = usersMap.map { case (k, v) => (v, k) },<commit_msg>Need to concrete Seq to avoid serialization error<commit_after>package io.prediction.engines.itemrank

import io.prediction.{ Evaluator, BaseEvaluationResults }
import io.prediction.{ EvaluatorFactory }
import io.prediction.core.AbstractEvaluator
import io.prediction.storage.Config
import io.prediction.storage.{ Item, U2IAction, User, ItemSet }

import scala.collection.mutable.ArrayBuffer
import com.github.nscala_time.time.Imports._
import scala.math.BigDecimal

object ItemRankEvaluator extends EvaluatorFactory {

  val config = new Config
  val usersDb = config.getAppdataUsers
  val itemsDb = config.getAppdataItems
  val u2iDb = config.getAppdataU2IActions
  val itemSetsDb = config.getAppdataItemSets

  override def apply(): AbstractEvaluator = {
    new ItemRankEvaluator
  }
}

class ItemRankEvaluator
  extends Evaluator[EvalParams, TrainDataPrepParams, EvalDataPrepParams,
      TrainigData, Feature, Prediction, Actual, EvalUnit, EvalResults] {

  final val CONFLICT_LATEST: String = "latest"
  final val CONFLICT_HIGHEST: String = "highest"
  final val CONFLICT_LOWEST: String = "lowest"

  val usersDb = ItemRankEvaluator.usersDb
  val itemsDb = ItemRankEvaluator.itemsDb
  val u2iDb = ItemRankEvaluator.u2iDb
  val itemSetsDb = ItemRankEvaluator.itemSetsDb

  // Data generation
  override def getParamsSet(params: EvalParams): Seq[(TrainDataPrepParams,
    EvalDataPrepParams)] = {

    var testStart = params.testStart
    val testStartSeq = ArrayBuffer[DateTime]()
    val period = Period.hours(params.hours)

    while (testStart < params.testUntil) {
      testStartSeq += testStart
      testStart = testStart + period
    }
    //println(testStartSeq)

    val paramSeq = testStartSeq.toList.map { ts =>
      val trainingP = new TrainDataPrepParams(
        appid = params.appid,
        itypes = params.itypes,
        actions = params.actions,
        conflict = params.conflict,
        seenActions = params.seenActions,
        startUntil = Some((params.trainStart, ts))
      )
      val evalP = new EvalDataPrepParams(
        appid = params.appid,
        itypes = params.itypes,
        startUntil = (ts, ts + period),
        goal = params.goal
      )
      (trainingP, evalP)
    }
    paramSeq
  }


  override def prepareTraining(params: TrainDataPrepParams): TrainigData = {
    val usersMap: Map[String, Int] = usersDb.getByAppid(params.appid)
      .map(_.id).zipWithIndex
      .map { case (uid, index) => (uid, index + 1) }.toMap

    val itemsMap: Map[String, (ItemTD, Int)] = params.itypes.map { itypes =>
      itemsDb.getByAppidAndItypes(params.appid, itypes.toSeq)
    }.getOrElse {
      itemsDb.getByAppid(params.appid)
    }.zipWithIndex.map {
      case (item, index) =>
        val itemTD = new ItemTD(
          iid = item.id,
          itypes = item.itypes,
          starttime = item.starttime.map[Long](_.getMillis()),
          endtime = item.endtime.map[Long](_.getMillis()),
          inactive = item.inactive.getOrElse(false)
        )
        (item.id -> (itemTD, index + 1))
    }.toMap

    // NOTE: only contain valid items (eg. valid starttime and endtime,
    // inactive=false)
    /*
    val possibleItems: Set[Int] = itemsMap.filter {
      case (iid, (itemTD, iindex)) =>
        val validTime = itemTimeFilter(true,
          itemTD.starttime, itemTD.endtime,
          params.recommendationTime)

        validTime && (!itemTD.inactive)
    }.map {
      case (iid, (itemTD, iindex)) => iindex
    }.toSet
    */

    val u2iActions = params.startUntil.map{ startUntil =>
      u2iDb.getByAppidAndTime(params.appid, startUntil._1,
        startUntil._2).toSeq
    }.getOrElse{
      u2iDb.getAllByAppid(params.appid).toSeq
    }

    val u2iRatings = u2iActions
      .filter { u2i =>
        val validAction = params.actions.contains(u2i.action)
        val validUser = usersMap.contains(u2i.uid)
        val validItem = itemsMap.contains(u2i.iid)
        (validAction && validUser && validItem)
      }.map { u2i =>
        val rating = params.actions(u2i.action).getOrElse(u2i.v.getOrElse(0))

        new RatingTD(
          uindex = usersMap(u2i.uid), // map to index
          iindex = itemsMap(u2i.iid)._2,
          rating = rating,
          t = u2i.t.getMillis
        )
      }

    val ratingReduced = u2iRatings.groupBy(x => (x.iindex, x.uindex))
      .mapValues { v =>
        v.reduce { (a, b) =>
          resolveConflict(a, b, params.conflict)
        }
      }.values
      .toList

    /* write u2i seen */
    val u2iSeen = u2iActions
      .filter { u2i =>
        val validAction = params.seenActions.map(seenActions =>
          seenActions.contains(u2i.action)).getOrElse(
          // same as training actions if seenActions is not defined
          params.actions.contains(u2i.action))

        val validUser = usersMap.contains(u2i.uid)
        val validItem = itemsMap.contains(u2i.iid)
        (validAction && validUser && validItem)
      }
      // convert to index
      .map { u2i => (usersMap(u2i.uid), itemsMap(u2i.iid)._2) }
      .toSet

    new TrainigData(
      users = usersMap.map { case (k, v) => (v, k) },
{'fraction_non_alphanumeric': 0.10742340926944226, 'fraction_numerical': 0.007659073055773763, 'mean_word_length': 2.6768953068592056, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5827674', 'n_tokens_mistral': 3412, 'n_tokens_neox': 3312, 'n_words': 702}