<commit_before>package com.criteo.slab.app

import java.util.concurrent.{Executors, TimeUnit}

import com.criteo.slab.core.{Board, BoardView, Status}
import org.joda.time.DateTime
import org.slf4j.LoggerFactory

import scala.collection.concurrent.TrieMap
import scala.concurrent.ExecutionContext

class StateService(
                    val boards: Seq[Board],
                    intervalSeconds: Int
                  )(implicit ec: ExecutionContext) {

  import StateService._

  private val logger = LoggerFactory.getLogger(this.getClass)

  // history in (board name -> (timestamp -> view tree))
  private val history = TrieMap.empty[String, TrieMap[Long, BoardView]]
  // current state in (board name -> view tree)
  private val current = TrieMap.empty[String, BoardView]
  // stats in (board name -> (timestamp -> Stats))
  private val stats = TrieMap.empty[String, TrieMap[Long, Stats]]

  private lazy val scheduler = Executors.newSingleThreadScheduledExecutor()

  def start(): Unit = {
    logger.info(s"started, checking interval: $intervalSeconds seconds")
    scheduler.scheduleAtFixedRate(poller, 0, intervalSeconds, TimeUnit.SECONDS)
    loadHistory()
    loadStats()
  }

  // Load the history of last 24 hours
  private def loadHistory(): Unit = {
    boards foreach { board =>
      board
        .fetchHistory(DateTime.now.minusDays(1), DateTime.now)
        .foreach { entries =>
          history.get(board.title) match {
            case Some(value) => value ++= entries
            case None =>
              history += board.title -> (TrieMap.empty ++= entries)
          }
        }
    }
  }

  // Load the stats of last 7 days
  private def loadStats(): Unit = {
    boards foreach { board =>
      logger.info(s"loading stats for ${board.title}")
      board
        .fetchHistory(DateTime.now.minusDays(7), DateTime.now)
        .map(getStatsByDay)
        .foreach { newStats =>
          stats.get(board.title) match {
            case Some(value) => value ++= newStats
            case None =>
              stats += board.title -> (TrieMap.empty ++= newStats)
          }
        }
    }
  }

  def getCurrent(name: String): Option[BoardView] = {
    current.get(name)
  }

  def getHistory(name: String): Option[Map[Long, BoardView]] = {
    history.get(name).map(_.toMap)
  }

  def getStats(name: String): Option[Map[Long, Stats]] = {
    stats.get(name).map(_.toMap)
  }

  val poller: Runnable = () => {
    val checkTime = DateTime.now().withMillisOfSecond(0).withSecondOfMinute(0)
    logger.info(s"updating ${checkTime.toString()} (${checkTime.getMillis})")
    boards foreach { board =>
      board
        .apply(None)
        .foreach { viewTree =>
          // Update the current view tree of the board
          current += board.title -> viewTree
          // Update history cache
          val records = history.getOrElseUpdate(board.title, TrieMap(checkTime.getMillis -> viewTree)) += checkTime.getMillis -> viewTree
          // evict old entries
          val obsoleted = records.keys.filter(_ < checkTime.minusDays(1).getMillis)
          if (obsoleted.size > 0) {
            logger.info(s"evicted ${obsoleted.size} history entries")
            records --= obsoleted
          }
          logger.info(s"history cache updated, new size: ${records.size}")
          // Update stats
          stats
            .getOrElseUpdate(board.title, TrieMap(checkTime.getMillis -> Stats(0, 0, 0, 0)))
            .get(checkTime.getMillis)
            .map(updateStatsWithStatus(_, viewTree.status))
        }
    }
  }

  sys.addShutdownHook {
    logger.info(s"shutting down StateService")
    scheduler.shutdown()
  }
}

object StateService {
  def getStatsByDay(history: Map[Long, BoardView]): Map[Long, Stats] = {
    history
      .groupBy { case (ts, _) =>
        ts - ts % 86400000 // normalize to the start of the day
      }
      .mapValues { entries =><commit_msg>Get rid of SAM in order to support cross compiling<commit_after>package com.criteo.slab.app

import java.util.concurrent.{Executors, TimeUnit}

import com.criteo.slab.core.{Board, BoardView, Status}
import org.joda.time.DateTime
import org.slf4j.LoggerFactory

import scala.collection.concurrent.TrieMap
import scala.concurrent.ExecutionContext

class StateService(
                    val boards: Seq[Board],
                    intervalSeconds: Int
                  )(implicit ec: ExecutionContext) {

  import StateService._

  private val logger = LoggerFactory.getLogger(this.getClass)

  // history in (board name -> (timestamp -> view tree))
  private val history = TrieMap.empty[String, TrieMap[Long, BoardView]]
  // current state in (board name -> view tree)
  private val current = TrieMap.empty[String, BoardView]
  // stats in (board name -> (timestamp -> Stats))
  private val stats = TrieMap.empty[String, TrieMap[Long, Stats]]

  private lazy val scheduler = Executors.newSingleThreadScheduledExecutor()

  def start(): Unit = {
    logger.info(s"started, checking interval: $intervalSeconds seconds")
    scheduler.scheduleAtFixedRate(Poller, 0, intervalSeconds, TimeUnit.SECONDS)
    loadHistory()
    loadStats()
  }

  // Load the history of last 24 hours
  private def loadHistory(): Unit = {
    boards foreach { board =>
      board
        .fetchHistory(DateTime.now.minusDays(1), DateTime.now)
        .foreach { entries =>
          history.get(board.title) match {
            case Some(value) => value ++= entries
            case None =>
              history += board.title -> (TrieMap.empty ++= entries)
          }
        }
    }
  }

  // Load the stats of last 7 days
  private def loadStats(): Unit = {
    boards foreach { board =>
      logger.info(s"loading stats for ${board.title}")
      board
        .fetchHistory(DateTime.now.minusDays(7), DateTime.now)
        .map(getStatsByDay)
        .foreach { newStats =>
          stats.get(board.title) match {
            case Some(value) => value ++= newStats
            case None =>
              stats += board.title -> (TrieMap.empty ++= newStats)
          }
        }
    }
  }

  def getCurrent(name: String): Option[BoardView] = {
    current.get(name)
  }

  def getHistory(name: String): Option[Map[Long, BoardView]] = {
    history.get(name).map(_.toMap)
  }

  def getStats(name: String): Option[Map[Long, Stats]] = {
    stats.get(name).map(_.toMap)
  }

  object Poller extends Runnable {
    override def run(): Unit = {
      val checkTime = DateTime.now().withMillisOfSecond(0).withSecondOfMinute(0)
      logger.info(s"updating ${checkTime.toString()} (${checkTime.getMillis})")
      boards foreach { board =>
        board
          .apply(None)
          .foreach { viewTree =>
            // Update the current view tree of the board
            current += board.title -> viewTree
            // Update history cache
            val records = history.getOrElseUpdate(board.title, TrieMap(checkTime.getMillis -> viewTree)) += checkTime.getMillis -> viewTree
            // evict old entries
            val obsoleted = records.keys.filter(_ < checkTime.minusDays(1).getMillis)
            if (obsoleted.size > 0) {
              logger.info(s"evicted ${obsoleted.size} history entries")
              records --= obsoleted
            }
            logger.info(s"history cache updated, new size: ${records.size}")
            // Update stats
            stats
              .getOrElseUpdate(board.title, TrieMap(checkTime.getMillis -> Stats(0, 0, 0, 0)))
              .get(checkTime.getMillis)
              .map(updateStatsWithStatus(_, viewTree.status))
          }
      }
    }
  }

  sys.addShutdownHook {
    logger.info(s"shutting down StateService")
    scheduler.shutdown()
  }
}

object StateService {
  def getStatsByDay(history: Map[Long, BoardView]): Map[Long, Stats] = {
    history
      .groupBy { case (ts, _) =>
        ts - ts % 86400000 // normalize to the start of the day
      }
      .mapValues { entries =>
{'fraction_non_alphanumeric': 0.11831198191409194, 'fraction_numerical': 0.005777442853554383, 'mean_word_length': 2.4367716875269747, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1457409', 'n_tokens_mistral': 2388, 'n_tokens_neox': 2318, 'n_words': 560}