<commit_before>package uk.gov.dvla.vehicles.presentation.common.webserviceclients.healthstats

import com.google.inject.Inject
import org.joda.time.Instant
import uk.gov.dvla.vehicles.presentation.common.services.DateService

import Math.max

import scala.collection.mutable
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

sealed trait HealthStatsEvent {
  val msName: String
  val time: Instant
}

case class HealthStatsSuccess(msName: String, time: Instant) extends HealthStatsEvent

case class HealthStatsFailure(msName: String, time: Instant, t: Throwable) extends HealthStatsEvent

case class NotHealthyStats(msName: String, details: String)

class HealthStats @Inject()(config: HealthStatsConfig, dateService: DateService) {
  private type MsStats = mutable.ArrayBuffer[HealthStatsEvent]
  private type Stats = collection.mutable.HashMap[String, MsStats]
  private type FailCounts = collection.mutable.HashMap[String, Int]

  private val events = new Stats()
  private val consecutiveFailCounts = new FailCounts()

  def report[T](msName: String)
               (future: Future[T]): Future[T] = {
    future.onSuccess {
      case _ => success(HealthStatsSuccess(msName, dateService.now))
    }
    future.onFailure {
      case e: Throwable =>
        failure(HealthStatsFailure(msName, dateService.now, e))
    }
    future
  }

  def failure(failure: HealthStatsFailure): Unit = this.synchronized {
    if (config.numberOfConsecutiveFailures > 0)
      consecutiveFailCounts.put(failure.msName, consecutiveFailCounts.getOrElse(failure.msName, 0) + 1)
    events.put(
      failure.msName,
      events.getOrElse(failure.msName, new MsStats()).:+(failure)
    )
  }

  def success(success: HealthStatsSuccess): Unit = this.synchronized {
    if (config.numberOfConsecutiveFailures > 0)
      consecutiveFailCounts.put(success.msName, 0)
    events.put(
      success.msName,
      events.getOrElse(success.msName, new MsStats()).:+(success)
    )
  }

  def healthy: Option[NotHealthyStats] = this.synchronized {
    try hasConsecutive(events) orElse {
      events.foreach { case (msName, eventsPerMs) =>
        if (config.numberOfRequests < 0) return None
        else if (config.numberOfRequests < requestRate(eventsPerMs, dateService.now, numberOfRequestsThreshold))
          hasRelativeFailures(msName, eventsPerMs, relativeFailuresThreshold) match {
            case Some(result) => return Some(result)
            case _ =>
          }
        else hasAbsoluteFailures(eventsPerMs, numberOfFailuresThreshold) match {
          case Some(result) => return Some(result)
          case _ =>
        }
      }
      None
    } finally events.transform((_, value) => value.dropWhile(_.time.isBefore(oldThreshold)))
  }

  private def hasConsecutive(events: Stats): Option[NotHealthyStats]  = {
    consecutiveFailCounts.foreach { case (msName, msFailures) =>
      if (msFailures >= config.numberOfConsecutiveFailures)
        return Some(NotHealthyStats(
          msName, s"The number of consecutive failures in $msName is $msFailures and " +
            s"the fail threshold of ${config.numberOfConsecutiveFailures}"
        ))
    }
    None
  }

  private def requestRate(events: MsStats, now: Instant, numberOfRequestsThreshold: Instant): Float =
    if (events.isEmpty || config.numberOfRequestsTimeFrame < 1) 0
    else
      events.dropWhile(_.time.isBefore(numberOfRequestsThreshold)).size * config.numberOfRequestsTimeFrame /
      now.minus(events.head.time.getMillis).getMillis

  private def hasAbsoluteFailures(events: MsStats, numberOfFailuresThreshold: Instant): Option[NotHealthyStats]  =
    if (config.numberOfFailures < 0) None
    else {
      events.foldRight(config.numberOfFailures)((event, numberOfFailures) =>
        if (event.time.isBefore(numberOfFailuresThreshold)) return None
        else if (event.isInstanceOf[HealthStatsFailure]) {
          if (numberOfFailures < 2)
            return Some(NotHealthyStats(s"${event.msName}", s"${event.msName} has more" +
              s" then ${config.numberOfFailures} failures for the last ${config.numberOfFailures}ms" ))
          numberOfFailures - 1
        }<commit_msg>US2454 add some logging to the HealthStats<commit_after>package uk.gov.dvla.vehicles.presentation.common.webserviceclients.healthstats

import com.google.inject.Inject
import org.joda.time.Instant
import uk.gov.dvla.vehicles.presentation.common.services.DateService

import Math.max
import play.api.Logger

import scala.collection.mutable
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

sealed trait HealthStatsEvent {
  val msName: String
  val time: Instant
}

case class HealthStatsSuccess(msName: String, time: Instant) extends HealthStatsEvent

case class HealthStatsFailure(msName: String, time: Instant, t: Throwable) extends HealthStatsEvent

case class NotHealthyStats(msName: String, details: String)

class HealthStats @Inject()(config: HealthStatsConfig, dateService: DateService) {
  private type MsStats = mutable.ArrayBuffer[HealthStatsEvent]
  private type Stats = collection.mutable.HashMap[String, MsStats]
  private type FailCounts = collection.mutable.HashMap[String, Int]

  private val events = new Stats()
  private val consecutiveFailCounts = new FailCounts()

  def report[T](msName: String)
               (future: Future[T]): Future[T] = {
    future.onSuccess {
      case _ => success(HealthStatsSuccess(msName, dateService.now))
    }
    future.onFailure {
      case e: Throwable =>
        failure(HealthStatsFailure(msName, dateService.now, e))
    }
    future
  }

  def failure(failure: HealthStatsFailure): Unit = this.synchronized {
    Logger.debug(s"HealthStats recieved a failure event: $failure")
    if (config.numberOfConsecutiveFailures > 0)
      consecutiveFailCounts.put(failure.msName, consecutiveFailCounts.getOrElse(failure.msName, 0) + 1)
    events.put(
      failure.msName,
      events.getOrElse(failure.msName, new MsStats()).:+(failure)
    )
  }

  def success(success: HealthStatsSuccess): Unit = this.synchronized {
    Logger.debug(s"HealthStats recieved a success event: $success")
    if (config.numberOfConsecutiveFailures > 0)
      consecutiveFailCounts.put(success.msName, 0)
    events.put(
      success.msName,
      events.getOrElse(success.msName, new MsStats()).:+(success)
    )
  }

  def healthy: Option[NotHealthyStats] = this.synchronized {
    val healthyStatus = try hasConsecutive(events) orElse {
      events.foreach { case (msName, eventsPerMs) =>
        if (config.numberOfRequests < 0) return None
        else if (config.numberOfRequests < requestRate(eventsPerMs, dateService.now, numberOfRequestsThreshold))
          hasRelativeFailures(msName, eventsPerMs, relativeFailuresThreshold) match {
            case Some(result) => return Some(result)
            case _ =>
          }
        else hasAbsoluteFailures(eventsPerMs, numberOfFailuresThreshold) match {
          case Some(result) => return Some(result)
          case _ =>
        }
      }
      None
    } finally events.transform((_, value) => value.dropWhile(_.time.isBefore(oldThreshold)))
    Logger.debug(s"HealthStats recieved a healthy query. The answer is $healthyStatus")
    healthyStatus
  }

  private def hasConsecutive(events: Stats): Option[NotHealthyStats]  = {
    Logger.debug(s"HealthStats consecuteFailCounts: $consecutiveFailCounts allEvents: $events")
    consecutiveFailCounts.foreach { case (msName, msFailures) =>
      if (msFailures >= config.numberOfConsecutiveFailures)
        return Some(NotHealthyStats(
          msName, s"The number of consecutive failures in $msName is $msFailures and " +
            s"the fail threshold of ${config.numberOfConsecutiveFailures}"
        ))
    }
    None
  }

  private def requestRate(events: MsStats, now: Instant, numberOfRequestsThreshold: Instant): Float =
    if (events.isEmpty || config.numberOfRequestsTimeFrame < 1) 0
    else
      events.dropWhile(_.time.isBefore(numberOfRequestsThreshold)).size * config.numberOfRequestsTimeFrame /
      now.minus(events.head.time.getMillis).getMillis

  private def hasAbsoluteFailures(events: MsStats, numberOfFailuresThreshold: Instant): Option[NotHealthyStats]  =
    if (config.numberOfFailures < 0) None
    else {
      events.foldRight(config.numberOfFailures)((event, numberOfFailures) =>
        if (event.time.isBefore(numberOfFailuresThreshold)) return None
        else if (event.isInstanceOf[HealthStatsFailure]) {
          if (numberOfFailures < 2)
            return Some(NotHealthyStats(s"${event.msName}", s"${event.msName} has more" +
              s" then ${config.numberOfFailures} failures for the last ${config.numberOfFailures}ms" ))
          numberOfFailures - 1
        }
{'fraction_non_alphanumeric': 0.0974143796779315, 'fraction_numerical': 0.0029485144023588114, 'mean_word_length': 3.813864628820961, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3528182', 'n_tokens_mistral': 2596, 'n_tokens_neox': 2572, 'n_words': 561}