<commit_before>package lila.video

import org.joda.time.DateTime
import play.api.libs.functional.syntax._
import play.api.libs.json._
import play.api.libs.ws.WS
import play.api.Play.current

private[video] final class Youtube(
    url: String,
    apiKey: String,
    max: Int,
    api: VideoApi) {

  import Youtube._

  private implicit val readSnippet = Json.reads[Snippet]
  private implicit val readStatistics = Json.reads[Statistics]
  private implicit val readEntry = Json.reads[Entry]
  private implicit val readEntries: Reads[Seq[Entry]] =
    (__ \ "items").read(Reads seq readEntry)

  def updateAll: Funit = fetch flatMap { entries =>
    entries.map { entry =>
      api.video.setMetadata(entry.id, Metadata(
        views = ~parseIntOption(entry.statistics.viewCount),
        likes = ~parseIntOption(entry.statistics.likeCount) -
          ~parseIntOption(entry.statistics.dislikeCount),
        description = entry.snippet.description,
        publishedAt = entry.snippet.publishedAt.flatMap { at =>
          scala.util.Try { new DateTime(at) }.toOption
        })
      ).recover {
        case e: Exception => logerr(s"[video youtube] ${e.getMessage}")
      }
    }.sequenceFu.void
  }

  private def fetch: Fu[List[Entry]] = api.video.allIds flatMap { ids =>
    WS.url(url).withQueryString(
      "id" -> scala.util.Random.shuffle(ids).take(max).mkString(","),
      "part" -> "id,statistics,snippet",
      "key" -> apiKey
    ).get() flatMap {
        case res if res.status == 200 => readEntries reads res.json match {
          case JsError(err)          => fufail(err.toString)
          case JsSuccess(entries, _) => fuccess(entries.toList)
        }
        case res =>
          println(s"key: $apiKey")
          fufail(s"[video youtube] fetch ${res.status}")
      }
  }
}

object Youtube {

  def empty = Metadata(0, 0, None, None)

  case class Metadata(
    views: Int,
    likes: Int,
    description: Option[String],
    publishedAt: Option[DateTime])

  private[video] case class Entry(
    id: String,<commit_msg>tweak debug<commit_after>package lila.video

import org.joda.time.DateTime
import play.api.libs.functional.syntax._
import play.api.libs.json._
import play.api.libs.ws.WS
import play.api.Play.current

private[video] final class Youtube(
    url: String,
    apiKey: String,
    max: Int,
    api: VideoApi) {

  import Youtube._

  private implicit val readSnippet = Json.reads[Snippet]
  private implicit val readStatistics = Json.reads[Statistics]
  private implicit val readEntry = Json.reads[Entry]
  private implicit val readEntries: Reads[Seq[Entry]] =
    (__ \ "items").read(Reads seq readEntry)

  def updateAll: Funit = fetch flatMap { entries =>
    entries.map { entry =>
      api.video.setMetadata(entry.id, Metadata(
        views = ~parseIntOption(entry.statistics.viewCount),
        likes = ~parseIntOption(entry.statistics.likeCount) -
          ~parseIntOption(entry.statistics.dislikeCount),
        description = entry.snippet.description,
        publishedAt = entry.snippet.publishedAt.flatMap { at =>
          scala.util.Try { new DateTime(at) }.toOption
        })
      ).recover {
        case e: Exception => logerr(s"[video youtube] ${e.getMessage}")
      }
    }.sequenceFu.void
  }

  private def fetch: Fu[List[Entry]] = api.video.allIds flatMap { ids =>
    WS.url(url).withQueryString(
      "id" -> scala.util.Random.shuffle(ids).take(max).mkString(","),
      "part" -> "id,statistics,snippet",
      "key" -> apiKey
    ).get() flatMap {
        case res if res.status == 200 => readEntries reads res.json match {
          case JsError(err)          => fufail(err.toString)
          case JsSuccess(entries, _) => fuccess(entries.toList)
        }
        case res =>
          println(res.body)
          fufail(s"[video youtube] fetch ${res.status}")
      }
  }
}

object Youtube {

  def empty = Metadata(0, 0, None, None)

  case class Metadata(
    views: Int,
    likes: Int,
    description: Option[String],
    publishedAt: Option[DateTime])

  private[video] case class Entry(
    id: String,
{'fraction_non_alphanumeric': 0.12625152625152625, 'fraction_numerical': 0.002442002442002442, 'mean_word_length': 2.8641509433962264, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3294620', 'n_tokens_mistral': 1325, 'n_tokens_neox': 1270, 'n_words': 289}