<commit_before>package ch.epfl.scala.index.data.bintray

import java.nio.file.Files

import akka.actor.ActorSystem
import akka.stream.Materializer
import ch.epfl.scala.index.data.DataPaths
import ch.epfl.scala.index.data.download.PlayWsDownloader
import ch.epfl.scala.index.data.maven._
import me.tongfei.progressbar.ProgressBar
import org.apache.ivy.core.module.descriptor.ModuleDescriptor
import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.plugins.parser.xml.XmlModuleDescriptorParser
import org.joda.time.{DateTime, DateTimeZone}
import org.joda.time.format.ISODateTimeFormat
import org.json4s.JsonAST.JValue
import org.slf4j.LoggerFactory
import play.api.libs.ws.{WSClient, WSResponse}

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future}
import scala.util.{Failure, Success, Try}

/**
  * Fetches sbt plugins information
  */
final class BintrayDownloadSbtPlugins(paths: DataPaths)(implicit
  val materializer: Materializer,
  val system: ActorSystem
) extends PlayWsDownloader
    with BintrayClient
    with BintrayProtocol {

  import BintrayDownloadSbtPlugins._

  private val logger = LoggerFactory.getLogger(this.getClass)

  private val ivysData = SbtPluginsData(paths)

  // TODO Define this list in the config
  val pluginsRepositories =
    List(
      "sbt" -> "sbt-plugin-releases"
    )

  /** Matches an ivy.xml file that describes the release of an sbt-plugin */
  // <organization>/<artifact>/scala_<version>/sbt_<version>/<version>/ivys/ivy.xml
  private val sbtPluginPath = "(.+?)/(.+?)/scala_(.+?)/sbt_(.+?)/(.+?)/ivys/ivy.xml".r

  def run(): Unit = {
    val _ = Await.ready(downloadSbtPlugins(), Duration.Inf)
  }

  /**
    * Downloads all sbt plugin releases from bintray.
    * If some releases have already been downloaded in the past, only newer releases are downloaded.
    */
  def downloadSbtPlugins(): Future[_] =
    for {
      (oldReleases, lastDownload) <- Future.successful(ivysData.read())
      newReleases <- fetchNewReleases(lastDownload)
    } yield {
      ivysData.update(oldReleases, newReleases.map(parseRelease))
    }

  private def parseRelease(release: SbtPluginReleaseWithModuleDescriptor): SbtPluginReleaseModel = {
    val descriptor = release.moduleDescriptor
    val releaseModel =
      ReleaseModel(
        groupId = release.sbtRelease.organization,
        artifactId = release.sbtRelease.artifact,
        version = release.sbtRelease.version,
        packaging = "jar",
        name = None,
        description = Some(descriptor.getDescription),
        inceptionYear = None,
        url = None,
        scm = None,
        issueManagement = None,
        mailingLists = Nil,
        contributors = Nil,
        developers = Nil,
        licenses = Nil,
        dependencies = descriptor.getDependencies.to[List].map { dependency =>
          val dep = dependency.getDependencyRevisionId
          Dependency(
            dep.getOrganisation,
            dep.getName,
            dep.getRevision,
            scope = /* TODO */ None,
            exclusions = dependency.getAllExcludeRules.map { rule =>
              Exclusion(rule.getId.getModuleId.getOrganisation, rule.getId.getModuleId.getName)
            }.to[Set]
          )
        },
        repositories = Nil,
        organization = None,
        sbtPluginTarget = Some(
          SbtPluginTarget(
            release.sbtRelease.scalaVersion,
            release.sbtRelease.sbtVersion
          )
        )
      )

    SbtPluginReleaseModel(releaseModel, new DateTime(descriptor.getPublicationDate).toString, release.sbtRelease.sha1)
  }

  private def fetchNewReleases(lastDownload: Option[String]): Future[List[SbtPluginReleaseWithModuleDescriptor]] =
    managed { client =>
      val lastDownload =
        Try(new String(Files.readAllBytes(paths.ivysLastDownload))).toOption

      Future.traverse(pluginsRepositories)((listSbtPluginReleases(client, lastDownload) _).tupled)
        .map(_.flatten) // Concatenate the plugins of all the `pluginRepositories`

      // TODO Check sha1

    }.andThen { case Success(newReleases) =>
      writeSbtPluginReleases(newReleases)
    }

  /**
    * @return All the sbt plugin releases found in the given repository<commit_msg>#191 Null defenses<commit_after>package ch.epfl.scala.index.data.bintray

import java.nio.file.Files

import akka.actor.ActorSystem
import akka.stream.Materializer
import ch.epfl.scala.index.data.DataPaths
import ch.epfl.scala.index.data.download.PlayWsDownloader
import ch.epfl.scala.index.data.maven._
import me.tongfei.progressbar.ProgressBar
import org.apache.ivy.core.module.descriptor.ModuleDescriptor
import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.plugins.parser.xml.XmlModuleDescriptorParser
import org.joda.time.{DateTime, DateTimeZone}
import org.joda.time.format.ISODateTimeFormat
import org.json4s.JsonAST.JValue
import org.slf4j.LoggerFactory
import play.api.libs.ws.{WSClient, WSResponse}

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future}
import scala.util.{Failure, Success, Try}

/**
  * Fetches sbt plugins information
  */
final class BintrayDownloadSbtPlugins(paths: DataPaths)(implicit
  val materializer: Materializer,
  val system: ActorSystem
) extends PlayWsDownloader
    with BintrayClient
    with BintrayProtocol {

  import BintrayDownloadSbtPlugins._

  private val logger = LoggerFactory.getLogger(this.getClass)

  private val ivysData = SbtPluginsData(paths)

  // TODO Define this list in the config
  val pluginsRepositories =
    List(
      "sbt" -> "sbt-plugin-releases"
    )

  /** Matches an ivy.xml file that describes the release of an sbt-plugin */
  // <organization>/<artifact>/scala_<version>/sbt_<version>/<version>/ivys/ivy.xml
  private val sbtPluginPath = "(.+?)/(.+?)/scala_(.+?)/sbt_(.+?)/(.+?)/ivys/ivy.xml".r

  def run(): Unit = {
    val _ = Await.ready(downloadSbtPlugins(), Duration.Inf)
  }

  /**
    * Downloads all sbt plugin releases from bintray.
    * If some releases have already been downloaded in the past, only newer releases are downloaded.
    */
  def downloadSbtPlugins(): Future[_] =
    for {
      (oldReleases, lastDownload) <- Future.successful(ivysData.read())
      newReleases <- fetchNewReleases(lastDownload)
    } yield {
      ivysData.update(oldReleases, newReleases.map(parseRelease))
    }

  private def parseRelease(release: SbtPluginReleaseWithModuleDescriptor): SbtPluginReleaseModel = {
    val descriptor = release.moduleDescriptor
    val releaseModel =
      ReleaseModel(
        groupId = release.sbtRelease.organization,
        artifactId = release.sbtRelease.artifact,
        version = release.sbtRelease.version,
        packaging = "jar",
        name = None,
        description = Option(descriptor.getDescription),
        inceptionYear = None,
        url = None,
        scm = None,
        issueManagement = None,
        mailingLists = Nil,
        contributors = Nil,
        developers = Nil,
        licenses = Nil,
        dependencies = descriptor.getDependencies.to[List].map { dependency =>
          val dep = dependency.getDependencyRevisionId
          Dependency(
            dep.getOrganisation,
            dep.getName,
            dep.getRevision,
            scope = /* TODO */ None,
            exclusions = dependency.getAllExcludeRules.map { rule =>
              Exclusion(rule.getId.getModuleId.getOrganisation, rule.getId.getModuleId.getName)
            }.to[Set]
          )
        },
        repositories = Nil,
        organization = None,
        sbtPluginTarget = Some(
          SbtPluginTarget(
            release.sbtRelease.scalaVersion,
            release.sbtRelease.sbtVersion
          )
        )
      )

    val publicationDate =
      new DateTime(Option(descriptor.getPublicationDate).getOrElse(sys.error("Missing publication date"))).toString
    SbtPluginReleaseModel(releaseModel, publicationDate, release.sbtRelease.sha1)
  }

  private def fetchNewReleases(lastDownload: Option[String]): Future[List[SbtPluginReleaseWithModuleDescriptor]] =
    managed { client =>
      val lastDownload =
        Try(new String(Files.readAllBytes(paths.ivysLastDownload))).toOption

      Future.traverse(pluginsRepositories)((listSbtPluginReleases(client, lastDownload) _).tupled)
        .map(_.flatten) // Concatenate the plugins of all the `pluginRepositories`

      // TODO Check sha1

    }.andThen { case Success(newReleases) =>
      writeSbtPluginReleases(newReleases)
    }

  /**
    * @return All the sbt plugin releases found in the given repository
{'fraction_non_alphanumeric': 0.09945004582951421, 'fraction_numerical': 0.0012603116406966087, 'mean_word_length': 3.1986531986531985, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1467950', 'n_tokens_mistral': 2635, 'n_tokens_neox': 2520, 'n_words': 514}