<commit_before>package ch.epfl.scala.index.data
package bintray

import download.PlayWsDownloader

import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}

import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import play.api.libs.ws.{WSRequest, WSResponse}

class DownloadPoms  (implicit val system: ActorSystem, implicit val materializer: ActorMaterializer) extends PlayWsDownloader {

  /**
   * resolve the filename for a specific pom by sha1
   *
   * @param search the bintray search object
   * @return
   */
  private def pomPath(search: BintraySearch) = bintrayPomBase.resolve(s"${search.sha1}.pom")

  /**
   * will compute ans sha1 hash from given string and verify that against
   * given compare hash
   * @param toVerify the string to compute
   * @param sha1 the verify hash
   * @return
   */
  def verifyChecksum(toVerify: String, sha1: String) = {

    val md = java.security.MessageDigest.getInstance("SHA-1")
    val computed = md.digest(toVerify.getBytes("UTF-8")).map("%02x".format(_)).mkString

    computed == sha1
  }

  /**
   * verify the sha1 file hash
   * @param path the path to the file to verify
   * @param sha1 the sha1 hash
   * @return
   */
  private def verifySHA1FileHash(path: Path, sha1: String): Boolean = {

    val source = scala.io.Source.fromFile(path.toFile)
    val content = source.mkString.split(nl).mkString("")
    source.close()

    verifyChecksum(content, sha1)
  }
  /**
   * get a list of bintraySearch object where no sha1 file exists
   */
  private val searchesBySha1: Set[BintraySearch] = {

    BintrayMeta.readQueriedPoms(bintrayCheckpoint)
      .filter(s => !Files.exists(pomPath(s)) || !verifySHA1FileHash(pomPath(s), s.sha1))
      .groupBy(_.sha1)                         // remove duplicates with sha1
      .map { case (_, vs) => vs.head }
      .toSet
  }

  /**
   * partly url encode - replaces only spaces
   *
   * @param url the url to "encode"
   * @return
   */
  private def escape(url: String) = url.replace(" ", "%20")

  /**
   * get the download request object
   * downloads pom from
   * - jcenter.bintray.com or<commit_msg>seems like sha1 verification does not work<commit_after>package ch.epfl.scala.index.data
package bintray

import download.PlayWsDownloader

import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}

import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import play.api.libs.ws.{WSRequest, WSResponse}

class DownloadPoms  (implicit val system: ActorSystem, implicit val materializer: ActorMaterializer) extends PlayWsDownloader {

  /**
   * resolve the filename for a specific pom by sha1
   *
   * @param search the bintray search object
   * @return
   */
  private def pomPath(search: BintraySearch) = bintrayPomBase.resolve(s"${search.sha1}.pom")

  /**
   * will compute ans sha1 hash from given string and verify that against
   * given compare hash
   * @param toVerify the string to compute
   * @param sha1 the verify hash
   * @return
   */
  def verifyChecksum(toVerify: String, sha1: String) = {

    val md = java.security.MessageDigest.getInstance("SHA-1")
    val computed = md.digest(toVerify.getBytes("UTF-8")).map("%02x".format(_)).mkString

    computed == sha1
  }

  /**
   * verify the sha1 file hash
   * @param path the path to the file to verify
   * @param sha1 the sha1 hash
   * @return
   */
  private def verifySHA1FileHash(path: Path, sha1: String): Boolean = {

    val source = scala.io.Source.fromFile(path.toFile)
    val content = source.mkString.split(nl).mkString("")
    source.close()

    verifyChecksum(content, sha1)
  }
  /**
   * get a list of bintraySearch object where no sha1 file exists
   */
  private val searchesBySha1: Set[BintraySearch] = {

    BintrayMeta.readQueriedPoms(bintrayCheckpoint)
      .filter(s => !Files.exists(pomPath(s)))// || !verifySHA1FileHash(pomPath(s), s.sha1))
      .groupBy(_.sha1)                         // remove duplicates with sha1
      .map { case (_, vs) => vs.head }
      .toSet
  }

  /**
   * partly url encode - replaces only spaces
   *
   * @param url the url to "encode"
   * @return
   */
  private def escape(url: String) = url.replace(" ", "%20")

  /**
   * get the download request object
   * downloads pom from
   * - jcenter.bintray.com or
{'fraction_non_alphanumeric': 0.11278369615562761, 'fraction_numerical': 0.011347846225104215, 'mean_word_length': 3.181026137463698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5713065', 'n_tokens_mistral': 1424, 'n_tokens_neox': 1381, 'n_words': 377}