<commit_before>package amora.backend.indexer

import java.io.ByteArrayOutputStream
import java.io.{ File ⇒ JFile }
import java.io.InputStream
import java.util.zip.ZipEntry
import java.util.zip.ZipFile

import amora.backend.Logger
import amora.backend.schema
import amora.converter.ClassfileConverter
import amora.converter.protocol._
import scalaz._
import scalaz.concurrent.Task

object ArtifactFetcher {
  sealed trait DownloadStatus {
    def isError: Boolean
  }
  final case class DownloadSuccess(artifact: Artifact, file: JFile) extends DownloadStatus {
    override def isError = false
  }
  final case class DownloadError(artifactName: String, reason: Option[String]) extends DownloadStatus {
    override def isError = true
  }
}

trait ArtifactFetcher {
  import ArtifactFetcher._

  def logger: Logger
  def sparqlUpdate(query: String, errMsg: ⇒ String): Unit
  def cacheLocation: JFile

  def handleArtifacts(artifacts: Seq[Artifact]): Seq[DownloadStatus] = {
    val res = artifacts flatMap fetchArtifact
    val (errors, succs) = res.partition(_.isError)
    val succMsgs = succs.collect {
      case DownloadSuccess(_, file) ⇒
        file.getName
    }
    val errMsgs = errors.collect {
      case DownloadError(artifactName, reasonOpt) ⇒
        if (reasonOpt.isDefined)
          artifactName+"because of: "+reasonOpt.get
        else
          artifactName
    }
    val succMsg = if (succs.isEmpty) Nil else Seq(s"Fetched artifacts:" + succMsgs.sorted.mkString("\n  ", "\n  ", ""))
    val errMsg = if (errors.isEmpty) Nil else Seq(s"Failed to fetch artifacts:" + errMsgs.sorted.mkString("\n  ", "\n  ", ""))
    val msg = Seq(succMsg, errMsg).flatten.mkString("\n")

    logger.info(msg)

    if (errors.isEmpty)<commit_msg>Do not resolve amora artifacts through coursier<commit_after>package amora.backend.indexer

import java.io.ByteArrayOutputStream
import java.io.{ File ⇒ JFile }
import java.io.InputStream
import java.util.zip.ZipEntry
import java.util.zip.ZipFile

import amora.backend.Logger
import amora.backend.schema
import amora.converter.ClassfileConverter
import amora.converter.protocol._
import scalaz._
import scalaz.concurrent.Task

object ArtifactFetcher {
  sealed trait DownloadStatus {
    def isError: Boolean
  }
  final case class DownloadSuccess(artifact: Artifact, file: JFile) extends DownloadStatus {
    override def isError = false
  }
  final case class DownloadError(artifactName: String, reason: Option[String]) extends DownloadStatus {
    override def isError = true
  }
}

trait ArtifactFetcher {
  import ArtifactFetcher._

  def logger: Logger
  def sparqlUpdate(query: String, errMsg: ⇒ String): Unit
  def cacheLocation: JFile

  def handleArtifacts(artifacts: Seq[Artifact]): Seq[DownloadStatus] = {
    val (ignored, relevant) = artifacts.partition(_.organization == "amora.center")
    if (ignored.nonEmpty)
      logger.info("Ignoring artifacts:" + ignored.map(a ⇒ s"${a.organization}:${a.name}:${a.version}").sorted.mkString("\n  ", "\n  ", ""))

    val res = relevant flatMap fetchArtifact
    val (errors, succs) = res.partition(_.isError)
    val succMsgs = succs.collect {
      case DownloadSuccess(_, file) ⇒
        file.getName
    }
    val errMsgs = errors.collect {
      case DownloadError(artifactName, reasonOpt) ⇒
        if (reasonOpt.isDefined)
          artifactName+"because of: "+reasonOpt.get
        else
          artifactName
    }
    val succMsg = if (succs.isEmpty) Nil else Seq(s"Fetched artifacts:" + succMsgs.sorted.mkString("\n  ", "\n  ", ""))
    val errMsg = if (errors.isEmpty) Nil else Seq(s"Failed to fetch artifacts:" + errMsgs.sorted.mkString("\n  ", "\n  ", ""))
    val msg = Seq(succMsg, errMsg).flatten.mkString("\n")

    logger.info(msg)

    if (errors.isEmpty)
{'fraction_non_alphanumeric': 0.1083156779661017, 'fraction_numerical': 0.0, 'mean_word_length': 3.756926952141058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '257958', 'n_tokens_mistral': 1198, 'n_tokens_neox': 1173, 'n_words': 302}