<commit_before>import sbt._

import Build._
import Keys._
import Project.Initialize
import scala.collection.{ mutable, immutable }




/** Helpers to resolve SHA artifacts from typesafe repo. */
object ShaResolve {
  import dispatch.{Http,url}
  val remote_urlbase="http://typesafe.artifactoryonline.com/typesafe/scala-sha-bootstrap/org/scala-lang/bootstrap"  
  
  val pullBinaryLibs = TaskKey[Unit]("pull-binary-libs", "Pulls binary libs by the SHA key.")
  val pushBinaryLibs = TaskKey[Unit]("push-binary-libs", "Pushes binary libs whose SHA has changed.")
  val binaryLibCache = SettingKey[File]("binary-lib-cache", "Location of the cache of binary libs for this scala build.")

  def settings: Seq[Setting[_]] = Seq(
    binaryLibCache in ThisBuild := file(System.getProperty("user.home")) / ".sbt" / "cache" / "scala",
    pullBinaryLibs in ThisBuild <<= (baseDirectory, binaryLibCache, streams) map resolveLibs
  )

  def resolveLibs(dir: File, cacheDir: File, s: TaskStreams): Unit = {
     val files = (dir / "test" / "files" ** "*.desired.sha1") +++ (dir / "lib" ** "*.desired.sha1")
     for {
       (file, name) <- (files x relativeTo(dir)).par
       uri = name.dropRight(13)       
       jar = dir / uri
       if !jar.exists || !isValidSha(file)
       sha = getShaFromShafile(file)
     } pullFile(jar, sha + "/" + uri, cacheDir, s)
  }

  def getShaFromShafile(file: File): String = (IO read file split "\\s" headOption) getOrElse error("No SHA found for " + file)


  def isValidSha(file: File): Boolean =
    try (Process(Seq("shasum", "-p", "--check", file.getAbsolutePath), Some(file.getParentFile)).!!  contains "OK") 
    catch {
      case t: Exception => false
    }
     

  def pullFile(file: File, uri: String, cacheDir: File, s: TaskStreams): Unit = {<commit_msg>Improved error output.<commit_after>import sbt._

import Build._
import Keys._
import Project.Initialize
import scala.collection.{ mutable, immutable }
import scala.collection.parallel.CompositeThrowable



/** Helpers to resolve SHA artifacts from typesafe repo. */
object ShaResolve {
  import dispatch.{Http,url}
  val remote_urlbase="http://typesafe.artifactoryonline.com/typesafe/scala-sha-bootstrap/org/scala-lang/bootstrap"  
  
  val pullBinaryLibs = TaskKey[Unit]("pull-binary-libs", "Pulls binary libs by the SHA key.")
  val pushBinaryLibs = TaskKey[Unit]("push-binary-libs", "Pushes binary libs whose SHA has changed.")
  val binaryLibCache = SettingKey[File]("binary-lib-cache", "Location of the cache of binary libs for this scala build.")

  def settings: Seq[Setting[_]] = Seq(
    binaryLibCache in ThisBuild := file(System.getProperty("user.home")) / ".sbt" / "cache" / "scala",
    pullBinaryLibs in ThisBuild <<= (baseDirectory, binaryLibCache, streams) map resolveLibs
  )

  def resolveLibs(dir: File, cacheDir: File, s: TaskStreams): Unit = loggingParallelExceptions(s) {
     val files = (dir / "test" / "files" ** "*.desired.sha1") +++ (dir / "lib" ** "*.desired.sha1")
     for {
       (file, name) <- (files x relativeTo(dir)).par
       uri = name.dropRight(13)       
       jar = dir / uri
       if !jar.exists || !isValidSha(file)
       sha = getShaFromShafile(file)
     } pullFile(jar, sha + "/" + uri, cacheDir, s)
  }

  @inline final def loggingParallelExceptions[U](s: TaskStreams)(f: => U): U = try f catch {
    case t: CompositeThrowable =>
      s.log.error("Error during parallel execution, GET READ FOR STACK TRACES!!")
      t.throwables foreach (t2 => s.log.trace(t2))
      throw t
  }

  def getShaFromShafile(file: File): String = (IO read file split "\\s" headOption) getOrElse error("No SHA found for " + file)


  def isValidSha(file: File): Boolean =
    try (Process(Seq("shasum", "-p", "--check", file.getAbsolutePath), Some(file.getParentFile)).!!  contains "OK") 
    catch {
      case t: Exception => false
    }
     

  def pullFile(file: File, uri: String, cacheDir: File, s: TaskStreams): Unit = {
{'fraction_non_alphanumeric': 0.14206268958543983, 'fraction_numerical': 0.0025278058645096056, 'mean_word_length': 3.6065192083818394, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5968942', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1278, 'n_words': 353}