<commit_before>package omegaup.grader.drivers

import omegaup._
import omegaup.data._
import omegaup.grader._
import java.io._
import java.util.concurrent._
import scala.util.matching.Regex
import scala.collection.mutable.{ListBuffer, TreeSet}
import Language._
import Veredict._
import Status._
import Validator._

object OmegaUpDriver extends Driver with Log with Using {
  def getInputName(alias: String): String = {
    val runtime = Runtime.getRuntime
    val params = List("/usr/bin/git", "ls-tree", "HEAD", "-d", "cases/in")
    val env = List[String]()
    val path = new File(Config.get("problems.root", "problems"), alias)

    // Call git to obtain the hash of the cases/in directory.
    pusing (runtime.exec(params.toArray, env.toArray, path)) { p => {
      using (new BufferedReader(new InputStreamReader(p.getInputStream))) { reader =>
        reader.readLine.split("\\s")(2)
      }
    }}
  }

  def getInputEntries(alias: String, inputName: String): Iterable[InputEntry] = {
    val runtime = Runtime.getRuntime
    val params = List("/usr/bin/git", "ls-tree", inputName)
    val env = List[String]()
    val path = new File(Config.get("problems.root", "problems"), alias)
    val objectsPath = new File(path, ".git/objects")
    val treeHashes = new TreeSet[(String, String)]

    // Get the files in the directory from git.
    pusing (runtime.exec(params.toArray, env.toArray, path)) { p => {
      var line: String = null
      using (new BufferedReader(new InputStreamReader(p.getInputStream))) { reader =>{
        while ({ line = reader.readLine ; line != null } ){
          var tokens = line.split("\\s")
          treeHashes += ((tokens(3), tokens(2)))
        }
      }}
    }}

    // Return a lazy view with the hashes plus their stream.
    treeHashes.view.map { case (name: String, hash: String) => {
        val file = new File(objectsPath, hash.substring(0, 2) + "/" + hash.substring(2))
        new InputEntry(name, new FileInputStream(file), file.length, hash)
    }}
  }

  override def run(ctx: RunContext, run: Run): Run = {
    // If using the literal validator, we can skip the run.
    if (run.problem.validator == Validator.Literal) return run

    val id = run.id
    val alias = run.problem.alias
    val lang = run.language
    val errorFile = new File(Config.get("grader.root", "grader"), + id + ".err")

    info("Compiling {} {} on {}", alias, id, ctx.service.name)
	
    if (errorFile.exists) {
      errorFile.delete<commit_msg>Take into account that sometimes git packs repositories<commit_after>package omegaup.grader.drivers

import omegaup._
import omegaup.data._
import omegaup.grader._
import java.io._
import java.util.concurrent._
import java.util.zip.DeflaterOutputStream
import scala.util.matching.Regex
import scala.collection.mutable.{ListBuffer, TreeSet}
import Language._
import Veredict._
import Status._
import Validator._

object OmegaUpDriver extends Driver with Log with Using {
  def getInputName(alias: String): String = {
    val runtime = Runtime.getRuntime
    val params = List("/usr/bin/git", "ls-tree", "HEAD", "-d", "cases/in")
    val env = List[String]()
    val path = new File(Config.get("problems.root", "problems"), alias)

    // Call git to obtain the hash of the cases/in directory.
    pusing (runtime.exec(params.toArray, env.toArray, path)) { p => {
      using (new BufferedReader(new InputStreamReader(p.getInputStream))) { reader =>
        reader.readLine.split("\\s")(2)
      }
    }}
  }

  def getInputEntries(alias: String, inputName: String): Iterable[InputEntry] = {
    val runtime = Runtime.getRuntime
    val params = List("/usr/bin/git", "ls-tree", inputName)
    val env = List[String]()
    val path = new File(Config.get("problems.root", "problems"), alias)
    val objectsPath = new File(path, ".git/objects")
    val casesPath = new File(path, "cases/in")
    val treeHashes = new TreeSet[(String, String)]

    // Get the files in the directory from git.
    pusing (runtime.exec(params.toArray, env.toArray, path)) { p => {
      var line: String = null
      using (new BufferedReader(new InputStreamReader(p.getInputStream))) { reader =>{
        while ({ line = reader.readLine ; line != null } ){
          var tokens = line.split("\\s")
          treeHashes += ((tokens(3), tokens(2)))
        }
      }}
    }}

    // Return a lazy view with the hashes plus their stream.
    treeHashes.view.map { case (name: String, hash: String) => {
        val file = new File(objectsPath, hash.substring(0, 2) + "/" + hash.substring(2))
        if (file.exists) {
          new InputEntry(name, new FileInputStream(file), file.length, hash)
        } else {
          // The file is likely within a .pack file. We need to compress it manually.
          // TODO(lhchavez): It's probably not a great idea to do this in-memory.
          val originalFile = new File(casesPath, name)
          val header = s"blob ${originalFile.length}\u0000".getBytes
          val bytes = new ByteArrayOutputStream
          using (new DeflaterOutputStream(bytes)) { zlib => {
            zlib.write(header)
            using (new FileInputStream(originalFile)) {
              FileUtil.copy(_, zlib)
            }
          }}

          new InputEntry(name, new ByteArrayInputStream(bytes.toByteArray), bytes.size, null)
        }
    }}
  }

  override def run(ctx: RunContext, run: Run): Run = {
    // If using the literal validator, we can skip the run.
    if (run.problem.validator == Validator.Literal) return run

    val id = run.id
    val alias = run.problem.alias
    val lang = run.language
    val errorFile = new File(Config.get("grader.root", "grader"), + id + ".err")

    info("Compiling {} {} on {}", alias, id, ctx.service.name)
	
    if (errorFile.exists) {
      errorFile.delete
{'fraction_non_alphanumeric': 0.1317923063653614, 'fraction_numerical': 0.0027600483008452647, 'mean_word_length': 2.960382513661202, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '254265', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1728, 'n_words': 506}