<commit_before>package worldmake.executionstrategy

import worldmake._
import scalax.file.Path
import scala.collection.GenMap
import scala.concurrent.{ExecutionContext, Future}
import com.typesafe.scalalogging.slf4j.Logging
import worldmake.WorldMakeConfig._
import scalax.io.Resource
import java.io.{IOException, File}
import worldmake.storage.StoredProvenances
import scala.Some
import scala.sys.process.{ProcessLogger, Process}
import worldmake.derivationstrategy.{CallbackNotifier, PollingAction, Notifier}
import edu.umass.cs.iesl.scalacommons.XMLIgnoreDTD
import ExecutionContext.Implicits.global

/**
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */

trait QsubRunningInfo extends RunningInfo {
  def jobId: Int

  def workingDir: Path

  def outputPath: Path
}

case class MemoryQsubRunningInfo(jobId: Int, workingDir: Path, outputPath: Path, node: Option[String]) extends RunningInfo

class QsubExecutionStrategy(notifier: Notifier) extends SystemExecutionStrategy with Logging {

  def apply(pr: BlockedProvenance[Path], reifiedScriptF: Future[Successful[String]], reifiedDependenciesF: Future[Iterable[(String, Successful[Any])]]): Future[Successful[Path]] = {
    for (reifiedScript <- reifiedScriptF;
         reifiedDependencies <- reifiedDependenciesF;
         x <- systemExecuteWithArgs(pr.pending(Set(reifiedScript), reifiedDependencies.toMap), reifiedScript, reifiedDependencies.toMap)
    ) yield x
  }

  private def systemExecuteWithArgs(pp: PendingProvenance[Path], reifiedScript: Successful[String], reifiedDependencies: GenMap[String, Successful[_]]): Future[Successful[Path]] = {


    // this path does not yet exist.
    // the derivation may write a single file to it, or create a directory there.
    val outputPath: Path = fileStore.newPath
    //val log: File = (outputPath / "worldmake.log").fileOption.getOrElse(throw new Error("can't create log: " + outputPath / "worldmake.log"))<commit_msg>more casbah layer bugfixes<commit_after>package worldmake.executionstrategy

import worldmake._
import scalax.file.Path
import scala.collection.GenMap
import scala.concurrent.{ExecutionContext, Future}
import com.typesafe.scalalogging.slf4j.Logging
import worldmake.WorldMakeConfig._
import scalax.io.Resource
import java.io.{IOException, File}
import worldmake.storage.StoredProvenances
import scala.Some
import scala.sys.process.{ProcessLogger, Process}
import worldmake.derivationstrategy.{CallbackNotifier, PollingAction, Notifier}
import edu.umass.cs.iesl.scalacommons.XMLIgnoreDTD
import ExecutionContext.Implicits.global

/**
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */

trait QsubRunningInfo extends RunningInfo {
  def jobId: Int

  def workingDir: Path

  def outputPath: Path
}

case class MemoryQsubRunningInfo(jobId: Int, workingDir: Path, outputPath: Path, node: Option[String]) extends QsubRunningInfo

class QsubExecutionStrategy(notifier: Notifier) extends SystemExecutionStrategy with Logging {

  def apply(pr: BlockedProvenance[Path], reifiedScriptF: Future[Successful[String]], reifiedDependenciesF: Future[Iterable[(String, Successful[Any])]]): Future[Successful[Path]] = {
    for (reifiedScript <- reifiedScriptF;
         reifiedDependencies <- reifiedDependenciesF;
         x <- systemExecuteWithArgs(pr.pending(Set(reifiedScript), reifiedDependencies.toMap), reifiedScript, reifiedDependencies.toMap)
    ) yield x
  }

  private def systemExecuteWithArgs(pp: PendingProvenance[Path], reifiedScript: Successful[String], reifiedDependencies: GenMap[String, Successful[_]]): Future[Successful[Path]] = {


    // this path does not yet exist.
    // the derivation may write a single file to it, or create a directory there.
    val outputPath: Path = fileStore.newPath
    //val log: File = (outputPath / "worldmake.log").fileOption.getOrElse(throw new Error("can't create log: " + outputPath / "worldmake.log"))
{'fraction_non_alphanumeric': 0.10270967741935484, 'fraction_numerical': 0.0005161290322580645, 'mean_word_length': 5.823943661971831, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3437583', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1135, 'n_words': 274}