<commit_before>package gridscale

import effectaside._
import gridscale.cluster.{ BatchScheduler, HeadNode, Requirement }
import gridscale.tools._
import monocle.macros._
import squants._
import squants.information._

import scala.language.higherKinds

package object slurm {

  case class Gres(gresName: String, gresValue: Int) {
    override def toString = gresName + ":" + gresValue.toString
  }

  @Lenses case class SLURMJobDescription(
    command: String,
    workDirectory: String,
    queue: Option[String] = None,
    wallTime: Option[Time] = None,
    memory: Option[Information] = None,
    nodes: Option[Int] = None,
    coresByNode: Option[Int] = Some(1),
    qos: Option[String] = None,
    gres: List[Gres] = List(),
    constraints: List[String] = List())

  object impl {

    import Requirement._

    import BatchScheduler.{ BatchJob, output, error }

    def toScript(description: SLURMJobDescription)(uniqId: String) = {
      import description._

      val header = "#!/bin/bash\n"

      val core = Seq(
        "-o " -> Some(output(uniqId)),
        "-e " -> Some(error(uniqId)),
        "-p " -> queue,
        "--mem=" -> memory.map(m ⇒ s"${m.toMBString}M"),
        "--nodes=" -> nodes.map(_.toString),
        "--cpus-per-task=" -> coresByNode.map(_.toString),
        "--time=" -> wallTime.map(_.toHHmmss),
        "--qos=" -> qos,
        "-D " -> Some(workDirectory))

      // must handle empty list separately since it is not done in mkString
      val gresList = gres match {
        case List() ⇒ ""
        case _      ⇒ gres.mkString("#SBATCH --gres=", "--gres=", "")
      }<commit_msg>add ntasks option to SLURM<commit_after>package gridscale

import effectaside._
import gridscale.cluster.{ BatchScheduler, HeadNode, Requirement }
import gridscale.tools._
import monocle.macros._
import squants._
import squants.information._

import scala.language.higherKinds

package object slurm {

  case class Gres(gresName: String, gresValue: Int) {
    override def toString = gresName + ":" + gresValue.toString
  }

  @Lenses case class SLURMJobDescription(
    command: String,
    workDirectory: String,
    queue: Option[String] = None,
    wallTime: Option[Time] = None,
    memory: Option[Information] = None,
    nodes: Option[Int] = None,
    ntasks: Option[Int] = None,
    coresByNode: Option[Int] = Some(1),
    qos: Option[String] = None,
    gres: List[Gres] = List(),
    constraints: List[String] = List())

  object impl {

    import Requirement._

    import BatchScheduler.{ BatchJob, output, error }

    def toScript(description: SLURMJobDescription)(uniqId: String) = {
      import description._

      val header = "#!/bin/bash\n"

      val core = Seq(
        "-o " -> Some(output(uniqId)),
        "-e " -> Some(error(uniqId)),
        "-p " -> queue,
        "--mem=" -> memory.map(m ⇒ s"${m.toMBString}M"),
        "--nodes=" -> nodes.map(_.toString),
        "--ntasks=" -> ntasks.map(_.toString),
        "--cpus-per-task=" -> coresByNode.map(_.toString),
        "--time=" -> wallTime.map(_.toHHmmss),
        "--qos=" -> qos,
        "-D " -> Some(workDirectory))

      // must handle empty list separately since it is not done in mkString
      val gresList = gres match {
        case List() ⇒ ""
        case _      ⇒ gres.mkString("#SBATCH --gres=", "--gres=", "")
      }
{'fraction_non_alphanumeric': 0.14802631578947367, 'fraction_numerical': 0.0005980861244019139, 'mean_word_length': 2.720800889877642, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4223864', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1069, 'n_words': 269}