<commit_before>package ml.wolfe

import java.io.{File, FileWriter}
import java.util.Calendar

import com.typesafe.config.{Config, ConfigFactory}

import scala.collection.mutable.ArrayBuffer
import scala.io.Source

/**
 * @author sriedel
 * @author rockt
 */
package object util {
  implicit class RichConfig(conf: Config) {
    import scala.collection.JavaConversions._

    def getStringList(path: String): List[String] =
      conf.getObjectList(path).asInstanceOf[java.util.ArrayList[String]].toList
  }

  object Conf {
    lazy val targetSlots = conf.getString("target-slots").split(",").map(_.trim).toSet
    lazy val parentOutDir = {
      val dir = new File(if (conf.hasPath("outDir")) conf.getString("outDir") else "data/out")
      dir.mkdirs()
      dir
    }
    lazy val outDir = {
      val date = Calendar.getInstance()
      val day = date.get(Calendar.DAY_OF_MONTH)
      val month = date.get(Calendar.MONTH) + 1
      val year = date.get(Calendar.YEAR)
      val hour = date.get(Calendar.HOUR_OF_DAY)
      val min = date.get(Calendar.MINUTE)
      val sec = date.get(Calendar.SECOND)
      val ms = date.get(Calendar.MILLISECOND)

      val dayDir = new File(parentOutDir, "%d_%d_%d".format(day, month, year))
      msDir = new File(dayDir, "run_%d_%d_%d_%d".format(hour, min, sec, ms))

      latest = new File(parentOutDir, "latest")
      if (latest.exists()) {
        latest.delete()
      }
      msDir.mkdirs()

      //write config files to output directory
      for (resource <- addedResources) {
        val lines = Source.fromFile(resource).getLines()
        val fileWriter = new FileWriter(msDir.getAbsolutePath + "/" + resource.split("/").last)
        fileWriter.write(lines.mkString("\n"))
        fileWriter.close()
      }

      createSymbolicLinkToLatest()
      msDir
    }
    val addedResources = new ArrayBuffer[String]
    var msDir: File = _
    var latest: File = _
    private var _conf: Config = ConfigFactory.parseFile(new File("conf/default.conf"))

    def createSymbolicLinkToLatest() =
      Runtime.getRuntime.exec("/bin/ln -sfn %s %s".format(msDir.getAbsolutePath, latest.getAbsolutePath))

    import scala.collection.JavaConversions._

    def add(filePath: String) {
      addedResources += filePath
      //_conf = ConfigFactory.parseResources(resources).withFallback(conf)
      _conf = ConfigFactory.parseFile(new File(filePath)).withFallback(conf)
      _conf.resolve()
    }

    def conf = _conf

    def getString(path: String): String = conf.getString(path)
    def getStringList(path: String): List[String] = conf.getStringList(path).asInstanceOf[java.util.ArrayList[String]].toList
    def getBoolean(path: String): Boolean = conf.getBoolean(path)
    def getInt(path: String): Int = conf.getInt(path)
    def getDouble(path: String): Double = conf.getDouble(path)
    def hasPath(path: String): Boolean = conf.hasPath(path)
  }

  object OverrideConfig {
    def apply(overrideWith: Map[String, Any], pathToNewConf: String = "conf/mf-overridden.conf", pathToOldConf: String = "conf/mf.conf"): String = {
      val conf = ConfigFactory.parseFile(new File(pathToOldConf)).entrySet().toString //string representations
                 .replaceAll("\"\\[|(Simple)?Config[^(]*\\(|\\]\"", "") //remove Config.*(
                 .split("\\),").mkString("\n").drop(1).dropRight(2).split("\n").toList.map(_.trim) //clean up
                 .map(s => s.split("=").head -> s.split("=").tail.mkString("=")) //map to key/value tuples
                 .toMap

      //println(ConfigFactory.parseFile(new File(pathToOldConf)).entrySet().toString)<commit_msg>make symlink creation windows aware<commit_after>package ml.wolfe

import java.io.{IOException, File, FileWriter}
import java.nio.file.{Paths, Path, Files}
import java.util.Calendar

import com.typesafe.config.{Config, ConfigFactory}

import scala.collection.mutable.ArrayBuffer
import scala.io.Source

/**
 * @author sriedel
 * @author rockt
 */
package object util {
  implicit class RichConfig(conf: Config) {
    import scala.collection.JavaConversions._

    def getStringList(path: String): List[String] =
      conf.getObjectList(path).asInstanceOf[java.util.ArrayList[String]].toList
  }

  object Conf {
    lazy val targetSlots = conf.getString("target-slots").split(",").map(_.trim).toSet
    lazy val parentOutDir = {
      val dir = new File(if (conf.hasPath("outDir")) conf.getString("outDir") else "data/out")
      dir.mkdirs()
      dir
    }
    lazy val outDir = {
      val date = Calendar.getInstance()
      val day = date.get(Calendar.DAY_OF_MONTH)
      val month = date.get(Calendar.MONTH) + 1
      val year = date.get(Calendar.YEAR)
      val hour = date.get(Calendar.HOUR_OF_DAY)
      val min = date.get(Calendar.MINUTE)
      val sec = date.get(Calendar.SECOND)
      val ms = date.get(Calendar.MILLISECOND)

      val dayDir = new File(parentOutDir, "%d_%d_%d".format(day, month, year))
      msDir = new File(dayDir, "run_%d_%d_%d_%d".format(hour, min, sec, ms))

      latest = new File(parentOutDir, "latest")
      if (latest.exists()) {
        latest.delete()
      }
      msDir.mkdirs()

      //write config files to output directory
      for (resource <- addedResources) {
        val lines = Source.fromFile(resource).getLines()
        val fileWriter = new FileWriter(msDir.getAbsolutePath + "/" + resource.split("/").last)
        fileWriter.write(lines.mkString("\n"))
        fileWriter.close()
      }

      createSymbolicLinkToLatest()
      msDir
    }
    val addedResources = new ArrayBuffer[String]
    var msDir: File = _
    var latest: File = _
    private var _conf: Config = ConfigFactory.parseFile(new File("conf/default.conf"))

    def createSymbolicLinkToLatest() =
      try {
        Files.createLink(Paths.get(latest.getAbsolutePath), Paths.get(msDir.getAbsolutePath))
      } catch {
        case e: IOException => println("Can't create link")
      }
      //Runtime.getRuntime.exec("/bin/ln -sfn %s %s".format(msDir.getAbsolutePath, latest.getAbsolutePath))

    import scala.collection.JavaConversions._

    def add(filePath: String) {
      addedResources += filePath
      //_conf = ConfigFactory.parseResources(resources).withFallback(conf)
      _conf = ConfigFactory.parseFile(new File(filePath)).withFallback(conf)
      _conf.resolve()
    }

    def conf = _conf

    def getString(path: String): String = conf.getString(path)
    def getStringList(path: String): List[String] = conf.getStringList(path).asInstanceOf[java.util.ArrayList[String]].toList
    def getBoolean(path: String): Boolean = conf.getBoolean(path)
    def getInt(path: String): Int = conf.getInt(path)
    def getDouble(path: String): Double = conf.getDouble(path)
    def hasPath(path: String): Boolean = conf.hasPath(path)
  }

  object OverrideConfig {
    def apply(overrideWith: Map[String, Any], pathToNewConf: String = "conf/mf-overridden.conf", pathToOldConf: String = "conf/mf.conf"): String = {
      val conf = ConfigFactory.parseFile(new File(pathToOldConf)).entrySet().toString //string representations
                 .replaceAll("\"\\[|(Simple)?Config[^(]*\\(|\\]\"", "") //remove Config.*(
                 .split("\\),").mkString("\n").drop(1).dropRight(2).split("\n").toList.map(_.trim) //clean up
                 .map(s => s.split("=").head -> s.split("=").tail.mkString("=")) //map to key/value tuples
                 .toMap

      //println(ConfigFactory.parseFile(new File(pathToOldConf)).entrySet().toString)
{'fraction_non_alphanumeric': 0.1426856914255234, 'fraction_numerical': 0.0008001066808907855, 'mean_word_length': 3.2253521126760565, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2364391', 'n_tokens_mistral': 2359, 'n_tokens_neox': 2339, 'n_words': 470}