<commit_before>package opennlp.scalabha.util

import java.io.BufferedWriter
import java.io.FileWriter
import java.io.BufferedReader
import java.io.FileReader
import java.io.File
import scala.io.BufferedSource
import scala.io.Source
import java.io.Writer

object FileUtils {
  var FILE_SEPARATOR = System.getProperty("file.separator")

  //
  // Path string manipulation
  //

  def pathjoin(parts: String*): String =
    (parts.dropRight(1).filter(_.nonEmpty).map(getPathNameStripEndSlash) :+ parts.last).mkString(File.separator)

  def getPathParent(path: String): String =
    path.split(FILE_SEPARATOR).dropRight(1).mkString(FILE_SEPARATOR)

  def getPathNameStripEndSlash(str: String): String =
    ((FILE_SEPARATOR + "*$").r).replaceFirstIn(str, "")

  def trimSlashes(str: String): String =
    (("^%s*|%s*$".format(FILE_SEPARATOR, FILE_SEPARATOR).r).replaceAllIn(str, ""))

  def getStrippedOutputFileName(outputPath: String, newSubdirectories: String, inputBaseName: String): String =
    List(getPathNameStripEndSlash(outputPath), trimSlashes(newSubdirectories), trimSlashes(inputBaseName)).filter((s) => s != "").mkString(FILE_SEPARATOR)

  //
  // Basic file utilities
  //

  def getName(filepath: String) =
    filepath.split(FILE_SEPARATOR).last

  def exists(filename: String) =
    new File(filename).exists()

  def mkdir(filename: String) {
    val f = new File(filename)
    if (!f.exists) f.mkdir()
  }

  def mkdirpath(dirpath: String) {
    val parts = dirpath.split(FILE_SEPARATOR)
    for (i <- 1 to parts.size)
      mkdir(parts.take(i).mkString("", "/", "/"))
  }

  def remove(filename: String) =
    new File(filename).delete()

  def mktemp(prefix: String = "temp-", suffix: String = ""): String = {
    val f = File.createTempFile(prefix, suffix)
    f.delete()
    f.getAbsolutePath
  }

  def findBinary(name: String, binDir: Option[String] = None, envar: Option[String] = None, verbose: Boolean = false): String = {
    val checked = collection.mutable.Buffer[String]()

    if (binDir.isDefined) {
      val path = binDir.get + "/" + name
      if (FileUtils.exists(path))<commit_msg>added a recursive `findAllFiles` function<commit_after>package opennlp.scalabha.util

import java.io.BufferedWriter
import java.io.FileWriter
import java.io.BufferedReader
import java.io.FileReader
import java.io.File
import scala.io.BufferedSource
import scala.io.Source
import java.io.Writer

object FileUtils {
  var FILE_SEPARATOR = System.getProperty("file.separator")

  //
  // Path string manipulation
  //

  def pathjoin(parts: String*): String =
    (parts.dropRight(1).filter(_.nonEmpty).map(getPathNameStripEndSlash) :+ parts.last).mkString(File.separator)

  def getPathParent(path: String): String =
    path.split(FILE_SEPARATOR).dropRight(1).mkString(FILE_SEPARATOR)

  def getPathNameStripEndSlash(str: String): String =
    ((FILE_SEPARATOR + "*$").r).replaceFirstIn(str, "")

  def trimSlashes(str: String): String =
    (("^%s*|%s*$".format(FILE_SEPARATOR, FILE_SEPARATOR).r).replaceAllIn(str, ""))

  def getStrippedOutputFileName(outputPath: String, newSubdirectories: String, inputBaseName: String): String =
    List(getPathNameStripEndSlash(outputPath), trimSlashes(newSubdirectories), trimSlashes(inputBaseName)).filter((s) => s != "").mkString(FILE_SEPARATOR)

  //
  // Basic file utilities
  //

  def getName(filepath: String) =
    filepath.split(FILE_SEPARATOR).last

  def exists(filename: String) =
    new File(filename).exists()

  def mkdir(filename: String) {
    val f = new File(filename)
    if (!f.exists) f.mkdir()
  }

  def mkdirpath(dirpath: String) {
    val parts = dirpath.split(FILE_SEPARATOR)
    for (i <- 1 to parts.size)
      mkdir(parts.take(i).mkString("", "/", "/"))
  }

  def remove(filename: String) =
    new File(filename).delete()

  def mktemp(prefix: String = "temp-", suffix: String = ""): String = {
    val f = File.createTempFile(prefix, suffix)
    f.delete()
    f.getAbsolutePath
  }

  def findAllFiles(topDir: String, path: Vector[String] = Vector()): Set[Vector[String]] = {
    new File(pathjoin((topDir +: path): _*)).listFiles.flatMap { f =>
      if (f.isDirectory)
        findAllFiles(topDir, path :+ f.getName)
      else
        Set(path :+ f.getName)
    }.toSet
  }

  def findBinary(name: String, binDir: Option[String] = None, envar: Option[String] = None, verbose: Boolean = false): String = {
    val checked = collection.mutable.Buffer[String]()

    if (binDir.isDefined) {
      val path = binDir.get + "/" + name
      if (FileUtils.exists(path))
{'fraction_non_alphanumeric': 0.1441105505593332, 'fraction_numerical': 0.0013160780872998464, 'mean_word_length': 4.112107623318385, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5826411', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1522, 'n_words': 300}