<commit_before>package latis.util

import java.io.File
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import scala.collection.mutable.ArrayBuffer
import java.io.InputStream
import java.net.URL
import java.nio.file.FileSystems

/**
 * Utility methods for working with files.
 * 
 * This object is an analog of FileUtils. Specifically, it is where I'm
 * putting file-utility type methods that rely on the nio package in
 * Java7. Since we're still on Java6 for the time being and these
 * methods are just experimental for now, it makes sense to keep them
 * separate. In the future when Java7/nio becomes standard enough, it
 * would be nice to merge these two classes.
 */
object FileUtilsNio {
  
  /**
   * Private inner class used by listAllFilesWithSize
   */
  private class FileSizeAccumulatorVisitor(basePath: String, buffer: ArrayBuffer[String]) extends SimpleFileVisitor[Path] {
    override def visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult = {
      val file = new File(path.toString())
      // buffer += file.getPath.drop(basePath.length+1)+","+file.length()
      buffer += file.getName+","+file.length()
      return FileVisitResult.CONTINUE;
      
    }
  }
  
  /**<commit_msg>Add back support for subdirectories in FileUtilsNio.<commit_after>package latis.util

import java.io.File
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import scala.collection.mutable.ArrayBuffer
import java.io.InputStream
import java.net.URL
import java.nio.file.FileSystems

/**
 * Utility methods for working with files.
 * 
 * This object is an analog of FileUtils. Specifically, it is where I'm
 * putting file-utility type methods that rely on the nio package in
 * Java7. Since we're still on Java6 for the time being and these
 * methods are just experimental for now, it makes sense to keep them
 * separate. In the future when Java7/nio becomes standard enough, it
 * would be nice to merge these two classes.
 */
object FileUtilsNio {
  
  /**
   * Private inner class used by listAllFilesWithSize
   */
  private class FileSizeAccumulatorVisitor(basePath: String, buffer: ArrayBuffer[String]) extends SimpleFileVisitor[Path] {
    override def visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult = {
      val file = new File(path.toString())
      // buffer += file.getPath.drop(basePath.length+1) + "," + file.length()
      buffer += getFileNameWithSubdirectory(basePath, file) + "," + file.length()
      return FileVisitResult.CONTINUE;
      
    }
  }
  
  /*
   * Remove the leading basePath and '/' from the file path,
   * or just basePath if there is no '/'. Do not remove first
   * char of file name or subdirectory path.
   */
  def getFileNameWithSubdirectory(basePath: String, file: File): String = {
    if (file.getPath.length > basePath.length && file.getPath.charAt(basePath.length) != '/') 
      file.getPath.drop(basePath.length) 
    else 
      file.getPath.drop(basePath.length+1)
  }
  
  /**
{'fraction_non_alphanumeric': 0.09470158343483556, 'fraction_numerical': 0.0027405602923264312, 'mean_word_length': 3.795620437956204, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7049528', 'n_tokens_mistral': 956, 'n_tokens_neox': 940, 'n_words': 305}