<commit_before>package irene

import collection.JavaConversions._
import collection.mutable.{LinkedHashMap, LinkedHashSet, ListBuffer, ArrayBuffer}
import java.nio.charset.Charset
import java.util.{ArrayList, HashMap, Calendar}
import java.util.logging.{Level, Logger}

import com.google.common.io.Files
import com.google.common.css
import com.google.javascript.jscomp
import com.google.javascript.jscomp.JSSourceFile
import com.google.template.soy.SoyFileSet
import com.google.template.soy.jssrc.SoyJsSrcOptions
import org.jsoup.Jsoup
import org.jsoup.nodes.{Element, DataNode, Document, Node, TextNode, Comment}
import org.jsoup.select.NodeVisitor

object ResourceCompiler {

  /** Adds functionality to java.io.File helpful for the compiler */
  class File (pathname: String) extends java.io.File (pathname) {
    lazy val (base, extension) = {
      val path = this getPath
      val idx = path lastIndexOf '.'
      if (idx == -1) (path, "")
      else (path substring (0, idx), path substring (idx, path length))
    }
    lazy val isGen = getName.contains(".min.") || getName.contains("__deps__")
    lazy val isSource = !isGen && exists
    def getSibling (pathname: String): File = new java.io.File (getParentFile, pathname)
  }
  object File {
    implicit def fromJavaFile (f: java.io.File): File = new File (f getPath)
  }

  lazy val logger = Logger getLogger classOf[ResourceCompiler].getName

  /**
   * Builds the resource tree rooted at the given file or directory. If
   * a file is given, it is built into a single unit. If a directory is given,
   * it will recurse, building files that match:
   * - *.html
   * - main.js
   * - DIR.js
   * where DIR is the parent directory of that given file.
   * 
   * @param filepath the file or directory to process
   * @param charset the encoding assumed for all input files
   * @param visitedFiles (optional) if specified, will be filled with all
   *    source files visited during compilation
   */
  def apply (filepath: String, charset: Charset,
             visitedFiles: collection.mutable.Set[File] =
                           collection.mutable.Set[File]()) = {

    def compile (file: File) = {
      (new ResourceCompiler (math.round(System.currentTimeMillis / 1000.0) * 1000L,
                            charset, visitedFiles)
        .fileExtToProcessor(file.extension)(file))
      ()
    }

    def visitDir (dir: File) {
      val targetableJs = dir.getName + ".js"
      for (file <- dir.listFiles map (File fromJavaFile _)) {
        if (file isFile) {
          val name = file.getName
          if (!file.isGen && (file.extension == ".html"
                              || name == "main.js"
                              || name == targetableJs)) {
            compile (file)
          }
        } else if (file isDirectory) {
          visitDir (file)
        }
      }
    }

    val file = new File (filepath)
    if (file isFile) {
      compile (file)
    } else if (file isDirectory) {
      visitDir (file getCanonicalFile)
    }

  }

}
import ResourceCompiler._

class ResourceCompiler (startTime: Long, charset: Charset,
                        visitedFiles: collection.mutable.Set[File]) {

  /**
   * converts "foo.bar" to "foo.min.bar"
   */
  def defaultTarget (f: File) = new File (f.base + ".min" + f.extension)

  def logProcess (f: File, name: String) =
    logger info ("Processing " + f.getPath + " with " + name)
  def logVisit (f: File) = logger info ("Checking " + f.getPath)

  /**
   * Base class for generating resource processors that use JSDoc style annotations
   * for declaring dependencies (and have tools that compile multiple files as a unit)
   */
  abstract class JsDocProcessor extends (File => (File, Long)) {
    def fileExt: String 
    def processName: String
    def compile (deps: Iterable[File], target: File): Unit
    def makeTarget (in: File) = defaultTarget (in)

    def apply (file: File) = {
      val deps = LinkedHashSet[File]()

      def visit (file: File): Long = {
        val (dep, modified) = 
          if (file.extension != fileExt) fileExtToProcessor(file.extension)(file)
          else {
            logVisit (file)
            visitedFiles += file
            (file, (JsDocScraper (file, charset) flatMap (_ flatMap (_ match {
                      case DepTag (path) if !path.contains(".min.") =>
                        Some (visit (file getSibling path))
                      case _ => None
                    }))).foldLeft(file lastModified)(math.max _))
          }
        deps += dep
        modified
      }

      println (deps mkString)

      val target = makeTarget (file)
      val modified = visit (file)
      if (modified > target.lastModified) {
        logProcess (file, processName)<commit_msg>Remove a superfluous print statement<commit_after>package irene

import collection.JavaConversions._
import collection.mutable.{LinkedHashMap, LinkedHashSet, ListBuffer, ArrayBuffer}
import java.nio.charset.Charset
import java.util.{ArrayList, HashMap, Calendar}
import java.util.logging.{Level, Logger}

import com.google.common.io.Files
import com.google.common.css
import com.google.javascript.jscomp
import com.google.javascript.jscomp.JSSourceFile
import com.google.template.soy.SoyFileSet
import com.google.template.soy.jssrc.SoyJsSrcOptions
import org.jsoup.Jsoup
import org.jsoup.nodes.{Element, DataNode, Document, Node, TextNode, Comment}
import org.jsoup.select.NodeVisitor

object ResourceCompiler {

  /** Adds functionality to java.io.File helpful for the compiler */
  class File (pathname: String) extends java.io.File (pathname) {
    lazy val (base, extension) = {
      val path = this getPath
      val idx = path lastIndexOf '.'
      if (idx == -1) (path, "")
      else (path substring (0, idx), path substring (idx, path length))
    }
    lazy val isGen = getName.contains(".min.") || getName.contains("__deps__")
    lazy val isSource = !isGen && exists
    def getSibling (pathname: String): File = new java.io.File (getParentFile, pathname)
  }
  object File {
    implicit def fromJavaFile (f: java.io.File): File = new File (f getPath)
  }

  lazy val logger = Logger getLogger classOf[ResourceCompiler].getName

  /**
   * Builds the resource tree rooted at the given file or directory. If
   * a file is given, it is built into a single unit. If a directory is given,
   * it will recurse, building files that match:
   * - *.html
   * - main.js
   * - DIR.js
   * where DIR is the parent directory of that given file.
   * 
   * @param filepath the file or directory to process
   * @param charset the encoding assumed for all input files
   * @param visitedFiles (optional) if specified, will be filled with all
   *    source files visited during compilation
   */
  def apply (filepath: String, charset: Charset,
             visitedFiles: collection.mutable.Set[File] =
                           collection.mutable.Set[File]()) = {

    def compile (file: File) = {
      (new ResourceCompiler (math.round(System.currentTimeMillis / 1000.0) * 1000L,
                            charset, visitedFiles)
        .fileExtToProcessor(file.extension)(file))
      ()
    }

    def visitDir (dir: File) {
      val targetableJs = dir.getName + ".js"
      for (file <- dir.listFiles map (File fromJavaFile _)) {
        if (file isFile) {
          val name = file.getName
          if (!file.isGen && (file.extension == ".html"
                              || name == "main.js"
                              || name == targetableJs)) {
            compile (file)
          }
        } else if (file isDirectory) {
          visitDir (file)
        }
      }
    }

    val file = new File (filepath)
    if (file isFile) {
      compile (file)
    } else if (file isDirectory) {
      visitDir (file getCanonicalFile)
    }

  }

}
import ResourceCompiler._

class ResourceCompiler (startTime: Long, charset: Charset,
                        visitedFiles: collection.mutable.Set[File]) {

  /**
   * converts "foo.bar" to "foo.min.bar"
   */
  def defaultTarget (f: File) = new File (f.base + ".min" + f.extension)

  def logProcess (f: File, name: String) =
    logger info ("Processing " + f.getPath + " with " + name)
  def logVisit (f: File) = logger info ("Checking " + f.getPath)

  /**
   * Base class for generating resource processors that use JSDoc style annotations
   * for declaring dependencies (and have tools that compile multiple files as a unit)
   */
  abstract class JsDocProcessor extends (File => (File, Long)) {
    def fileExt: String 
    def processName: String
    def compile (deps: Iterable[File], target: File): Unit
    def makeTarget (in: File) = defaultTarget (in)

    def apply (file: File) = {
      val deps = LinkedHashSet[File]()

      def visit (file: File): Long = {
        val (dep, modified) = 
          if (file.extension != fileExt) fileExtToProcessor(file.extension)(file)
          else {
            logVisit (file)
            visitedFiles += file
            (file, (JsDocScraper (file, charset) flatMap (_ flatMap (_ match {
                      case DepTag (path) if !path.contains(".min.") =>
                        Some (visit (file getSibling path))
                      case _ => None
                    }))).foldLeft(file lastModified)(math.max _))
          }
        deps += dep
        modified
      }

      val target = makeTarget (file)
      val modified = visit (file)
      if (modified > target.lastModified) {
        logProcess (file, processName)
{'fraction_non_alphanumeric': 0.10667786644267115, 'fraction_numerical': 0.0023099538009239817, 'mean_word_length': 2.439869989165764, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4142447', 'n_tokens_mistral': 2794, 'n_tokens_neox': 2732, 'n_words': 866}