<commit_before>/**
 * TriGraM translator
 */
package core

import scalax.file.Path
import scalax.file.PathSet
import java.io.FileOutputStream
import com.hp.hpl.jena.rdf.model._
import com.hp.hpl.jena.vocabulary.{ RDF, RDFS }
import com.hp.hpl.jena.util.FileManager
import util.Logging

/**
 * @author ShiZhan
 * translate various resources to semantic model
 */
object Translator extends Logging {

  type Modeler = String => Model
  type ModelerMap = Map[String, (Modeler, String)]
  private val modelerMap: ModelerMap = Map(
    "directory" -> (modelDirectory, "Translate directory structure into TriGraM model"))
  private val modelerMapDefault = (modelUnkown _, null)

  def modelDirectory(name: String) = {
    val p = Path(name)
    if (p.isDirectory) {
      logger.info("creating model for root directory [%s]".format(p.name))

      val ps = p ***
      val m = ModelFactory.createDefaultModel
      val n = m.createResource
      for (i <- ps) {
        println("[%s] in [%s]: %d|%d|%s|%s|%s".format(
          i.name, i.parent.get.name, if(i.size.nonEmpty) i.size.get else 0,
          i.lastModified, i.canRead, i.canWrite, i.canExecute))
      }

      m
    } else {
      logger.info("[%s] is not a directory".format(p.toString))
<commit_msg>tidy and fix<commit_after>/**
 * TriGraM translator
 */
package core

import scalax.file.{ Path, PathSet }
import java.io.FileOutputStream
import com.hp.hpl.jena.rdf.model._
import com.hp.hpl.jena.vocabulary.{ RDF, RDFS }
import com.hp.hpl.jena.util.FileManager
import util.Logging

/**
 * @author ShiZhan
 * translate various resources to semantic model
 */
object Translator extends Logging {

  type Modeler = String => Model
  type ModelerMap = Map[String, (Modeler, String)]
  private val modelerMap: ModelerMap = Map(
    "directory" -> (modelDirectory, "Translate directory structure into TriGraM model"))
  private val modelerMapDefault = (modelUnkown _, null)

  def modelDirectory(name: String) = {
    val p = Path(name)
    if (p.isDirectory) {
      logger.info("creating model for root directory [%s]".format(p.name))

      val ps = p ***
      val m = ModelFactory.createDefaultModel
      val n = m.createResource
      for (i <- ps) {
        println("[%s] in [%s]: %d|%d|%s|%s|%s".format(
          i.name, i.parent.get.name, if (i.size.nonEmpty) i.size.get else 0,
          i.lastModified, i.canRead, i.canWrite, i.canExecute))
      }

      m
    } else {
      logger.info("[%s] is not a directory".format(p.name))

{'fraction_non_alphanumeric': 0.13341346153846154, 'fraction_numerical': 0.0008012820512820513, 'mean_word_length': 3.1547420965058235, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2348224', 'n_tokens_mistral': 864, 'n_tokens_neox': 839, 'n_words': 217}