<commit_before>package processors

import at.ait.dme.magicktiler.image.ImageFormat
import at.ait.dme.magicktiler.MagickTiler
import at.ait.dme.magicktiler.ptif.PTIFConverter
import models.dos.Task
import java.io.File
import play.Play

/**
 *
 * @author <NAME> <<EMAIL>>
 */

object PTIFTilingProcessor extends Processor {

  def process(task: Task, params: Map[String, AnyRef]) {

    // TODO warn / throw error if these ain't set
    val tilesOutputBasePath = new File(Play.configuration.getProperty("dos.tilesOutputBaseDir", "/tmp/tiles"))
    val tilesWorkingBasePath = new File(Play.configuration.getProperty("dos.tilesWorkingBaseDir", "/tmp/tilesWork"))

    def checkOrCreate(dir: File) = dir.exists() || !dir.exists() && dir.mkdir()

    if (!checkOrCreate(tilesOutputBasePath)) {
      error(task, "Cannot find / create tiles output directory '%s'".format(tilesOutputBasePath.getAbsolutePath))
      return
    }
    if (!checkOrCreate(tilesWorkingBasePath)) {
      error(task, "Cannot find / create tiles working directory '%s'".format(tilesWorkingBasePath.getAbsolutePath))
      return
    }

    val p = new File(task.path)
    if (!p.exists()) {
      error(task, "Path '%s' does not exist or is unreachable".format(task.path))
    } else {

      val outputPath = new File(tilesOutputBasePath, p.getName)
      if(outputPath.exists() && outputPath.isDirectory) {
        error(task, "Output directory '%s' already exists, delete it first if you want to re-tile".format(outputPath.getAbsolutePath))
        return
      }
      if(!outputPath.mkdir()) {
        error(task, "Cannot create output directory '%s'".format(outputPath.getAbsolutePath))
        return
      }

      val tiler: MagickTiler = new PTIFConverter()
      tiler.setWorkingDirectory(tilesWorkingBasePath)
      tiler.setTileFormat(ImageFormat.JPEG);
      tiler.setJPEGCompressionQuality(75);
      tiler.setBackgroundColor("#ffffffff");
      tiler.setGeneratePreviewHTML(false);

      val images = p.listFiles().filter(f => isImage(f.getName))

      images foreach {
        i => {
          info(task, "Making PTIF tile for image '%s'".format(i.getAbsolutePath))

          val targetFileName = if (i.getName.indexOf(".") > -1) i.getName.substring(0, i.getName.indexOf(".")) else i.getName
          val targetFile: File = new File(outputPath, targetFileName + ".tif")
          targetFile.createNewFile()

          try {
            tiler.convert(i, targetFile)
          } catch {
            case t => error(task, "Could not create tile for image '%s': %s".format(i.getAbsolutePath, t.getMessage))
          }
        }
      }
    }
  }
}<commit_msg>More verbose tiling log<commit_after>package processors

import at.ait.dme.magicktiler.image.ImageFormat
import at.ait.dme.magicktiler.MagickTiler
import at.ait.dme.magicktiler.ptif.PTIFConverter
import models.dos.Task
import java.io.File
import play.Play

/**
 *
 * @author <NAME> <<EMAIL>>
 */

object PTIFTilingProcessor extends Processor {

  def process(task: Task, params: Map[String, AnyRef]) {

    // TODO warn / throw error if these ain't set
    val tilesOutputBasePath = new File(Play.configuration.getProperty("dos.tilesOutputBaseDir", "/tmp/tiles"))
    val tilesWorkingBasePath = new File(Play.configuration.getProperty("dos.tilesWorkingBaseDir", "/tmp/tilesWork"))

    def checkOrCreate(dir: File) = dir.exists() || !dir.exists() && dir.mkdir()

    if (!checkOrCreate(tilesOutputBasePath)) {
      error(task, "Cannot find / create tiles output directory '%s'".format(tilesOutputBasePath.getAbsolutePath))
      return
    }
    if (!checkOrCreate(tilesWorkingBasePath)) {
      error(task, "Cannot find / create tiles working directory '%s'".format(tilesWorkingBasePath.getAbsolutePath))
      return
    }

    val p = new File(task.path)
    if (!p.exists()) {
      error(task, "Path '%s' does not exist or is unreachable".format(task.path))
    } else {

      val outputPath = new File(tilesOutputBasePath, p.getName)
      if (outputPath.exists() && outputPath.isDirectory) {
        error(task, "Output directory '%s' already exists, delete it first if you want to re-tile".format(outputPath.getAbsolutePath))
        return
      }
      if (!outputPath.mkdir()) {
        error(task, "Cannot create output directory '%s'".format(outputPath.getAbsolutePath))
        return
      }

      val tiler: MagickTiler = new PTIFConverter()
      tiler.setWorkingDirectory(tilesWorkingBasePath)
      tiler.setTileFormat(ImageFormat.JPEG);
      tiler.setJPEGCompressionQuality(75);
      tiler.setBackgroundColor("#ffffffff");
      tiler.setGeneratePreviewHTML(false);

      val images = p.listFiles().filter(f => isImage(f.getName))

      images foreach {
        i => {
          info(task, "Making PTIF tile for image '%s'".format(i.getAbsolutePath))

          val targetFileName = if (i.getName.indexOf(".") > -1) i.getName.substring(0, i.getName.indexOf(".")) else i.getName
          val targetFile: File = new File(outputPath, targetFileName + ".tif")
          targetFile.createNewFile()

          try {
            val tileInfo = tiler.convert(i, targetFile)
            info(task, "Generated PTIF for file " + i.getName + ": " + tileInfo.getImageWidth + "x" + tileInfo.getImageHeight + ", " + tileInfo.getZoomLevels + " zoom levels")
          } catch {
            case t => error(task, "Could not create tile for image '%s': %s".format(i.getAbsolutePath, t.getMessage))
          }
        }
      }
    }
  }
}
{'fraction_non_alphanumeric': 0.127024567788899, 'fraction_numerical': 0.0014558689717925387, 'mean_word_length': 3.179467680608365, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5437274', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1641, 'n_words': 377}