<commit_before>/*
 * Copyright (C) 20011-2014 Scalable minds UG (haftungsbeschränkt) & Co. KG. <http://scm.io>
 */
package braingames.binary.repository

import scalax.file.{PathSet, PathMatcher, Path}
import braingames.binary.models._
import javax.imageio.ImageIO
import braingames.geometry.{BoundingBox, Scale, Point3D}
import java.io._
import braingames.binary.store.{FileDataStore, DataStore}
import braingames.binary.Logger._
import java.awt.image.{BufferedImage, DataBufferByte, DataBufferInt}
import braingames.util.ProgressTracking.ProgressTracker
import scala.collection.JavaConversions._
import javax.imageio.spi.IIORegistry
import com.twelvemonkeys.imageio.plugins.tiff.TIFFImageReaderSpi
import scala.Some
import braingames.binary.models.UnusableDataSource
import net.liftweb.common.Full
import scala.concurrent.Future
import braingames.util.BlockedArray3D
import play.api.libs.concurrent.Execution.Implicits._
import scala.util.matching.Regex

object TiffDataSourceType extends DataSourceType with TiffDataSourceTypeHandler {
  val name = "tiff"

  val fileExtension = "tif"
}


object KnossosMultiResCreator {
  val CubeSize = 128

  def fileSize(bytesPerElement: Int) = CubeSize * CubeSize * CubeSize * bytesPerElement

  val Parallelism = 4

  val InterpolationNeighbours = Array((0,0,0), (0,0,1), (0,1,0), (0,1,1), (1,0,0), (1,0,1), (1,1,0), (1,1,1)).map(Point3D.apply)

  @inline
  private def byteToUnsignedInt(b: Byte): Int = 0xff & b.asInstanceOf[Int]

  private def downScale(data: BlockedArray3D[Byte], width: Int, height: Int, depth: Int, bytesPerElement: Int ) = {
    // must be super fast is it is called for each pixel
    @inline
    def average(elements: Array[Array[Byte]]) = {
      val sum = Array.fill(bytesPerElement)(0)
      val result = new Array[Byte](bytesPerElement)

      var i = 0
      while(i < bytesPerElement){
        var idx = 0
        while(idx < elements.size){
          sum(i) = sum(i) + byteToUnsignedInt(elements(idx)(i))
          idx += 1
        }
        result(i) = (sum(i) / elements.size).toByte
        i += 1
      }
      result
    }

    val size = width * height * depth
    val result = new Array[Byte](size * bytesPerElement)
    var idx = 0
    while(idx < size){
      val base = Point3D(idx % width, idx / width % height, idx / width / height)
      val points = InterpolationNeighbours.map{ movement =>
        data(base.scale(2).move(movement))
      }
      average(points).copyToArray(result, idx * bytesPerElement)
      idx += 1
    }
    result
  }

  private def loadCubes(dataStore: FileDataStore, target: Path, dataSetId: String, start: Point3D, resolution: Int, fileSize: Int, neighbours: Array[Point3D]): Future[List[Array[Byte]]] = {
    Future.traverse(neighbours.toList){ movement =>
      val cubePosition = start.move(movement)
      dataStore.load(target, dataSetId, resolution, cubePosition, fileSize).map{
        case Full(data) =>
          data.padTo(fileSize, 0.toByte)
        case _ =>
          Array.fill(fileSize)(0.toByte)
      }
    }
  }

  def createResolutions(source: Path, target: Path, dataSetId: String, bytesPerElement: Int, baseResolution: Int, resolutions: Int, boundingBox: BoundingBox): Future[_] = {
    def createNextResolution(resolution: Int) = {
      val targetResolution = resolution * 2
      logger.info(s"About to create resolution $targetResolution for $dataSetId")
      val dataStore = new FileDataStore
      val points = for {
        x <- boundingBox.topLeft.x.to(boundingBox.bottomRight.x, CubeSize * targetResolution)
        y <- boundingBox.topLeft.y.to(boundingBox.bottomRight.y, CubeSize * targetResolution)
        z <- boundingBox.topLeft.z.to(boundingBox.bottomRight.z, CubeSize * targetResolution)
      } yield Point3D(x,y,z)

      val baseScale = 1.toFloat / CubeSize / resolution
      val targetScale = 1.toFloat / CubeSize / targetResolution

      val numberPerGroup = (points.size.toFloat / Parallelism).ceil.toInt

      Future.traverse(points.grouped(numberPerGroup)){ ps => ps.foldLeft(
        Future.successful[Any](0)){
          case (f, p) => f.flatMap{ _ =>
            val base = p.scale(baseScale)
            val goal = p.scale(targetScale)
            loadCubes(dataStore, target, dataSetId, base, resolution, fileSize(bytesPerElement), InterpolationNeighbours).flatMap{ cubes =>
              val block = BlockedArray3D[Byte](cubes.toVector, CubeSize, CubeSize, CubeSize, 2, 2, 2, bytesPerElement, 0)
              val data = downScale(block, CubeSize, CubeSize, CubeSize, bytesPerElement)
              dataStore.save(target, dataSetId, targetResolution, goal, data)
            }
          }
        }
      }
    }

    val resolutionsToCreate = List.fill(resolutions - 2)(2).scanLeft(baseResolution)(_ * _)
    resolutionsToCreate.foldLeft(Future.successful[Any](1)){
      case (previous, resolution) =>
        previous.flatMap(_ => createNextResolution(resolution))
    }
  }
}

case class TiffLayer(layer: Int, width: Int, height: Int, depth: Int, bytesPerPixel: Int, tiffs: Iterator[RawImage])

case class RawImage(width: Int, height: Int, bytesPerPixel: Int, data: Array[Byte])

case class StackInfo(boundingBox: BoundingBox, bytesPerPixel: Int)

trait TiffDataSourceTypeHandler extends DataSourceTypeHandler {
  val Target = "target"

  val LayerRxs = Seq(
    "_c([0-9]+)"r,
    "_ch([0-9]+)"r
  )

  val DefaultScale = Scale(200, 200, 200)

  val DefaultLayerType = DataLayer.COLOR

  val DefaultLayer = 1

  val Resolutions = List(1, 2, 4, 8, 16, 32, 64, 128)

  // Data points in each direction of a cube in the knossos cube structure
  val CubeSize = 128

  // must be a divisor of cubeSize
  val ContainerSize = 128

  registerTiffProvider()

  protected def registerTiffProvider() = {
    // sometimes there are problems with ImageIO finding the TiffImageReader
    // this should make sure the ImageReader is registered and can be used
    logger.info("Registering tiff provider")
    ImageIO.scanForPlugins()
    val registry = IIORegistry.getDefaultInstance()
    registry.registerServiceProvider(new TIFFImageReaderSpi())
    logger.info("Finished registering tiff provider")
  }

  protected def prepareTargetPath(target: Path): Unit = {
    target.deleteRecursively()
    target.createDirectory()
  }

  protected def elementClass(bytesPerPixel: Int) =
    s"uint${bytesPerPixel * 8}"

  def importDataSource(unusableDataSource: UnusableDataSource, progress: ProgressTracker): Option[DataSource] = {
    val target = (unusableDataSource.sourceFolder / Target).toAbsolute

    prepareTargetPath(target)

    val layers =
      DataLayer("segmentation", "segmentation", (target / "segmentation").path, None, "uint16") ::
      convertToKnossosStructure(unusableDataSource.id, unusableDataSource.sourceFolder, target, progress).toList

    Some(DataSource(
      unusableDataSource.id,
      target.path,
      DefaultScale,
      dataLayers = layers))
  }

  protected def extractImageInfo(tiffs: List[Path]): Option[RawImage] = {
    tiffs match {
      case head :: tail =>
        tiffToRawImage(head) match {
          case Some(tia) => Some(tia)
          case _ => extractImageInfo(tail)
        }
      case _ =>
        None
    }
  }

  def layerFromFileName(tiffFile: Path) = {
    def extractLayer(rs: Seq[Regex]): Int = {
      rs match{
        case r :: tail =>
          r.findFirstMatchIn(tiffFile.path).map(_.group(1).toInt) getOrElse extractLayer(tail)
        case _ =>
          DefaultLayer
      }<commit_msg>fixed missing sections for fake segmentation layer<commit_after>/*
 * Copyright (C) 20011-2014 Scalable minds UG (haftungsbeschränkt) & Co. KG. <http://scm.io>
 */
package braingames.binary.repository

import scalax.file.{PathSet, PathMatcher, Path}
import braingames.binary.models._
import javax.imageio.ImageIO
import braingames.geometry.{BoundingBox, Scale, Point3D}
import java.io._
import braingames.binary.store.{FileDataStore, DataStore}
import braingames.binary.Logger._
import java.awt.image.{BufferedImage, DataBufferByte, DataBufferInt}
import braingames.util.ProgressTracking.ProgressTracker
import scala.collection.JavaConversions._
import javax.imageio.spi.IIORegistry
import com.twelvemonkeys.imageio.plugins.tiff.TIFFImageReaderSpi
import scala.Some
import braingames.binary.models.UnusableDataSource
import net.liftweb.common.Full
import scala.concurrent.Future
import braingames.util.BlockedArray3D
import play.api.libs.concurrent.Execution.Implicits._
import scala.util.matching.Regex

object TiffDataSourceType extends DataSourceType with TiffDataSourceTypeHandler {
  val name = "tiff"

  val fileExtension = "tif"
}


object KnossosMultiResCreator {
  val CubeSize = 128

  def fileSize(bytesPerElement: Int) = CubeSize * CubeSize * CubeSize * bytesPerElement

  val Parallelism = 4

  val InterpolationNeighbours = Array((0,0,0), (0,0,1), (0,1,0), (0,1,1), (1,0,0), (1,0,1), (1,1,0), (1,1,1)).map(Point3D.apply)

  @inline
  private def byteToUnsignedInt(b: Byte): Int = 0xff & b.asInstanceOf[Int]

  private def downScale(data: BlockedArray3D[Byte], width: Int, height: Int, depth: Int, bytesPerElement: Int ) = {
    // must be super fast is it is called for each pixel
    @inline
    def average(elements: Array[Array[Byte]]) = {
      val sum = Array.fill(bytesPerElement)(0)
      val result = new Array[Byte](bytesPerElement)

      var i = 0
      while(i < bytesPerElement){
        var idx = 0
        while(idx < elements.size){
          sum(i) = sum(i) + byteToUnsignedInt(elements(idx)(i))
          idx += 1
        }
        result(i) = (sum(i) / elements.size).toByte
        i += 1
      }
      result
    }

    val size = width * height * depth
    val result = new Array[Byte](size * bytesPerElement)
    var idx = 0
    while(idx < size){
      val base = Point3D(idx % width, idx / width % height, idx / width / height)
      val points = InterpolationNeighbours.map{ movement =>
        data(base.scale(2).move(movement))
      }
      average(points).copyToArray(result, idx * bytesPerElement)
      idx += 1
    }
    result
  }

  private def loadCubes(dataStore: FileDataStore, target: Path, dataSetId: String, start: Point3D, resolution: Int, fileSize: Int, neighbours: Array[Point3D]): Future[List[Array[Byte]]] = {
    Future.traverse(neighbours.toList){ movement =>
      val cubePosition = start.move(movement)
      dataStore.load(target, dataSetId, resolution, cubePosition, fileSize).map{
        case Full(data) =>
          data.padTo(fileSize, 0.toByte)
        case _ =>
          Array.fill(fileSize)(0.toByte)
      }
    }
  }

  def createResolutions(source: Path, target: Path, dataSetId: String, bytesPerElement: Int, baseResolution: Int, resolutions: Int, boundingBox: BoundingBox): Future[_] = {
    def createNextResolution(resolution: Int) = {
      val targetResolution = resolution * 2
      logger.info(s"About to create resolution $targetResolution for $dataSetId")
      val dataStore = new FileDataStore
      val points = for {
        x <- boundingBox.topLeft.x.to(boundingBox.bottomRight.x, CubeSize * targetResolution)
        y <- boundingBox.topLeft.y.to(boundingBox.bottomRight.y, CubeSize * targetResolution)
        z <- boundingBox.topLeft.z.to(boundingBox.bottomRight.z, CubeSize * targetResolution)
      } yield Point3D(x,y,z)

      val baseScale = 1.toFloat / CubeSize / resolution
      val targetScale = 1.toFloat / CubeSize / targetResolution

      val numberPerGroup = (points.size.toFloat / Parallelism).ceil.toInt

      Future.traverse(points.grouped(numberPerGroup)){ ps => ps.foldLeft(
        Future.successful[Any](0)){
          case (f, p) => f.flatMap{ _ =>
            val base = p.scale(baseScale)
            val goal = p.scale(targetScale)
            loadCubes(dataStore, target, dataSetId, base, resolution, fileSize(bytesPerElement), InterpolationNeighbours).flatMap{ cubes =>
              val block = BlockedArray3D[Byte](cubes.toVector, CubeSize, CubeSize, CubeSize, 2, 2, 2, bytesPerElement, 0)
              val data = downScale(block, CubeSize, CubeSize, CubeSize, bytesPerElement)
              dataStore.save(target, dataSetId, targetResolution, goal, data)
            }
          }
        }
      }
    }

    val resolutionsToCreate = List.fill(resolutions - 2)(2).scanLeft(baseResolution)(_ * _)
    resolutionsToCreate.foldLeft(Future.successful[Any](1)){
      case (previous, resolution) =>
        previous.flatMap(_ => createNextResolution(resolution))
    }
  }
}

case class TiffLayer(layer: Int, width: Int, height: Int, depth: Int, bytesPerPixel: Int, tiffs: Iterator[RawImage])

case class RawImage(width: Int, height: Int, bytesPerPixel: Int, data: Array[Byte])

case class StackInfo(boundingBox: BoundingBox, bytesPerPixel: Int)

trait TiffDataSourceTypeHandler extends DataSourceTypeHandler {
  val Target = "target"

  val LayerRxs = Seq(
    "_c([0-9]+)"r,
    "_ch([0-9]+)"r
  )

  val DefaultScale = Scale(200, 200, 200)

  val DefaultLayerType = DataLayer.COLOR

  val DefaultLayer = 1

  val Resolutions = List(1, 2, 4, 8, 16, 32, 64, 128)

  // Data points in each direction of a cube in the knossos cube structure
  val CubeSize = 128

  // must be a divisor of cubeSize
  val ContainerSize = 128

  registerTiffProvider()

  protected def registerTiffProvider() = {
    // sometimes there are problems with ImageIO finding the TiffImageReader
    // this should make sure the ImageReader is registered and can be used
    logger.info("Registering tiff provider")
    ImageIO.scanForPlugins()
    val registry = IIORegistry.getDefaultInstance()
    registry.registerServiceProvider(new TIFFImageReaderSpi())
    logger.info("Finished registering tiff provider")
  }

  protected def prepareTargetPath(target: Path): Unit = {
    target.deleteRecursively()
    target.createDirectory()
  }

  protected def elementClass(bytesPerPixel: Int) =
    s"uint${bytesPerPixel * 8}"

  def importDataSource(unusableDataSource: UnusableDataSource, progress: ProgressTracker): Option[DataSource] = {
    val target = (unusableDataSource.sourceFolder / Target).toAbsolute

    prepareTargetPath(target)

    val segmentationLayer = DataLayer("segmentation", "segmentation", (target / "segmentation").path, None, "uint16", None, List(
      DataLayerSection("segmentation", "segmentation", List(1), BoundingBox(Point3D(0,0,0),0,0,0), BoundingBox(Point3D(0,0,0),0,0,0))
    ))

    val layers =
      segmentationLayer ::
      convertToKnossosStructure(unusableDataSource.id, unusableDataSource.sourceFolder, target, progress).toList

    Some(DataSource(
      unusableDataSource.id,
      target.path,
      DefaultScale,
      dataLayers = layers))
  }

  protected def extractImageInfo(tiffs: List[Path]): Option[RawImage] = {
    tiffs match {
      case head :: tail =>
        tiffToRawImage(head) match {
          case Some(tia) => Some(tia)
          case _ => extractImageInfo(tail)
        }
      case _ =>
        None
    }
  }

  def layerFromFileName(tiffFile: Path) = {
    def extractLayer(rs: Seq[Regex]): Int = {
      rs match{
        case r :: tail =>
          r.findFirstMatchIn(tiffFile.path).map(_.group(1).toInt) getOrElse extractLayer(tail)
        case _ =>
          DefaultLayer
      }
{'fraction_non_alphanumeric': 0.11096563011456628, 'fraction_numerical': 0.014729950900163666, 'mean_word_length': 3.4536443148688045, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1464235', 'n_tokens_mistral': 4971, 'n_tokens_neox': 4706, 'n_words': 1118}