<commit_before>package brainflight.binary

import collection.mutable.HashMap
import play.api.Play.current
import play.api.Play
import play.Logger
import java.io.{ FileNotFoundException, InputStream, FileInputStream, File }
import brainflight.tools.geometry.Point3D
import models.DataSet

/**
 * Scalable Minds - Brainflight
 * User: tom
 * Date: 10/11/11
 * Time: 12:20 PM
 */

/**
 * A data store implementation which uses the hdd as data storage
 */
class FileDataStore extends DataStore {
  case class DataBlockInformation(
    dataSetId: String,
    point: Point3D,
    resolution: Int )

  lazy val nullBlock = ( for ( x <- 0 to 128 * 128 * 128 ) yield 0.toByte ).toArray

  // defines the maximum count of cached file handles
  val maxCacheSize = 500

  // defines how many file handles are deleted when the limit is reached
  val dropCount = 50

  // try to prevent loading a file multiple times into memory
  var fileCache = new HashMap[DataBlockInformation, Array[Byte]]

  /**
   * Uses the coordinates of a point to calculate the data block the point
   * lays in.
   */
  def extractBlockCoordinates( point: Point3D ) =
    Point3D( point.x / 128, point.y / 128, point.z / 256 )

  /**
   * Load the binary data of the given coordinate from file
   */
  override def load( dataSet: DataSet, resolution: Int )( globalPoint: Point3D ): Byte = {
    if ( dataSet doesContain globalPoint ) {
      val point = extractBlockCoordinates( globalPoint )

      val byteArray: Array[Byte] =
        fileCache.get( DataBlockInformation( dataSet.id, point, resolution ) ) getOrElse ( loadBlock( dataSet, point, resolution ) )

      val zB = ( globalPoint.z % 256 ) / 2
      byteArray( ( zB * 128 * 128 + ( globalPoint.y % 128 ) * 128 + globalPoint.x % 128 ) )
    } else {
      return 0
    }
  }

  /**
   * Loads the due to x,y and z defined block into the cache array and
   * returns it.
   */
  def loadBlock( dataSet: DataSet, point: Point3D, resolution: Int ): Array[Byte] = {
    ensureCacheMaxSize
    val dataBlock =
      try {
        val binaryStream =
          new FileInputStream( createFilename( dataSet, resolution, point ) )
        inputStreamToByteArray( binaryStream )
      } catch {
        case e: FileNotFoundException =>
          Logger.warn( "Block %s not found!".format( createFilename( dataSet, resolution, point ) ) )
          // if the file block isn't found, a nullBlock is associated with 
          // the coordinates
          nullBlock
      }
    fileCache += ( ( DataBlockInformation( dataSet.id, point, resolution ), dataBlock ) )
    println(fileCache.size)<commit_msg>BF-73 added lastUsed cache to FileDataStore<commit_after>package brainflight.binary

import collection.mutable.HashMap
import play.api.Play.current
import play.api.Play
import play.Logger
import java.io.{ FileNotFoundException, InputStream, FileInputStream, File }
import brainflight.tools.geometry.Point3D
import models.DataSet

/**
 * Scalable Minds - Brainflight
 * User: tom
 * Date: 10/11/11
 * Time: 12:20 PM
 */

/**
 * A data store implementation which uses the hdd as data storage
 */
class FileDataStore extends DataStore {
  case class DataBlockInformation(
    dataSetId: String,
    point: Point3D,
    resolution: Int )

  lazy val nullBlock = ( for ( x <- 0 to 128 * 128 * 128 ) yield 0.toByte ).toArray

  // defines the maximum count of cached file handles
  val maxCacheSize = 500

  // defines how many file handles are deleted when the limit is reached
  val dropCount = 50

  // try to prevent loading a file multiple times into memory
  var fileCache = new HashMap[DataBlockInformation, Array[Byte]]
  var lastUsed: Option[Tuple2[DataBlockInformation, Array[Byte]]] = None

  /**
   * Uses the coordinates of a point to calculate the data block the point
   * lays in.
   */
  def extractBlockCoordinates( point: Point3D ) =
    Point3D( point.x / 128, point.y / 128, point.z / 256 )

  def useLastUsed( blockInfo: DataBlockInformation ) = {
    lastUsed.flatMap{
      case (info, data ) if info == blockInfo => 
        Some(data)
      case _ => 
        None
    }
  }  
    
  /**
   * Load the binary data of the given coordinate from file
   */
  override def load( dataSet: DataSet, resolution: Int )( globalPoint: Point3D ): Byte = {
    if ( dataSet doesContain globalPoint ) {
      val point = extractBlockCoordinates( globalPoint )     
      val blockInfo = DataBlockInformation( dataSet.id, point, resolution )
      
      val byteArray: Array[Byte] =
        (useLastUsed( blockInfo ) orElse fileCache.get( blockInfo )) getOrElse ( 
              loadBlock( dataSet, point, resolution ) )
      
      lastUsed = Some(blockInfo -> byteArray)
       
      val zB = ( globalPoint.z % 256 ) / 2
      byteArray( ( zB * 128 * 128 + ( globalPoint.y % 128 ) * 128 + globalPoint.x % 128 ) )
    } else {
      return 0
    }
  }

  /**
   * Loads the due to x,y and z defined block into the cache array and
   * returns it.
   */
  def loadBlock( dataSet: DataSet, point: Point3D, resolution: Int ): Array[Byte] = {
    ensureCacheMaxSize
    val dataBlock =
      try {
        val binaryStream =
          new FileInputStream( createFilename( dataSet, resolution, point ) )
        inputStreamToByteArray( binaryStream )
      } catch {
        case e: FileNotFoundException =>
          Logger.warn( "Block %s not found!".format( createFilename( dataSet, resolution, point ) ) )
          // if the file block isn't found, a nullBlock is associated with 
          // the coordinates
          nullBlock
      }
    fileCache += ( ( DataBlockInformation( dataSet.id, point, resolution ), dataBlock ) )
{'fraction_non_alphanumeric': 0.09111818825194622, 'fraction_numerical': 0.022116065109695684, 'mean_word_length': 2.7762191048764193, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '812338', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1601, 'n_words': 561}