<commit_before>Backbone = require("backbone")
_ = require("lodash")
ErrorHandling = require("../../../libs/error_handling")
{Bucket, NullBucket} = require("./bucket")
ArbitraryCubeAdapter = require("./arbitrary_cube_adapter")
TemporalBucketManager = require("./temporal_bucket_manager")

class Cube

  # Constants
  BUCKET_SIZE_P : 5
  CUBE_SIZE_P   : 7
  LOCAL_ID_SIZE_P : 16
  BUCKET_LENGTH : 0
  ZOOM_STEP_COUNT : 0
  LOOKUP_DEPTH_UP : 0
  LOOKUP_DEPTH_DOWN : 1
  MAXIMUM_BUCKET_COUNT : 5000
  ARBITRARY_MAX_ZOOMSTEP : 2

  EMPTY_MAPPING : null

  arbitraryCube : null
  dataCubes : null
  upperBoundary : null

  buckets : null
  bucketIterator : 0
  bucketCount : 0


  # The cube stores the buckets in a seperate array for each zoomStep. For each
  # zoomStep the cube-array contains the boundaries and an array holding the buckets.
  # The bucket-arrays are initialized large enough to hold the whole cube. Thus no
  # expanding is necessary. bucketCount keeps track of how many buckets are currently
  # in the cube.
  #
  # Each bucket consists of an access-value, the zoomStep and the actual data.
  # The access-values are used for garbage collection. When a bucket is accessed, its
  # access-flag is set to true.
  # When buckets have to be collected, an iterator will loop through the the buckets at the beginning of the queue will be removed
  # from the queue and the access-value will be decreased. If the access-value of a
  # bucket becomes 0, itsis no longer in the access-queue and is least resently used.
  # It is then removed from the cube.


  constructor : (@upperBoundary, @ZOOM_STEP_COUNT, @BIT_DEPTH) ->

    _.extend(this, Backbone.Events)

    @NULL_BUCKET = new NullBucket()

    @LOOKUP_DEPTH_UP = @ZOOM_STEP_COUNT - 1
    @MAX_ZOOM_STEP   = @ZOOM_STEP_COUNT - 1
    @BUCKET_LENGTH   = (1 << @BUCKET_SIZE_P * 3) * (@BIT_DEPTH >> 3)
    @BYTE_OFFSET     = (@BIT_DEPTH >> 3)

    @cubes = []
    @buckets = new Array(@MAXIMUM_BUCKET_COUNT)

    @mapping = @EMPTY_MAPPING
    @currentMapping = @mapping

    # Initializing the cube-arrays with boundaries
    cubeBoundary = [
      Math.ceil(@upperBoundary[0] / (1 << @BUCKET_SIZE_P))
      Math.ceil(@upperBoundary[1] / (1 << @BUCKET_SIZE_P))
      Math.ceil(@upperBoundary[2] / (1 << @BUCKET_SIZE_P))
    ]

    @arbitraryCube = new ArbitraryCubeAdapter(@, cubeBoundary.slice())

    for i in [0...@ZOOM_STEP_COUNT]

      @cubes[i] = {}
      @cubes[i].data = new Array(cubeBoundary[0] * cubeBoundary[1] * cubeBoundary[2])
      @cubes[i].boundary = cubeBoundary.slice()

      cubeBoundary = [
        (cubeBoundary[0] + 1) >> 1
        (cubeBoundary[1] + 1) >> 1
        (cubeBoundary[2] + 1) >> 1
      ]


  initializeWithQueues : (@pullQueue, @pushQueue) ->
    # Due to cyclic references, this method has to be called before the cube is
    # used with the instantiated queues

    @temporalBucketManager = new TemporalBucketManager(@pullQueue, @pushQueue)


  setMapping : (@mapping) ->

    @trigger("mappingChanged")


  setMappingEnabled : (isEnabled) ->

    @currentMapping = if isEnabled then @mapping else @EMPTY_MAPPING
    @trigger("newMapping")


  hasMapping : ->

    return @mapping?


  setMapping : (newMapping) ->

    # Generate fake mapping
    #if not newMapping.length
    #  newMapping = new newMapping.constructor(1 << @BIT_DEPTH)
    #  for i in [0...(1 << @BIT_DEPTH)]
    #    newMapping[i] = if i == 0 then 0 else i + 1

    if @currentMapping == @mapping
      @currentMapping = newMapping
    @mapping = newMapping

    @trigger("newMapping")


  mapId : (idToMap) ->

    return if @currentMapping? and (mappedId = @currentMapping[idToMap])? then mappedId else idToMap


  getArbitraryCube : ->

    return @arbitraryCube


  getVoxelIndexByVoxelOffset : ([x, y, z]) ->

    return x + y * (1 << @BUCKET_SIZE_P) + z * (1 << @BUCKET_SIZE_P * 2)


  getBucketIndexByZoomedAddress : ([x, y, z, zoomStep]) ->

    ErrorHandling.assertExists(@cubes[zoomStep], "Cube for given zoomStep does not exist"
      cubeCount: @cubes.length
      zoomStep: zoomStep
      zoomStepCount: @ZOOM_STEP_COUNT
    )

    boundary = @cubes[zoomStep].boundary

    if  x >= 0 and x < boundary[0] and
        y >= 0 and y < boundary[1] and
        z >= 0 and z < boundary[2]

      return x * boundary[2] * boundary[1] + y * boundary[2] + z

    else

      return undefined


  getBucketByZoomedAddress : (address) ->

    if address[3] >= @ZOOM_STEP_COUNT
      return @NULL_BUCKET

    bucketIndex = @getBucketIndexByZoomedAddress(address)
    cube = @cubes[address[3]].data

    if bucketIndex?
      unless cube[bucketIndex]?
        cube[bucketIndex] = @createBucket(address)
      return cube[bucketIndex]

    return @NULL_BUCKET


  createBucket : (address) ->

    bucket = new Bucket(@BIT_DEPTH, address, @temporalBucketManager)
    bucket.on
      bucketLoaded : => @trigger("bucketLoaded", address)
    @addBucketToGarbageCollection(bucket)
    return bucket


  addBucketToGarbageCollection : (bucket) ->

    if @bucketCount >= @MAXIMUM_BUCKET_COUNT

      for i in [0...(2 * @bucketCount)]

        @bucketIterator = ++@bucketIterator % @MAXIMUM_BUCKET_COUNT
        break if @buckets[@bucketIterator].shouldCollect()

      if not @buckets[@bucketIterator].shouldCollect()
        throw new Error("All buckets have shouldCollect == false permanently")
<commit_msg>#1394 Add Buckets to garbage collection only when thez are pulled<commit_after>Backbone = require("backbone")
_ = require("lodash")
ErrorHandling = require("../../../libs/error_handling")
{Bucket, NullBucket} = require("./bucket")
ArbitraryCubeAdapter = require("./arbitrary_cube_adapter")
TemporalBucketManager = require("./temporal_bucket_manager")

class Cube

  # Constants
  BUCKET_SIZE_P : 5
  CUBE_SIZE_P   : 7
  LOCAL_ID_SIZE_P : 16
  BUCKET_LENGTH : 0
  ZOOM_STEP_COUNT : 0
  LOOKUP_DEPTH_UP : 0
  LOOKUP_DEPTH_DOWN : 1
  MAXIMUM_BUCKET_COUNT : 5000
  ARBITRARY_MAX_ZOOMSTEP : 2

  EMPTY_MAPPING : null

  arbitraryCube : null
  dataCubes : null
  upperBoundary : null

  buckets : null
  bucketIterator : 0
  bucketCount : 0


  # The cube stores the buckets in a seperate array for each zoomStep. For each
  # zoomStep the cube-array contains the boundaries and an array holding the buckets.
  # The bucket-arrays are initialized large enough to hold the whole cube. Thus no
  # expanding is necessary. bucketCount keeps track of how many buckets are currently
  # in the cube.
  #
  # Each bucket consists of an access-value, the zoomStep and the actual data.
  # The access-values are used for garbage collection. When a bucket is accessed, its
  # access-flag is set to true.
  # When buckets have to be collected, an iterator will loop through the the buckets at the beginning of the queue will be removed
  # from the queue and the access-value will be decreased. If the access-value of a
  # bucket becomes 0, itsis no longer in the access-queue and is least resently used.
  # It is then removed from the cube.


  constructor : (@upperBoundary, @ZOOM_STEP_COUNT, @BIT_DEPTH) ->

    _.extend(this, Backbone.Events)

    @NULL_BUCKET = new NullBucket()

    @LOOKUP_DEPTH_UP = @ZOOM_STEP_COUNT - 1
    @MAX_ZOOM_STEP   = @ZOOM_STEP_COUNT - 1
    @BUCKET_LENGTH   = (1 << @BUCKET_SIZE_P * 3) * (@BIT_DEPTH >> 3)
    @BYTE_OFFSET     = (@BIT_DEPTH >> 3)

    @cubes = []
    @buckets = new Array(@MAXIMUM_BUCKET_COUNT)

    @mapping = @EMPTY_MAPPING
    @currentMapping = @mapping

    # Initializing the cube-arrays with boundaries
    cubeBoundary = [
      Math.ceil(@upperBoundary[0] / (1 << @BUCKET_SIZE_P))
      Math.ceil(@upperBoundary[1] / (1 << @BUCKET_SIZE_P))
      Math.ceil(@upperBoundary[2] / (1 << @BUCKET_SIZE_P))
    ]

    @arbitraryCube = new ArbitraryCubeAdapter(@, cubeBoundary.slice())

    for i in [0...@ZOOM_STEP_COUNT]

      @cubes[i] = {}
      @cubes[i].data = new Array(cubeBoundary[0] * cubeBoundary[1] * cubeBoundary[2])
      @cubes[i].boundary = cubeBoundary.slice()

      cubeBoundary = [
        (cubeBoundary[0] + 1) >> 1
        (cubeBoundary[1] + 1) >> 1
        (cubeBoundary[2] + 1) >> 1
      ]


  initializeWithQueues : (@pullQueue, @pushQueue) ->
    # Due to cyclic references, this method has to be called before the cube is
    # used with the instantiated queues

    @temporalBucketManager = new TemporalBucketManager(@pullQueue, @pushQueue)


  setMapping : (@mapping) ->

    @trigger("mappingChanged")


  setMappingEnabled : (isEnabled) ->

    @currentMapping = if isEnabled then @mapping else @EMPTY_MAPPING
    @trigger("newMapping")


  hasMapping : ->

    return @mapping?


  setMapping : (newMapping) ->

    # Generate fake mapping
    #if not newMapping.length
    #  newMapping = new newMapping.constructor(1 << @BIT_DEPTH)
    #  for i in [0...(1 << @BIT_DEPTH)]
    #    newMapping[i] = if i == 0 then 0 else i + 1

    if @currentMapping == @mapping
      @currentMapping = newMapping
    @mapping = newMapping

    @trigger("newMapping")


  mapId : (idToMap) ->

    return if @currentMapping? and (mappedId = @currentMapping[idToMap])? then mappedId else idToMap


  getArbitraryCube : ->

    return @arbitraryCube


  getVoxelIndexByVoxelOffset : ([x, y, z]) ->

    return x + y * (1 << @BUCKET_SIZE_P) + z * (1 << @BUCKET_SIZE_P * 2)


  getBucketIndexByZoomedAddress : ([x, y, z, zoomStep]) ->

    ErrorHandling.assertExists(@cubes[zoomStep], "Cube for given zoomStep does not exist"
      cubeCount: @cubes.length
      zoomStep: zoomStep
      zoomStepCount: @ZOOM_STEP_COUNT
    )

    boundary = @cubes[zoomStep].boundary

    if  x >= 0 and x < boundary[0] and
        y >= 0 and y < boundary[1] and
        z >= 0 and z < boundary[2]

      return x * boundary[2] * boundary[1] + y * boundary[2] + z

    else

      return undefined


  getBucketByZoomedAddress : (address) ->

    if address[3] >= @ZOOM_STEP_COUNT
      return @NULL_BUCKET

    bucketIndex = @getBucketIndexByZoomedAddress(address)
    cube = @cubes[address[3]].data

    if bucketIndex?
      unless cube[bucketIndex]?
        cube[bucketIndex] = @createBucket(address)
      return cube[bucketIndex]

    return @NULL_BUCKET


  createBucket : (address) ->

    bucket = new Bucket(@BIT_DEPTH, address, @temporalBucketManager)
    bucket.on
      bucketLoaded : =>
        @trigger("bucketLoaded", address)
        @addBucketToGarbageCollection(bucket)
    return bucket


  addBucketToGarbageCollection : (bucket) ->

    if @bucketCount >= @MAXIMUM_BUCKET_COUNT

      for i in [0...(2 * @bucketCount)]

        @bucketIterator = ++@bucketIterator % @MAXIMUM_BUCKET_COUNT
        break if @buckets[@bucketIterator].shouldCollect()

      if not @buckets[@bucketIterator].shouldCollect()
        throw new Error("All buckets have shouldCollect == false permanently")

{'fraction_non_alphanumeric': 0.10639674616380107, 'fraction_numerical': 0.012017008689221667, 'mean_word_length': 3.2278233685033215, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '526545', 'n_tokens_mistral': 3801, 'n_tokens_neox': 3470, 'n_words': 964}