<commit_before>package de.fuberlin.wiwiss.silk.instance

import collection.mutable.ArrayBuffer

/**
 * An instance cache, which caches the instance in memory and allows adding new instances at runtime.
 */
class MemoryInstanceCache(val blockCount : Int = 1, maxPartitionSize : Int = 1000) extends InstanceCache
{
    private var blocks = IndexedSeq.fill(blockCount)(new Block)

    private var instanceCounter = 0

    /**
     * Writes to this cache.
     */
    override def write(instances : Traversable[Instance], blockingFunction : Instance => Set[Int] = _ => Set(0))
    {
        for(instance <- instances)
        {
            add(instance, blockingFunction)
        }
    }

    /**
     * Adds a single instance to the cache.
     */
    private def add(instance : Instance, blockingFunction : Instance => Set[Int])
    {
        for(block <- blockingFunction(instance))
        {
            blocks(block).add(instance)
        }
        instanceCounter += 1
    }

    def clear()
    {
        instanceCounter = 0
        blocks = IndexedSeq.fill(blockCount)(new Block)
    }

    def instanceCount = instanceCounter
<commit_msg>MemoryInstanceCache: Check for URI duplicates<commit_after>package de.fuberlin.wiwiss.silk.instance

import collection.mutable.ArrayBuffer

/**
 * An instance cache, which caches the instance in memory and allows adding new instances at runtime.
 */
class MemoryInstanceCache(val blockCount : Int = 1, maxPartitionSize : Int = 1000) extends InstanceCache
{
    private var blocks = IndexedSeq.fill(blockCount)(new Block)

    private var allInstances = Set[String]()

    private var instanceCounter = 0

    /**
     * Writes to this cache.
     */
    override def write(instances : Traversable[Instance], blockingFunction : Option[Instance => Set[Int]] = None)
    {
        for(instance <- instances)
        {
            add(instance, blockingFunction)
        }
    }

    /**
     * Adds a single instance to the cache.
     */
    private def add(instance : Instance, blockingFunction : Option[Instance => Set[Int]])
    {
        if(!allInstances.contains(instance.uri))
        {
            for(block <- blockingFunction.map(f => f(instance)).getOrElse(Set(0)))
            {
                blocks(block).add(instance)
            }
            allInstances += instance.uri 
            instanceCounter += 1
        }
    }

    def clear()
    {
        instanceCounter = 0
        blocks = IndexedSeq.fill(blockCount)(new Block)
        allInstances = Set[String]()
    }

    def instanceCount = instanceCounter

{'fraction_non_alphanumeric': 0.09918319719953325, 'fraction_numerical': 0.007001166861143524, 'mean_word_length': 2.447721179624665, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3024716', 'n_tokens_mistral': 719, 'n_tokens_neox': 691, 'n_words': 182}