<commit_before>package net.serverpeon.twitcharchiver.fx.network

import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater
import java.util.concurrent.atomic.AtomicLongFieldUpdater

class ConcurrentProgressTracker {
    @Volatile private var totalSize: Long = 0
    @Volatile private var downloadedSize: Long = 0
    @Volatile private var activeDownloads: Int = 0
    private val objectKeys: MutableSet<Any> = Collections.newSetFromMap(ConcurrentHashMap())
    private var alreadyDownloaded = 0
    private var totalExpectedDownloads = -1

    fun addAndGetProgress(delta: Long, resolution: Long): Long = calculateProgress(resolution, updateProgress(delta))

    fun progress(resolution: Long): Long = calculateProgress(resolution, DOWNLOADED_SIZE_UPDATER.get(this))

    fun modifyExpectedTotal(delta: Long, key: Any) {
        TOTAL_SIZE_UPDATER.addAndGet(this, delta)
        if (objectKeys.add(key)) {
            ACTIVE_DOWNLOADS_UPDATER.incrementAndGet(this)
        }
    }

    private fun calculateProgress(resolution: Long, providedTotal: Long): Long {
        val totalSize = Math.max(TOTAL_SIZE_UPDATER.get(this), 1) //Ensure its always positive
        val activeDownloads = ACTIVE_DOWNLOADS_UPDATER.get(this)

        //RESOLUTION * (alreadyDownloaded / expectedDownloaded) +
        // RESOLUTION * (downloadedSize / totalSize) * (activeDownloads / expectedDownloads)
        //
        // a * (b / c) + a * (d / e) * (f / c)
        // a * (b * e + d * f) / (c * e)

        return (resolution * (alreadyDownloaded * totalSize + providedTotal * activeDownloads) / (totalSize * totalExpectedDownloads))
    }<commit_msg>Use weak keys in the object set to avoid weird memory leaks<commit_after>package net.serverpeon.twitcharchiver.fx.network

import com.google.common.collect.MapMaker
import java.util.*
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater
import java.util.concurrent.atomic.AtomicLongFieldUpdater

class ConcurrentProgressTracker {
    @Volatile private var totalSize: Long = 0
    @Volatile private var downloadedSize: Long = 0
    @Volatile private var activeDownloads: Int = 0
    private val objectKeys: MutableSet<Any> = Collections.newSetFromMap(
            MapMaker().weakKeys().makeMap()
    )
    private var alreadyDownloaded = 0
    private var totalExpectedDownloads = -1

    fun addAndGetProgress(delta: Long, resolution: Long): Long = calculateProgress(resolution, updateProgress(delta))

    fun progress(resolution: Long): Long = calculateProgress(resolution, DOWNLOADED_SIZE_UPDATER.get(this))

    fun modifyExpectedTotal(delta: Long, key: Any) {
        TOTAL_SIZE_UPDATER.addAndGet(this, delta)
        if (objectKeys.add(key)) {
            ACTIVE_DOWNLOADS_UPDATER.incrementAndGet(this)
        }
    }

    private fun calculateProgress(resolution: Long, providedTotal: Long): Long {
        val totalSize = Math.max(TOTAL_SIZE_UPDATER.get(this), 1) //Ensure its always positive
        val activeDownloads = ACTIVE_DOWNLOADS_UPDATER.get(this)

        //RESOLUTION * (alreadyDownloaded / expectedDownloaded) +
        // RESOLUTION * (downloadedSize / totalSize) * (activeDownloads / expectedDownloads)
        //
        // a * (b / c) + a * (d / e) * (f / c)
        // a * (b * e + d * f) / (c * e)

        return (resolution * (alreadyDownloaded * totalSize + providedTotal * activeDownloads) / (totalSize * totalExpectedDownloads))
    }
{'fraction_non_alphanumeric': 0.0980448533640023, 'fraction_numerical': 0.0034502587694077054, 'mean_word_length': 3.57161629434954, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1468639', 'n_tokens_mistral': 1056, 'n_tokens_neox': 1013, 'n_words': 244}