<commit_before>package org.vitrivr.cineast.core.data.raw;

import static java.lang.Thread.MIN_PRIORITY;

import java.lang.ref.PhantomReference;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import org.vitrivr.cineast.core.config.CacheConfig;
import org.vitrivr.cineast.core.data.raw.bytes.ByteData;
import org.vitrivr.cineast.core.data.raw.bytes.CachedByteData;
import org.vitrivr.cineast.core.data.raw.bytes.InMemoryByteData;
import org.vitrivr.cineast.core.data.raw.images.CachedMultiImage;
import org.vitrivr.cineast.core.data.raw.images.InMemoryMultiImage;
import org.vitrivr.cineast.core.data.raw.images.MultiImage;

import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;


/**
 * This factory class generates {@link ByteData} objects either in memory or backed by a file cache based on a heuristic involving
 * the size of the allocated data chunks.
 *
 * @author <NAME>
 * @version 1.1
 *
 * @see ByteData
 * @see InMemoryByteData
 * @see CachedByteData
 * @see InMemoryMultiImage
 * @see CachedMultiImage
 */
public class CachedDataFactory {

    /** Default instance of {@link CachedDataFactory}. */
    public static final CachedDataFactory DEFAULT_INSTANCE = new CachedDataFactory(new CacheConfig());

    /** {@link ReferenceQueue} that keeps track of {@link CachedByteData} objects. */
    private static final ReferenceQueue<CachedByteData> CACHED_REF_QUEUE = new ReferenceQueue<>();

    /** Logger instance used to log errors. */
    private static final Logger LOGGER = LogManager.getLogger();

    /** Thread that cleans the cached byte data files. */
    private static final Thread CLEANER_THREAD = new Thread(() -> {
        while (true) {
            final Reference<?> ref = CACHED_REF_QUEUE.poll();
            if (ref instanceof CachedByteDataReference) {
                final Path path = ((CachedByteDataReference) ref).path;
                try {
                    Files.deleteIfExists(path);
                } catch (IOException e) {
                    LOGGER.fatal("Failed to delete temporary cache file: {}", path.toString());
                }
            } else {
                Thread.yield();
            }
        }
    });

    static {
        CLEANER_THREAD.setName("Cache-File-Cleaner");
        CLEANER_THREAD.setPriority(MIN_PRIORITY);
        CLEANER_THREAD.start();
    }

    /** Reference to {@link CacheConfig} used to setup this {@link CachedDataFactory}. */
    private final CacheConfig config;

    /** Location where this instance of {@link CachedDataFactory} will store its cached images. */
    private final Path cacheLocation;

    /**
     * Inner {@link PhantomReference} implementations that keeps track of the cache path for every {@link CachedByteData}.
     */
    private static class CachedByteDataReference extends PhantomReference<CachedByteData> {
        private final Path path;
        private CachedByteDataReference(CachedByteData data) {
            super(data, CachedDataFactory.CACHED_REF_QUEUE);
            this.path = data.getPath();
        }<commit_msg>Cleaner Thread is now a daemon thread.<commit_after>package org.vitrivr.cineast.core.data.raw;

import static java.lang.Thread.MIN_PRIORITY;

import java.lang.ref.PhantomReference;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import org.vitrivr.cineast.core.config.CacheConfig;
import org.vitrivr.cineast.core.data.raw.bytes.ByteData;
import org.vitrivr.cineast.core.data.raw.bytes.CachedByteData;
import org.vitrivr.cineast.core.data.raw.bytes.InMemoryByteData;
import org.vitrivr.cineast.core.data.raw.images.CachedMultiImage;
import org.vitrivr.cineast.core.data.raw.images.InMemoryMultiImage;
import org.vitrivr.cineast.core.data.raw.images.MultiImage;

import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;


/**
 * This factory class generates {@link ByteData} objects either in memory or backed by a file cache based on a heuristic involving
 * the size of the allocated data chunks.
 *
 * @author <NAME>
 * @version 1.1
 *
 * @see ByteData
 * @see InMemoryByteData
 * @see CachedByteData
 * @see InMemoryMultiImage
 * @see CachedMultiImage
 */
public class CachedDataFactory {

    /** Default instance of {@link CachedDataFactory}. */
    public static final CachedDataFactory DEFAULT_INSTANCE = new CachedDataFactory(new CacheConfig());

    /** {@link ReferenceQueue} that keeps track of {@link CachedByteData} objects. */
    private static final ReferenceQueue<CachedByteData> CACHED_REF_QUEUE = new ReferenceQueue<>();

    /** Logger instance used to log errors. */
    private static final Logger LOGGER = LogManager.getLogger();

    /** Thread that cleans the cached byte data files. */
    private static final Thread CLEANER_THREAD = new Thread(() -> {
        while (true) {
            final Reference<?> ref = CACHED_REF_QUEUE.poll();
            if (ref instanceof CachedByteDataReference) {
                final Path path = ((CachedByteDataReference) ref).path;
                try {
                    Files.deleteIfExists(path);
                } catch (IOException e) {
                    LOGGER.fatal("Failed to delete temporary cache file: {}", path.toString());
                }
            } else {
                Thread.yield();
            }
        }
    });

    static {
        CLEANER_THREAD.setName("Cache-File-Cleaner");
        CLEANER_THREAD.setPriority(MIN_PRIORITY);
        CLEANER_THREAD.setDaemon(true);
        CLEANER_THREAD.start();
    }

    /** Reference to {@link CacheConfig} used to setup this {@link CachedDataFactory}. */
    private final CacheConfig config;

    /** Location where this instance of {@link CachedDataFactory} will store its cached images. */
    private final Path cacheLocation;

    /**
     * Inner {@link PhantomReference} implementations that keeps track of the cache path for every {@link CachedByteData}.
     */
    private static class CachedByteDataReference extends PhantomReference<CachedByteData> {
        private final Path path;
        private CachedByteDataReference(CachedByteData data) {
            super(data, CachedDataFactory.CACHED_REF_QUEUE);
            this.path = data.getPath();
        }
{'fraction_non_alphanumeric': 0.10016518996846373, 'fraction_numerical': 0.0012013815888271512, 'mean_word_length': 3.2045454545454546, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6349745', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1869, 'n_words': 485}