<commit_before>package org.vitrivr.cineast.core.importer.handlers;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.importer.JsonObjectImporter;
import org.vitrivr.cineast.core.util.LogHelper;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

public class JsonDataImportHandler extends DataImportHandler {
    /** */
    private static final Logger LOGGER = LogManager.getLogger();

    /**
     *
     * @param threads
     * @param batchsize
     */
    public JsonDataImportHandler(int threads, int batchsize) {
        super(threads, batchsize);
    }

    /**
     * Starts data import process using JSON files. The method can either be used with a single file or a folder containing JSON files.
     *
     * @param path Path to the JSON file or a folder containing JSON files.
     */
    @Override
    public void doImport(Path path) {
        try {
            LOGGER.info("Starting data import with JSON files in: {}", path.toString());
            Files.walk(path, 2).filter(p -> p.toString().toLowerCase().endsWith(".json")).forEach(p -> {
                try {
                    final String suffix = path.getFileName().toString().substring(path.getFileName().toString().lastIndexOf("."));
                    this.futures.add(this.service.submit(new DataImportRunner(new JsonObjectImporter(p.toFile()), path.getFileName().toString().replace(suffix, ""))));
                } catch (IOException e) {
                    LOGGER.error("Could not start data import for file '{}'. Skipping...?", p.toString());
                }
            });
            this.waitForCompletion();
            LOGGER.info("Completed data import with JSON files in: {}", path.toString());
        } catch (IOException e) {
            LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e));
        }
    }
}<commit_msg>Fixed an issue with using the wrong variable.<commit_after>package org.vitrivr.cineast.core.importer.handlers;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.importer.JsonObjectImporter;
import org.vitrivr.cineast.core.util.LogHelper;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

public class JsonDataImportHandler extends DataImportHandler {
    /** */
    private static final Logger LOGGER = LogManager.getLogger();

    /**
     *
     * @param threads
     * @param batchsize
     */
    public JsonDataImportHandler(int threads, int batchsize) {
        super(threads, batchsize);
    }

    /**
     * Starts data import process using JSON files. The method can either be used with a single file or a folder containing JSON files.
     *
     * @param path Path to the JSON file or a folder containing JSON files.
     */
    @Override
    public void doImport(Path path) {
        try {
            LOGGER.info("Starting data import with JSON files in: {}", path.toString());
            Files.walk(path, 2).filter(p -> p.toString().toLowerCase().endsWith(".json")).forEach(p -> {
                final String filename = p.getFileName().toString();
                final String suffix = filename.substring(filename.lastIndexOf("."));
                try {
                    this.futures.add(this.service.submit(new DataImportRunner(new JsonObjectImporter(p.toFile()), filename.replace(suffix, ""))));
                } catch (IOException e) {
                    LOGGER.error("Could not start data import for file '{}'. Skipping...?", p.toString());
                }
            });
            this.waitForCompletion();
            LOGGER.info("Completed data import with JSON files in: {}", path.toString());
        } catch (IOException e) {
            LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e));
        }
    }
}
{'fraction_non_alphanumeric': 0.12435360748584093, 'fraction_numerical': 0.0014774686037921695, 'mean_word_length': 2.6074600355239785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1375268', 'n_tokens_mistral': 1117, 'n_tokens_neox': 1094, 'n_words': 289}