<commit_before>package org.vitrivr.cineast.standalone.importer.lsc2020;

import com.opencsv.exceptions.CsvException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.features.SpatialDistance;
import org.vitrivr.cineast.core.importer.Importer;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.*;

/**
 * Importer for the LSC 2020 metadata, delivered with the dataset.
 */
public class MetaImporter implements Importer<Map<String, PrimitiveTypeProvider>> {

    private static final Logger LOGGER = LogManager.getLogger(MetaImporter.class);
    private final Map<String, String[]> metadataMap;

    private Map<String, String> filenameToMinuteId;

    private List<String> metaNoPath = new ArrayList<>();
    private List<String> written = new ArrayList<>();

    private Path root;

    private Iterator<Map<String, PrimitiveTypeProvider>> currentDataIterator = null;
    private Iterator<Map.Entry<String, String>> iterator;

    public MetaImporter(Path path) {
        this.root = path;
        try {
            LSCUtilities lsc = LSCUtilities.create(path);
            lsc.initMetadata();
            filenameToMinuteId = lsc.getFilenameToMinuteIdLookUp();
            metadataMap = lsc.getMetaPerMinuteId();
            iterator = filenameToMinuteId.entrySet().iterator();
        } catch (IOException | CsvException e) {
            LOGGER.error("Failed to prepare metadata readout due to {}", e, e);
            throw new RuntimeException("Failed to prepare metadata readout", e);
        }
        try {
            Files.createFile(root.resolve(LSCUtilities.META_NO_PATH_FILE));
            Files.createFile(root.resolve(LSCUtilities.WRITTEN_FILE));
        } catch (IOException e) {
            LOGGER.error("Could not open important housekeeping files", e);
        }
        LOGGER.info("Finished setup of Importer. Importing now...");
    }

    static HashMap<String, PrimitiveTypeProvider> meta(String filename, String domain, String name, String value) {
        final HashMap<String, PrimitiveTypeProvider> map = new HashMap<>();
        // "id"
        map.put(MediaSegmentMetadataDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(LSCUtilities.pathToSegmentId(filename)));
        // domain
        map.put(MediaSegmentMetadataDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(domain));
        // key
        map.put(MediaSegmentMetadataDescriptor.FIELDNAMES[2], PrimitiveTypeProvider.fromObject(name));
        // value<commit_msg>sanity-check for file creation<commit_after>package org.vitrivr.cineast.standalone.importer.lsc2020;

import com.opencsv.exceptions.CsvException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.features.SpatialDistance;
import org.vitrivr.cineast.core.importer.Importer;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.*;

/**
 * Importer for the LSC 2020 metadata, delivered with the dataset.
 */
public class MetaImporter implements Importer<Map<String, PrimitiveTypeProvider>> {

    private static final Logger LOGGER = LogManager.getLogger(MetaImporter.class);
    private final Map<String, String[]> metadataMap;

    private Map<String, String> filenameToMinuteId;

    private List<String> metaNoPath = new ArrayList<>();
    private List<String> written = new ArrayList<>();

    private Path root;

    private Iterator<Map<String, PrimitiveTypeProvider>> currentDataIterator = null;
    private Iterator<Map.Entry<String, String>> iterator;

    public MetaImporter(Path path) {
        this.root = path;
        try {
            LSCUtilities lsc = LSCUtilities.create(path);
            lsc.initMetadata();
            filenameToMinuteId = lsc.getFilenameToMinuteIdLookUp();
            metadataMap = lsc.getMetaPerMinuteId();
            iterator = filenameToMinuteId.entrySet().iterator();
        } catch (IOException | CsvException e) {
            LOGGER.error("Failed to prepare metadata readout due to {}", e, e);
            throw new RuntimeException("Failed to prepare metadata readout", e);
        }
        try {
            if(!Files.exists(root.resolve(LSCUtilities.META_NO_PATH_FILE))){
                Files.createFile(root.resolve(LSCUtilities.META_NO_PATH_FILE));
            }
            if(!Files.exists(root.resolve(LSCUtilities.WRITTEN_FILE))){
                Files.createFile(root.resolve(LSCUtilities.WRITTEN_FILE));
            }
        } catch (IOException e) {
            LOGGER.error("Could not open important housekeeping files", e);
        }
        LOGGER.info("Finished setup of Importer. Importing now...");
    }

    static HashMap<String, PrimitiveTypeProvider> meta(String filename, String domain, String name, String value) {
        final HashMap<String, PrimitiveTypeProvider> map = new HashMap<>();
        // "id"
        map.put(MediaSegmentMetadataDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(LSCUtilities.pathToSegmentId(filename)));
        // domain
        map.put(MediaSegmentMetadataDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(domain));
        // key
        map.put(MediaSegmentMetadataDescriptor.FIELDNAMES[2], PrimitiveTypeProvider.fromObject(name));
        // value
{'fraction_non_alphanumeric': 0.1046046046046046, 'fraction_numerical': 0.004671338004671338, 'mean_word_length': 3.6726422447388933, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '381654', 'n_tokens_mistral': 1715, 'n_tokens_neox': 1695, 'n_words': 340}