<commit_before>package org.vitrivr.cineast.standalone.importer.lsc2020;

import com.opencsv.exceptions.CsvException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.importer.Importer;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.*;

/**
 * Importer for the LSC 2020 metadata, delivered with the dataset.
 */
public class MetaImporter implements Importer<Map<String, PrimitiveTypeProvider>> {

    private static final Logger LOGGER = LogManager.getLogger(MetaImporter.class);
    private final Map<String, String[]> metadataMap;

    private Map<String, String> filenameToMinuteId;

    private List<String> metaNoPath = new ArrayList<>();
    private List<String> written = new ArrayList<>();

    private Path root;

    private Iterator<Map<String, PrimitiveTypeProvider>> currentDataIterator = null;
    private Iterator<Map.Entry<String, String>> iterator;

    private Optional<List<Map<String, PrimitiveTypeProvider>>> parseLine(String filename, String[] items) {
        if (items.length != LSCUtilities.META_NAMES.length) {
            LOGGER.error("THe line's number of entries is illegal. Expected={}, Found={}. Line={}", LSCUtilities.META_NAMES.length, items.length, items);
            throw new RuntimeException("Too few metadata entries.");
        }
        String minuteId = LSCUtilities.filenameToMinuteId(filename).get();
        List<Map<String, PrimitiveTypeProvider>> list = new ArrayList<>();
        for (int i = 0; i < LSCUtilities.META_NAMES.length; i++) {
            if(items[i].equalsIgnoreCase("null")){
                continue;
            }
            list.add(parseMeta(filename, items, i));
        }
        // Processed temporal metadata, based on filename
        final LocalDateTime dt = LSCUtilities.fromMinuteId(minuteId);
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DATETIME, dt.toString()));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DAY_OF_WEEK, String.valueOf(dt.getDayOfWeek())));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DAY_OF_WEEK, LSCUtilities.extractMonth(dt)));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DAY_OF_WEEK, LSCUtilities.extractDay(dt)));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DAY_OF_WEEK, LSCUtilities.extractYear(dt)));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DAY_OF_WEEK, LSCUtilities.extractHour(dt)));
        if(StringUtils.isNotBlank(items[LSCUtilities.META_UTC_COL]) && !items[LSCUtilities.META_UTC_COL].equalsIgnoreCase("null")){
            list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_UTC, LSCUtilities.convertUtc(items[LSCUtilities.META_UTC_COL]).toString()));
        }
        // Processed temporal metadata, based on metadata file
        if((StringUtils.isNotBlank(items[LSCUtilities.META_LOCAL_COL]) && !items[LSCUtilities.META_LOCAL_COL].equalsIgnoreCase("null"))
        && (StringUtils.isNotBlank(items[LSCUtilities.META_TIMEZONE_COL]) && !items[LSCUtilities.META_TIMEZONE_COL].equalsIgnoreCase("null"))){
            final ZonedDateTime zdt = LSCUtilities.convertLocal(items[LSCUtilities.META_LOCAL_COL], items[LSCUtilities.META_TIMEZONE_COL]);
            list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_LOCAL, zdt.toString()));
            list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_PHASE_OF_DAY, LSCUtilities.extractPhaseOfDay(zdt)));
        }<commit_msg>Fixed copy-paste error<commit_after>package org.vitrivr.cineast.standalone.importer.lsc2020;

import com.opencsv.exceptions.CsvException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.importer.Importer;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.*;

/**
 * Importer for the LSC 2020 metadata, delivered with the dataset.
 */
public class MetaImporter implements Importer<Map<String, PrimitiveTypeProvider>> {

    private static final Logger LOGGER = LogManager.getLogger(MetaImporter.class);
    private final Map<String, String[]> metadataMap;

    private Map<String, String> filenameToMinuteId;

    private List<String> metaNoPath = new ArrayList<>();
    private List<String> written = new ArrayList<>();

    private Path root;

    private Iterator<Map<String, PrimitiveTypeProvider>> currentDataIterator = null;
    private Iterator<Map.Entry<String, String>> iterator;

    private Optional<List<Map<String, PrimitiveTypeProvider>>> parseLine(String filename, String[] items) {
        if (items.length != LSCUtilities.META_NAMES.length) {
            LOGGER.error("THe line's number of entries is illegal. Expected={}, Found={}. Line={}", LSCUtilities.META_NAMES.length, items.length, items);
            throw new RuntimeException("Too few metadata entries.");
        }
        String minuteId = LSCUtilities.filenameToMinuteId(filename).get();
        List<Map<String, PrimitiveTypeProvider>> list = new ArrayList<>();
        for (int i = 0; i < LSCUtilities.META_NAMES.length; i++) {
            if(items[i].equalsIgnoreCase("null")){
                continue;
            }
            list.add(parseMeta(filename, items, i));
        }
        // Processed temporal metadata, based on filename
        final LocalDateTime dt = LSCUtilities.fromMinuteId(minuteId);
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DATETIME, dt.toString()));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DAY_OF_WEEK, String.valueOf(dt.getDayOfWeek())));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_MONTH, LSCUtilities.extractMonth(dt)));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_DAY, LSCUtilities.extractDay(dt)));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_YEAR, LSCUtilities.extractYear(dt)));
        list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_HOUR_OF_DAY, LSCUtilities.extractHour(dt)));
        if(StringUtils.isNotBlank(items[LSCUtilities.META_UTC_COL]) && !items[LSCUtilities.META_UTC_COL].equalsIgnoreCase("null")){
            list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_UTC, LSCUtilities.convertUtc(items[LSCUtilities.META_UTC_COL]).toString()));
        }
        // Processed temporal metadata, based on metadata file
        if((StringUtils.isNotBlank(items[LSCUtilities.META_LOCAL_COL]) && !items[LSCUtilities.META_LOCAL_COL].equalsIgnoreCase("null"))
        && (StringUtils.isNotBlank(items[LSCUtilities.META_TIMEZONE_COL]) && !items[LSCUtilities.META_TIMEZONE_COL].equalsIgnoreCase("null"))){
            final ZonedDateTime zdt = LSCUtilities.convertLocal(items[LSCUtilities.META_LOCAL_COL], items[LSCUtilities.META_TIMEZONE_COL]);
            list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_LOCAL, zdt.toString()));
            list.add(meta(filename, LSCUtilities.DEFAULT_DOMAIN, LSCUtilities.PROCESSED_META_PHASE_OF_DAY, LSCUtilities.extractPhaseOfDay(zdt)));
        }
{'fraction_non_alphanumeric': 0.1138601674052191, 'fraction_numerical': 0.0029542097488921715, 'mean_word_length': 4.862193362193362, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3441767', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2556, 'n_words': 394}