<commit_before>package org.vitrivr.cineast.core.run;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.config.Config;
import org.vitrivr.cineast.core.config.IngestConfig;
import org.vitrivr.cineast.core.data.MediaType;
import org.vitrivr.cineast.core.run.filehandler.AudioExtractionFileHandler;
import org.vitrivr.cineast.core.run.filehandler.ImageExtractionFileHandler;
import org.vitrivr.cineast.core.run.filehandler.VideoExtractionFileHandler;
import org.vitrivr.cineast.core.util.LogHelper;
import org.vitrivr.cineast.core.util.json.JacksonJsonProvider;

/**
 * @author rgasser
 * @version 1.0
 * @created 13.01.17
 */
public class ExtractionDispatcher {

    private static final Logger LOGGER = LogManager.getLogger();

    /** ExtractionContextProvider used to setup the extraction. */
    private ExtractionContextProvider context;

    /** List of files due for extraction. */
    private Iterator<Path> paths;

    /** Reference to the thread that is being used to run the ExtractionFileHandler. */
    private Thread fileHandlerThread;

    /**
     *
     * @param jobFile
     */
    public boolean initialize(File jobFile) throws IOException {
        File outputLocation = Config.sharedConfig().getExtractor().getOutputLocation();
        if(outputLocation == null){
          LOGGER.error("invalid output location specified in config");
          return false;
        }
        outputLocation.mkdirs();
        if(!outputLocation.canWrite()){
          LOGGER.error("cannot write to specified output location: '{}'", outputLocation.getAbsolutePath());
          return false;
        }
      
        JacksonJsonProvider reader = new JacksonJsonProvider();
        this.context = reader.toObject(jobFile, IngestConfig.class);
        
        /* Check if context could be read and an input path was specified. */
        if (context == null || this.context.inputPath() == null) return false;
        Path path = this.context.inputPath();
        
        /*
         * Recursively add all files under that path to the List of files that should be processed. Uses the context-provider
         * to determine the depth of recursion, skip files and limit the number of files.
         */
        if (!Files.exists(path)) {
            LOGGER.warn("The path '{}' specified in the extraction configuration does not exist!", path.toString());
            return false;
        }
        this.paths = Files.walk(path, this.context.depth())
                     .filter(p -> {
                         try {
                             return Files.exists(p) && Files.isRegularFile(p) && !Files.isHidden(p) && Files.isReadable(p);
                         } catch (IOException e) {
                             LOGGER.error("An IO exception occurred while testing the media file at '{}'.", p.toString(), LogHelper.getStackTrace(e));
                             return false;
                         }
                     }).iterator();
        return true;
    }

    /**
     * Starts extraction by dispatching a new ExtractionFileHandler thread.
     *
     * @throws IOException If an error occurs during pre-processing of the files.
     */
    public void start() throws IOException {
        if (this.fileHandlerThread == null) {
            MediaType sourceType = this.context.sourceType();
            switch (sourceType) {
                case IMAGE:
                    this.fileHandlerThread = new Thread(new ImageExtractionFileHandler(this.paths, this.context));
                    break;
                case VIDEO:
                    this.fileHandlerThread = new Thread(new VideoExtractionFileHandler(this.paths, this.context));
                    break;
                case AUDIO:<commit_msg>extraction now follows links during file system walk<commit_after>package org.vitrivr.cineast.core.run;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.config.Config;
import org.vitrivr.cineast.core.config.IngestConfig;
import org.vitrivr.cineast.core.data.MediaType;
import org.vitrivr.cineast.core.run.filehandler.AudioExtractionFileHandler;
import org.vitrivr.cineast.core.run.filehandler.ImageExtractionFileHandler;
import org.vitrivr.cineast.core.run.filehandler.VideoExtractionFileHandler;
import org.vitrivr.cineast.core.util.LogHelper;
import org.vitrivr.cineast.core.util.json.JacksonJsonProvider;

/**
 * @author rgasser
 * @version 1.0
 * @created 13.01.17
 */
public class ExtractionDispatcher {

    private static final Logger LOGGER = LogManager.getLogger();

    /** ExtractionContextProvider used to setup the extraction. */
    private ExtractionContextProvider context;

    /** List of files due for extraction. */
    private Iterator<Path> paths;

    /** Reference to the thread that is being used to run the ExtractionFileHandler. */
    private Thread fileHandlerThread;

    /**
     *
     * @param jobFile
     */
    public boolean initialize(File jobFile) throws IOException {
        File outputLocation = Config.sharedConfig().getExtractor().getOutputLocation();
        if(outputLocation == null){
          LOGGER.error("invalid output location specified in config");
          return false;
        }
        outputLocation.mkdirs();
        if(!outputLocation.canWrite()){
          LOGGER.error("cannot write to specified output location: '{}'", outputLocation.getAbsolutePath());
          return false;
        }
      
        JacksonJsonProvider reader = new JacksonJsonProvider();
        this.context = reader.toObject(jobFile, IngestConfig.class);
        
        /* Check if context could be read and an input path was specified. */
        if (context == null || this.context.inputPath() == null) return false;
        Path path = this.context.inputPath();
        
        /*
         * Recursively add all files under that path to the List of files that should be processed. Uses the context-provider
         * to determine the depth of recursion, skip files and limit the number of files.
         */
        if (!Files.exists(path)) {
            LOGGER.warn("The path '{}' specified in the extraction configuration does not exist!", path.toString());
            return false;
        }
        this.paths = Files.walk(path, this.context.depth(), FileVisitOption.FOLLOW_LINKS)
                     .filter(p -> {
                         try {
                             return Files.exists(p) && Files.isRegularFile(p) && !Files.isHidden(p) && Files.isReadable(p);
                         } catch (IOException e) {
                             LOGGER.error("An IO exception occurred while testing the media file at '{}'.", p.toString(), LogHelper.getStackTrace(e));
                             return false;
                         }
                     }).iterator();
        return true;
    }

    /**
     * Starts extraction by dispatching a new ExtractionFileHandler thread.
     *
     * @throws IOException If an error occurs during pre-processing of the files.
     */
    public void start() throws IOException {
        if (this.fileHandlerThread == null) {
            MediaType sourceType = this.context.sourceType();
            switch (sourceType) {
                case IMAGE:
                    this.fileHandlerThread = new Thread(new ImageExtractionFileHandler(this.paths, this.context));
                    break;
                case VIDEO:
                    this.fileHandlerThread = new Thread(new VideoExtractionFileHandler(this.paths, this.context));
                    break;
                case AUDIO:
{'fraction_non_alphanumeric': 0.09805153991200503, 'fraction_numerical': 0.00251414204902577, 'mean_word_length': 2.366906474820144, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7587438', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2038, 'n_words': 547}