<commit_before>/*
 * Autopsy Forensic Browser
 * 
 * Copyright 2014 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import javax.swing.JOptionPane;
import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;

/**
 * Encapsulates a data source and the ingest module pipelines used to process
 * it.
 */
final class DataSourceIngestJob {

    private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());

    /**
     * These fields define a data source ingest job: the parent ingest job, an
     * ID, the user's ingest job settings, and the data source to be processed.
     */
    private final IngestJob parentJob;
    private static final AtomicLong nextJobId = new AtomicLong(0L);
    private final long id;
    private final IngestJobSettings settings;
    private final Content dataSource;

    /**
     * A data source ingest job runs in stages.
     */
    private static enum Stages {

        /**
         * Setting up for processing.
         */
        INITIALIZATION,
        /**
         * Running high priority data source level ingest modules and file level
         * ingest modules.
         */
        FIRST,
        /**
         * Running lower priority, usually long-running, data source level
         * ingest modules.
         */
        SECOND,
        /**
         * Cleaning up.
         */
        FINALIZATION
    };
    private Stages stage = DataSourceIngestJob.Stages.INITIALIZATION;
    private final Object stageCompletionCheckLock = new Object();

    /**
     * A data source ingest job has separate data source level ingest module
     * pipelines for the first and second processing stages. Longer running,
     * lower priority modules belong in the second stage pipeline, although this
     * cannot be enforced. Note that the pipelines for both stages are created
     * at job start up to allow for verification that they both can be started
     * up without errors.
     */
    private final Object dataSourceIngestPipelineLock = new Object();
    private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
    private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
    private DataSourceIngestPipeline currentDataSourceIngestPipeline;

    /**
     * A data source ingest job has a collection of identical file level ingest
     * module pipelines, one for each file level ingest thread in the ingest
     * manager. A blocking queue is used to dole out the pipelines to the
     * threads and an ordinary list is used when the ingest job needs to access
     * the pipelines to query their status.
     */
    private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue = new LinkedBlockingQueue<>();
    private final List<FileIngestPipeline> fileIngestPipelines = new ArrayList<>();

    /**
     * A data source ingest job supports cancellation of either the currently
     * running data source level ingest module or the entire ingest job.
     *
     * TODO: The currentDataSourceIngestModuleCancelled field and all of the
     * code concerned with it is a hack to avoid an API change. The next time an
     * API change is legal, a cancel() method needs to be added to the
     * IngestModule interface and this field should be removed. The "ingest job
     * is canceled" queries should also be removed from the IngestJobContext
     * class.
     */
    private volatile boolean currentDataSourceIngestModuleCancelled;
    private volatile boolean cancelled;
    private final List<String> cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>();

    /**
     * A data source ingest job uses the task scheduler singleton to create and
     * queue the ingest tasks that make up the job.
     */
    private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();

    /**
     * A data source ingest job can run interactively using NetBeans progress
     * handles.
     */
    private final boolean doUI;

    /**
     * A data source ingest job uses these fields to report data source level
     * ingest progress.
     */
    private final Object dataSourceIngestProgressLock = new Object();
    private ProgressHandle dataSourceIngestProgress;

    /**
     * A data source ingest job uses these fields to report file level ingest
     * progress.
     */
    private final Object fileIngestProgressLock = new Object();
    private final List<String> filesInProgress = new ArrayList<>();
    private long estimatedFilesToProcess;
    private long processedFiles;
    private ProgressHandle fileIngestProgress;
    private String currentFileIngestModule;
    private String currentFileIngestTask;

    /**
     * A data source ingest job uses this field to report its creation time.
     */
    private final long createTime;

    /**
     * Constructs an object that encapsulates a data source and the ingest
     * module pipelines used to process it.
     *
     * @param parentJob The ingest job of which this data source ingest job is a
     * part.
     * @param dataSource The data source to be ingested.
     * @param settings The settings for the ingest job.
     * @param runInteractively Whether or not this job should use NetBeans
     * progress handles.
     */
    DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) {
        this.parentJob = parentJob;
        this.id = DataSourceIngestJob.nextJobId.getAndIncrement();
        this.dataSource = dataSource;
        this.settings = settings;
        this.doUI = runInteractively;
        this.createTime = new Date().getTime();
        this.currentFileIngestModule = "";
        this.currentFileIngestTask = "";
        this.createIngestPipelines();
    }

    /**
     * Creates the file and data source ingest pipelines.
     */
    private void createIngestPipelines() {
        List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();

        /**
         * Make mappings of ingest module factory class names to templates.
         */
        Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new HashMap<>();
        Map<String, IngestModuleTemplate> fileModuleTemplates = new HashMap<>();
        for (IngestModuleTemplate template : ingestModuleTemplates) {
            if (template.isDataSourceIngestModuleTemplate()) {
                dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
            }
            if (template.isFileIngestModuleTemplate()) {
                fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
            }
        }<commit_msg>remove initialization in constructor<commit_after>/*
 * Autopsy Forensic Browser
 * 
 * Copyright 2014 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import javax.swing.JOptionPane;
import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;

/**
 * Encapsulates a data source and the ingest module pipelines used to process
 * it.
 */
final class DataSourceIngestJob {

    private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());

    /**
     * These fields define a data source ingest job: the parent ingest job, an
     * ID, the user's ingest job settings, and the data source to be processed.
     */
    private final IngestJob parentJob;
    private static final AtomicLong nextJobId = new AtomicLong(0L);
    private final long id;
    private final IngestJobSettings settings;
    private final Content dataSource;

    /**
     * A data source ingest job runs in stages.
     */
    private static enum Stages {

        /**
         * Setting up for processing.
         */
        INITIALIZATION,
        /**
         * Running high priority data source level ingest modules and file level
         * ingest modules.
         */
        FIRST,
        /**
         * Running lower priority, usually long-running, data source level
         * ingest modules.
         */
        SECOND,
        /**
         * Cleaning up.
         */
        FINALIZATION
    };
    private Stages stage = DataSourceIngestJob.Stages.INITIALIZATION;
    private final Object stageCompletionCheckLock = new Object();

    /**
     * A data source ingest job has separate data source level ingest module
     * pipelines for the first and second processing stages. Longer running,
     * lower priority modules belong in the second stage pipeline, although this
     * cannot be enforced. Note that the pipelines for both stages are created
     * at job start up to allow for verification that they both can be started
     * up without errors.
     */
    private final Object dataSourceIngestPipelineLock = new Object();
    private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
    private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
    private DataSourceIngestPipeline currentDataSourceIngestPipeline;

    /**
     * A data source ingest job has a collection of identical file level ingest
     * module pipelines, one for each file level ingest thread in the ingest
     * manager. A blocking queue is used to dole out the pipelines to the
     * threads and an ordinary list is used when the ingest job needs to access
     * the pipelines to query their status.
     */
    private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue = new LinkedBlockingQueue<>();
    private final List<FileIngestPipeline> fileIngestPipelines = new ArrayList<>();

    /**
     * A data source ingest job supports cancellation of either the currently
     * running data source level ingest module or the entire ingest job.
     *
     * TODO: The currentDataSourceIngestModuleCancelled field and all of the
     * code concerned with it is a hack to avoid an API change. The next time an
     * API change is legal, a cancel() method needs to be added to the
     * IngestModule interface and this field should be removed. The "ingest job
     * is canceled" queries should also be removed from the IngestJobContext
     * class.
     */
    private volatile boolean currentDataSourceIngestModuleCancelled;
    private volatile boolean cancelled;
    private final List<String> cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>();

    /**
     * A data source ingest job uses the task scheduler singleton to create and
     * queue the ingest tasks that make up the job.
     */
    private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();

    /**
     * A data source ingest job can run interactively using NetBeans progress
     * handles.
     */
    private final boolean doUI;

    /**
     * A data source ingest job uses these fields to report data source level
     * ingest progress.
     */
    private final Object dataSourceIngestProgressLock = new Object();
    private ProgressHandle dataSourceIngestProgress;

    /**
     * A data source ingest job uses these fields to report file level ingest
     * progress.
     */
    private final Object fileIngestProgressLock = new Object();
    private final List<String> filesInProgress = new ArrayList<>();
    private long estimatedFilesToProcess;
    private long processedFiles;
    private ProgressHandle fileIngestProgress;
    private String currentFileIngestModule = "";
    private String currentFileIngestTask = "";

    /**
     * A data source ingest job uses this field to report its creation time.
     */
    private final long createTime;

    /**
     * Constructs an object that encapsulates a data source and the ingest
     * module pipelines used to process it.
     *
     * @param parentJob The ingest job of which this data source ingest job is a
     * part.
     * @param dataSource The data source to be ingested.
     * @param settings The settings for the ingest job.
     * @param runInteractively Whether or not this job should use NetBeans
     * progress handles.
     */
    DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) {
        this.parentJob = parentJob;
        this.id = DataSourceIngestJob.nextJobId.getAndIncrement();
        this.dataSource = dataSource;
        this.settings = settings;
        this.doUI = runInteractively;
        this.createTime = new Date().getTime();
        this.createIngestPipelines();
    }

    /**
     * Creates the file and data source ingest pipelines.
     */
    private void createIngestPipelines() {
        List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();

        /**
         * Make mappings of ingest module factory class names to templates.
         */
        Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new HashMap<>();
        Map<String, IngestModuleTemplate> fileModuleTemplates = new HashMap<>();
        for (IngestModuleTemplate template : ingestModuleTemplates) {
            if (template.isDataSourceIngestModuleTemplate()) {
                dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
            }
            if (template.isFileIngestModuleTemplate()) {
                fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
            }
        }
{'fraction_non_alphanumeric': 0.06738015325185231, 'fraction_numerical': 0.001139889810651637, 'mean_word_length': 3.1060842433697347, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6887916', 'n_tokens_mistral': 4022, 'n_tokens_neox': 3914, 'n_words': 1513}