<commit_before>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.keywordsearch;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.TermsResponse.Term;
import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.IngestManagerProxy;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
import org.sleuthkit.autopsy.ingest.IngestServiceFsContent;
import org.sleuthkit.autopsy.ingest.ServiceDataEvent;
import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskException;

//service provider registered in layer.xml
public final class KeywordSearchIngestService implements IngestServiceFsContent {

    private static final Logger logger = Logger.getLogger(KeywordSearchIngestService.class.getName());
    public static final String MODULE_NAME = "Keyword Search";
    private static KeywordSearchIngestService instance = null;
    private IngestManagerProxy managerProxy;
    private static final long MAX_STRING_EXTRACT_SIZE = 10 * (1 << 10) * (1 << 10);
    private static final long MAX_INDEX_SIZE = 200 * (1 << 10) * (1 << 10);
    private Ingester ingester;
    private volatile boolean commitIndex = false; //whether to commit index next time
    private volatile boolean runTimer = false;
    private List<Keyword> keywords; //keywords to search
    //private final Object lock = new Object();
    private Thread timer;
    private Indexer indexer;
    private SwingWorker searcher;
    private volatile boolean searcherDone = true;
    private Map<Keyword, List<FsContent>> currentResults;
    private volatile int messageID = 0;
    private volatile boolean finalRun = false;
    private final SleuthkitCase caseHandle = Case.getCurrentCase().getSleuthkitCase();
    private static final String[] ingestibleExtensions = {"tar", "jar", "zip", "bzip2",
        "gz", "tgz", "doc", "xls", "ppt", "rtf", "pdf", "html", "xhtml", "txt",
        "bmp", "gif", "png", "jpeg", "tiff", "mp3", "aiff", "au", "midi", "wav",
        "pst", "xml", "class"};

    public enum IngestStatus {

        INGESTED, EXTRACTED_INGESTED, SKIPPED,
    };
    private Map<Long, IngestStatus> ingestStatus;
    private Map<String, List<FsContent>> reportedHits; //already reported hits

    public static synchronized KeywordSearchIngestService getDefault() {
        if (instance == null) {
            instance = new KeywordSearchIngestService();
        }
        return instance;
    }

    @Override
    public void process(FsContent fsContent) {
        //check if time to commit and previous search is not running
        //commiting while searching causes performance issues
        if (commitIndex && searcherDone) {
            logger.log(Level.INFO, "Commiting index");
            commit();
            commitIndex = false;
            indexChangeNotify();
            //start search if previous not running
            if (keywords != null && !keywords.isEmpty() && searcherDone) {
                searcher = new Searcher(keywords);
                searcher.execute();
            }
        }
        indexer.indexFile(fsContent);

    }

    @Override
    public void complete() {
        logger.log(Level.INFO, "complete()");
        runTimer = false;

        //handle case if previous search running
        //cancel it, will re-run after final commit
        //note: cancellation of Searcher worker is graceful (between keywords)
        if (searcher != null) {
            searcher.cancel(true);
        }

        //final commit
        commit();

        //signal a potential change in number of indexed files
        indexChangeNotify();

        postIndexSummary();

        //run one last search as there are probably some new files committed
        if (keywords != null && !keywords.isEmpty()) {
            finalRun = true;
            searcher = new Searcher(keywords);
            searcher.execute();
        } else {
            managerProxy.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, this, "Completed"));
        }
        //postSummary();
    }

    @Override
    public void stop() {
        logger.log(Level.INFO, "stop()");

        //stop timer
        runTimer = false;
        //stop searcher
        if (searcher != null) {
            searcher.cancel(true);
        }

        //commit uncommited files, don't search again
        commit();

        indexChangeNotify();
        //postSummary();
    }

    @Override
    public String getName() {
        return MODULE_NAME;
    }

    @Override
    public void init(IngestManagerProxy managerProxy) {
        logger.log(Level.INFO, "init()");

        this.managerProxy = managerProxy;

        final Server.Core solrCore = KeywordSearch.getServer().getCore();
        ingester = solrCore.getIngester();
<commit_msg>keyword search service: get case in init not constructor, as we constructor can be now invoked before case in open<commit_after>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.keywordsearch;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.TermsResponse.Term;
import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.IngestManagerProxy;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
import org.sleuthkit.autopsy.ingest.IngestServiceFsContent;
import org.sleuthkit.autopsy.ingest.ServiceDataEvent;
import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskException;

//service provider registered in layer.xml
public final class KeywordSearchIngestService implements IngestServiceFsContent {

    private static final Logger logger = Logger.getLogger(KeywordSearchIngestService.class.getName());
    public static final String MODULE_NAME = "Keyword Search";
    private static KeywordSearchIngestService instance = null;
    private IngestManagerProxy managerProxy;
    private static final long MAX_STRING_EXTRACT_SIZE = 10 * (1 << 10) * (1 << 10);
    private static final long MAX_INDEX_SIZE = 200 * (1 << 10) * (1 << 10);
    private Ingester ingester;
    private volatile boolean commitIndex = false; //whether to commit index next time
    private volatile boolean runTimer = false;
    private List<Keyword> keywords; //keywords to search
    //private final Object lock = new Object();
    private Thread timer;
    private Indexer indexer;
    private SwingWorker searcher;
    private volatile boolean searcherDone = true;
    private Map<Keyword, List<FsContent>> currentResults;
    private volatile int messageID = 0;
    private volatile boolean finalRun = false;
    private SleuthkitCase caseHandle = null;
    private static final String[] ingestibleExtensions = {"tar", "jar", "zip", "bzip2",
        "gz", "tgz", "doc", "xls", "ppt", "rtf", "pdf", "html", "xhtml", "txt",
        "bmp", "gif", "png", "jpeg", "tiff", "mp3", "aiff", "au", "midi", "wav",
        "pst", "xml", "class"};

    public enum IngestStatus {

        INGESTED, EXTRACTED_INGESTED, SKIPPED,
    };
    private Map<Long, IngestStatus> ingestStatus;
    private Map<String, List<FsContent>> reportedHits; //already reported hits

    public static synchronized KeywordSearchIngestService getDefault() {
        if (instance == null) {
            instance = new KeywordSearchIngestService();
        }
        return instance;
    }

    @Override
    public void process(FsContent fsContent) {
        //check if time to commit and previous search is not running
        //commiting while searching causes performance issues
        if (commitIndex && searcherDone) {
            logger.log(Level.INFO, "Commiting index");
            commit();
            commitIndex = false;
            indexChangeNotify();
            //start search if previous not running
            if (keywords != null && !keywords.isEmpty() && searcherDone) {
                searcher = new Searcher(keywords);
                searcher.execute();
            }
        }
        indexer.indexFile(fsContent);

    }

    @Override
    public void complete() {
        logger.log(Level.INFO, "complete()");
        runTimer = false;

        //handle case if previous search running
        //cancel it, will re-run after final commit
        //note: cancellation of Searcher worker is graceful (between keywords)
        if (searcher != null) {
            searcher.cancel(true);
        }

        //final commit
        commit();

        //signal a potential change in number of indexed files
        indexChangeNotify();

        postIndexSummary();

        //run one last search as there are probably some new files committed
        if (keywords != null && !keywords.isEmpty()) {
            finalRun = true;
            searcher = new Searcher(keywords);
            searcher.execute();
        } else {
            managerProxy.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, this, "Completed"));
        }
        //postSummary();
    }

    @Override
    public void stop() {
        logger.log(Level.INFO, "stop()");

        //stop timer
        runTimer = false;
        //stop searcher
        if (searcher != null) {
            searcher.cancel(true);
        }

        //commit uncommited files, don't search again
        commit();

        indexChangeNotify();
        //postSummary();
    }

    @Override
    public String getName() {
        return MODULE_NAME;
    }

    @Override
    public void init(IngestManagerProxy managerProxy) {
        logger.log(Level.INFO, "init()");
        
        caseHandle = Case.getCurrentCase().getSleuthkitCase();

        this.managerProxy = managerProxy;

        final Server.Core solrCore = KeywordSearch.getServer().getCore();
        ingester = solrCore.getIngester();

{'fraction_non_alphanumeric': 0.0986386619992221, 'fraction_numerical': 0.004356281602489304, 'mean_word_length': 3.145759432441148, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1377583', 'n_tokens_mistral': 3708, 'n_tokens_neox': 3573, 'n_words': 1026}