<commit_before>/*
 * Enterprise Artifacts Manager
 *
 * Copyright 2015-2017 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.ingestmodule;

import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamCase;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamDbException;
import java.util.List;
import java.util.logging.Level;
import java.util.stream.Collectors;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.services.Blackboard;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamArtifact;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamArtifactInstance;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamDataSource;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamOrganization;
import org.sleuthkit.datamodel.TskDataException;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamDb;

/**
 * Ingest module for inserting entries into the enterprise artifacts manager
 * database on ingest of a data source
 */
@Messages({"enterpriseartifactsmanager.prevcases.text=Previous Cases",
    "enterpriseartifactsmanager.ingestmodule.name=Enterprise Artifacts Manager"})
class IngestModule implements FileIngestModule {

    private final static Logger LOGGER = Logger.getLogger(IngestModule.class.getName());
    private final IngestServices services = IngestServices.getInstance();
    private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
    private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
    private long jobId;
    private EamCase eamCase;
    private EamDataSource eamDataSource;
    private Blackboard blackboard;
    private EamArtifact.Type filesType;

    @Override
    public ProcessResult process(AbstractFile af) {
        if (Boolean.parseBoolean(ModuleSettings.getConfigSetting("EnterpriseArtifactsManager", "db.enabled")) == false
                || EamDb.getInstance().isEnabled() == false) {
            /*
             * Not signaling an error for now. This is a workaround for the way
             * all newly didscovered ingest modules are automatically anabled.
             *
             * TODO (JIRA-2731): Add isEnabled API for ingest modules.
             */
            return ProcessResult.OK;
        }

        blackboard = Case.getCurrentCase().getServices().getBlackboard();

        if ((af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
                || (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
                || (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)
                || (af.getKnown() == TskData.FileKnown.KNOWN)
                || (af.isDir() == true)) {
            return ProcessResult.OK;
        }

        EamDb dbManager = EamDb.getInstance();

        // only continue if we are correlating filesType
        if (!filesType.isEnabled()) {
            return ProcessResult.OK;
        }

        // get the hash because we're going to correlate it
        String md5 = af.getMd5Hash();
        if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) {
            return ProcessResult.OK;
        }

        EamArtifact eamArtifact = new EamArtifact(filesType, md5);

        // If unknown to both the hash module and as a globally known artifact in the EAM DB, correlate to other cases
        if (af.getKnown() == TskData.FileKnown.UNKNOWN) {
            // query db for artifact instances having this MD5 and knownStatus = "Bad".
            try {
                // if af.getKnown() is "UNKNOWN" and this artifact instance was marked bad in a previous case, 
                // create TSK_INTERESTING_FILE artifact on BB.
                List<String> caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact);
                if (!caseDisplayNames.isEmpty()) {
                    postCorrelatedBadFileToBlackboard(af, caseDisplayNames);
                }
            } catch (EamDbException ex) {
                LOGGER.log(Level.SEVERE, "Error counting known bad artifacts.", ex); // NON-NLS
                return ProcessResult.ERROR;
            }
        }

        // Make a TSK_HASHSET_HIT blackboard artifact for global known bad files
        try {
            if (dbManager.isArtifactGlobalKnownBad(eamArtifact)) {
                postCorrelatedHashHitToBlackboard(af);
            }
        } catch (EamDbException ex) {
            LOGGER.log(Level.SEVERE, "Error retrieving global known status.", ex); // NON-NLS
            return ProcessResult.ERROR;
        }

        try {
            EamArtifactInstance cefi = new EamArtifactInstance(
                    eamCase,
                    eamDataSource,
                    af.getParentPath() + af.getName(),
                    "",
                    EamArtifactInstance.KnownStatus.UNKNOWN,
                    EamArtifactInstance.GlobalStatus.LOCAL
            );
            eamArtifact.addInstance(cefi);
            dbManager.prepareBulkArtifact(eamArtifact);
        } catch (EamDbException ex) {
            LOGGER.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
            return ProcessResult.ERROR;
        }

        return ProcessResult.OK;
    }

    @Override
    public void shutDown() {
        if (Boolean.parseBoolean(ModuleSettings.getConfigSetting("EnterpriseArtifactManager", "db.enabled")) == false<commit_msg>Correct typo in EAM IngestModule.shutDown<commit_after>/*
 * Enterprise Artifacts Manager
 *
 * Copyright 2015-2017 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.ingestmodule;

import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamCase;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamDbException;
import java.util.List;
import java.util.logging.Level;
import java.util.stream.Collectors;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.services.Blackboard;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamArtifact;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamArtifactInstance;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamDataSource;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamOrganization;
import org.sleuthkit.datamodel.TskDataException;
import org.sleuthkit.autopsy.experimental.enterpriseartifactsmanager.datamodel.EamDb;

/**
 * Ingest module for inserting entries into the enterprise artifacts manager
 * database on ingest of a data source
 */
@Messages({"enterpriseartifactsmanager.prevcases.text=Previous Cases",
    "enterpriseartifactsmanager.ingestmodule.name=Enterprise Artifacts Manager"})
class IngestModule implements FileIngestModule {

    private final static Logger LOGGER = Logger.getLogger(IngestModule.class.getName());
    private final IngestServices services = IngestServices.getInstance();
    private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
    private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
    private long jobId;
    private EamCase eamCase;
    private EamDataSource eamDataSource;
    private Blackboard blackboard;
    private EamArtifact.Type filesType;

    @Override
    public ProcessResult process(AbstractFile af) {
        if (Boolean.parseBoolean(ModuleSettings.getConfigSetting("EnterpriseArtifactsManager", "db.enabled")) == false
                || EamDb.getInstance().isEnabled() == false) {
            /*
             * Not signaling an error for now. This is a workaround for the way
             * all newly didscovered ingest modules are automatically anabled.
             *
             * TODO (JIRA-2731): Add isEnabled API for ingest modules.
             */
            return ProcessResult.OK;
        }

        blackboard = Case.getCurrentCase().getServices().getBlackboard();

        if ((af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
                || (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
                || (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)
                || (af.getKnown() == TskData.FileKnown.KNOWN)
                || (af.isDir() == true)) {
            return ProcessResult.OK;
        }

        EamDb dbManager = EamDb.getInstance();

        // only continue if we are correlating filesType
        if (!filesType.isEnabled()) {
            return ProcessResult.OK;
        }

        // get the hash because we're going to correlate it
        String md5 = af.getMd5Hash();
        if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) {
            return ProcessResult.OK;
        }

        EamArtifact eamArtifact = new EamArtifact(filesType, md5);

        // If unknown to both the hash module and as a globally known artifact in the EAM DB, correlate to other cases
        if (af.getKnown() == TskData.FileKnown.UNKNOWN) {
            // query db for artifact instances having this MD5 and knownStatus = "Bad".
            try {
                // if af.getKnown() is "UNKNOWN" and this artifact instance was marked bad in a previous case, 
                // create TSK_INTERESTING_FILE artifact on BB.
                List<String> caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact);
                if (!caseDisplayNames.isEmpty()) {
                    postCorrelatedBadFileToBlackboard(af, caseDisplayNames);
                }
            } catch (EamDbException ex) {
                LOGGER.log(Level.SEVERE, "Error counting known bad artifacts.", ex); // NON-NLS
                return ProcessResult.ERROR;
            }
        }

        // Make a TSK_HASHSET_HIT blackboard artifact for global known bad files
        try {
            if (dbManager.isArtifactGlobalKnownBad(eamArtifact)) {
                postCorrelatedHashHitToBlackboard(af);
            }
        } catch (EamDbException ex) {
            LOGGER.log(Level.SEVERE, "Error retrieving global known status.", ex); // NON-NLS
            return ProcessResult.ERROR;
        }

        try {
            EamArtifactInstance cefi = new EamArtifactInstance(
                    eamCase,
                    eamDataSource,
                    af.getParentPath() + af.getName(),
                    "",
                    EamArtifactInstance.KnownStatus.UNKNOWN,
                    EamArtifactInstance.GlobalStatus.LOCAL
            );
            eamArtifact.addInstance(cefi);
            dbManager.prepareBulkArtifact(eamArtifact);
        } catch (EamDbException ex) {
            LOGGER.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
            return ProcessResult.ERROR;
        }

        return ProcessResult.OK;
    }

    @Override
    public void shutDown() {
        if (Boolean.parseBoolean(ModuleSettings.getConfigSetting("EnterpriseArtifactsManager", "db.enabled")) == false
{'fraction_non_alphanumeric': 0.08506616257088846, 'fraction_numerical': 0.00322061191626409, 'mean_word_length': 3.081142857142857, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2684731', 'n_tokens_mistral': 4186, 'n_tokens_neox': 3975, 'n_words': 913}