<commit_before>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2013-16 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.report;

import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimaps;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.ImageUtils;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifactTag;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.Type;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentTag;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;

class TableReportGenerator {

    private final List<BlackboardArtifact.Type> artifactTypes = new ArrayList<>();
    private final HashSet<String> tagNamesFilter = new HashSet<>();

    private final Set<Content> images = new HashSet<>();
    private final ReportProgressPanel progressPanel;
    private final TableReportModule tableReport;
    private final Map<Integer, List<Column>> columnHeaderMap;
    private static final Logger logger = Logger.getLogger(TableReportGenerator.class.getName());

    private final List<String> errorList;

    TableReportGenerator(Map<BlackboardArtifact.Type, Boolean> artifactTypeSelections, Map<String, Boolean> tagNameSelections, ReportProgressPanel progressPanel, TableReportModule tableReport) {

        this.progressPanel = progressPanel;
        this.tableReport = tableReport;
        this.columnHeaderMap = new HashMap<>();
        errorList = new ArrayList<>();
        // Get the artifact types selected by the user.
        for (Map.Entry<BlackboardArtifact.Type, Boolean> entry : artifactTypeSelections.entrySet()) {
            if (entry.getValue()) {
                artifactTypes.add(entry.getKey());
            }
        }

        // Get the tag names selected by the user and make a tag names filter.
        if (null != tagNameSelections) {
            for (Map.Entry<String, Boolean> entry : tagNameSelections.entrySet()) {
                if (entry.getValue() == true) {
                    tagNamesFilter.add(entry.getKey());
                }
            }
        }
    }

    protected void execute() {
        // Start the progress indicators for each active TableReportModule.

        progressPanel.start();
        progressPanel.setIndeterminate(false);
        progressPanel.setMaximumProgress(this.artifactTypes.size() + 2); // +2 for content and blackboard artifact tags
        // report on the blackboard results
        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            makeBlackboardArtifactTables();
        }

        // report on the tagged files and artifacts
        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            makeContentTagsTables();
        }

        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            makeBlackboardArtifactTagsTables();
        }

        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            // report on the tagged images
            makeThumbnailTable();
        }
    }

    /**
     * Generate the tables for the selected blackboard artifacts
     */
    private void makeBlackboardArtifactTables() {
        // Make a comment string describing the tag names filter in effect. 
        String comment = "";
        if (!tagNamesFilter.isEmpty()) {
            comment += NbBundle.getMessage(this.getClass(), "ReportGenerator.artifactTable.taggedResults.text");
            comment += makeCommaSeparatedList(tagNamesFilter);
        }

        // Add a table to the report for every enabled blackboard artifact type.
        for (BlackboardArtifact.Type type : artifactTypes) {
            // Check for cancellaton.

            if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) {
                return;
            }

            progressPanel.updateStatusLabel(
                    NbBundle.getMessage(this.getClass(), "ReportGenerator.progress.processing",
                            type.getDisplayName()));

            // Keyword hits and hashset hit artifacts get special handling.
            if (type.getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) {
                writeKeywordHits(tableReport, comment, tagNamesFilter);
                continue;
            } else if (type.getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID()) {
                writeHashsetHits(tableReport, comment, tagNamesFilter);
                continue;
            }

            List<ArtifactData> artifactList = getFilteredArtifacts(type, tagNamesFilter);

            if (artifactList.isEmpty()) {
                continue;
            }

            /* TSK_ACCOUNT artifacts get grouped by their TSK_ACCOUNT_TYPE
             * attribute, and then handed off to the standard method for writing
             * tables. */
            if (type.getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID()) {
                //Group account artifacts by their account type
                ListMultimap<String, ArtifactData> groupedArtifacts = Multimaps.index(artifactList,
                        artifactData -> {
                            try {
                                return artifactData.getArtifact().getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE)).getValueString();
                            } catch (TskCoreException ex) {
                                logger.log(Level.SEVERE, "Unable to get value of TSK_ACCOUNT_TYPE attribute. Defaulting to \"unknown\"", ex);
                                return "unknown";
                            }
                        });
                for (String accountTypeStr : groupedArtifacts.keySet()) {
                    /* If the report is a ReportHTML, the data type name
                     * eventualy makes it to useDataTypeIcon which expects but
                     * does not require a artifact name, so we make a synthetic
                     * compund name by appending a ":" and the account type.
                     */
                    String accountDisplayname = accountTypeStr;
                    if (accountTypeStr != null) {
                        try {
                            Account.Type acctType = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().getAccountType(accountTypeStr);
                            if (acctType != null) {
                                 accountDisplayname = acctType.getDisplayName();
                            }
                        }
                        catch (TskCoreException ex) {
                            logger.log(Level.SEVERE, "Unable to get dusplay name for account type.", ex);   <commit_msg>Fixed a typo in previous commit.<commit_after>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2013-16 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.report;

import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimaps;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.ImageUtils;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifactTag;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.Type;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentTag;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;

class TableReportGenerator {

    private final List<BlackboardArtifact.Type> artifactTypes = new ArrayList<>();
    private final HashSet<String> tagNamesFilter = new HashSet<>();

    private final Set<Content> images = new HashSet<>();
    private final ReportProgressPanel progressPanel;
    private final TableReportModule tableReport;
    private final Map<Integer, List<Column>> columnHeaderMap;
    private static final Logger logger = Logger.getLogger(TableReportGenerator.class.getName());

    private final List<String> errorList;

    TableReportGenerator(Map<BlackboardArtifact.Type, Boolean> artifactTypeSelections, Map<String, Boolean> tagNameSelections, ReportProgressPanel progressPanel, TableReportModule tableReport) {

        this.progressPanel = progressPanel;
        this.tableReport = tableReport;
        this.columnHeaderMap = new HashMap<>();
        errorList = new ArrayList<>();
        // Get the artifact types selected by the user.
        for (Map.Entry<BlackboardArtifact.Type, Boolean> entry : artifactTypeSelections.entrySet()) {
            if (entry.getValue()) {
                artifactTypes.add(entry.getKey());
            }
        }

        // Get the tag names selected by the user and make a tag names filter.
        if (null != tagNameSelections) {
            for (Map.Entry<String, Boolean> entry : tagNameSelections.entrySet()) {
                if (entry.getValue() == true) {
                    tagNamesFilter.add(entry.getKey());
                }
            }
        }
    }

    protected void execute() {
        // Start the progress indicators for each active TableReportModule.

        progressPanel.start();
        progressPanel.setIndeterminate(false);
        progressPanel.setMaximumProgress(this.artifactTypes.size() + 2); // +2 for content and blackboard artifact tags
        // report on the blackboard results
        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            makeBlackboardArtifactTables();
        }

        // report on the tagged files and artifacts
        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            makeContentTagsTables();
        }

        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            makeBlackboardArtifactTagsTables();
        }

        if (progressPanel.getStatus() != ReportProgressPanel.ReportStatus.CANCELED) {
            // report on the tagged images
            makeThumbnailTable();
        }
    }

    /**
     * Generate the tables for the selected blackboard artifacts
     */
    private void makeBlackboardArtifactTables() {
        // Make a comment string describing the tag names filter in effect. 
        String comment = "";
        if (!tagNamesFilter.isEmpty()) {
            comment += NbBundle.getMessage(this.getClass(), "ReportGenerator.artifactTable.taggedResults.text");
            comment += makeCommaSeparatedList(tagNamesFilter);
        }

        // Add a table to the report for every enabled blackboard artifact type.
        for (BlackboardArtifact.Type type : artifactTypes) {
            // Check for cancellaton.

            if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) {
                return;
            }

            progressPanel.updateStatusLabel(
                    NbBundle.getMessage(this.getClass(), "ReportGenerator.progress.processing",
                            type.getDisplayName()));

            // Keyword hits and hashset hit artifacts get special handling.
            if (type.getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) {
                writeKeywordHits(tableReport, comment, tagNamesFilter);
                continue;
            } else if (type.getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID()) {
                writeHashsetHits(tableReport, comment, tagNamesFilter);
                continue;
            }

            List<ArtifactData> artifactList = getFilteredArtifacts(type, tagNamesFilter);

            if (artifactList.isEmpty()) {
                continue;
            }

            /* TSK_ACCOUNT artifacts get grouped by their TSK_ACCOUNT_TYPE
             * attribute, and then handed off to the standard method for writing
             * tables. */
            if (type.getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID()) {
                //Group account artifacts by their account type
                ListMultimap<String, ArtifactData> groupedArtifacts = Multimaps.index(artifactList,
                        artifactData -> {
                            try {
                                return artifactData.getArtifact().getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE)).getValueString();
                            } catch (TskCoreException ex) {
                                logger.log(Level.SEVERE, "Unable to get value of TSK_ACCOUNT_TYPE attribute. Defaulting to \"unknown\"", ex);
                                return "unknown";
                            }
                        });
                for (String accountTypeStr : groupedArtifacts.keySet()) {
                    /* If the report is a ReportHTML, the data type name
                     * eventualy makes it to useDataTypeIcon which expects but
                     * does not require a artifact name, so we make a synthetic
                     * compund name by appending a ":" and the account type.
                     */
                    String accountDisplayname = accountTypeStr;
                    if (accountTypeStr != null) {
                        try {
                            Account.Type acctType = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().getAccountType(accountTypeStr);
                            if (acctType != null) {
                                 accountDisplayname = acctType.getDisplayName();
                            }
                        }
                        catch (TskCoreException ex) {
                            logger.log(Level.SEVERE, "Unable to get display name for account type.", ex);   
{'fraction_non_alphanumeric': 0.08583049225432143, 'fraction_numerical': 0.0014354925533823794, 'mean_word_length': 2.4382068681883613, 'pattern_counts': {'":': 2, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7486633', 'n_tokens_mistral': 4417, 'n_tokens_neox': 4226, 'n_words': 1087}