<commit_before>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011-2016 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.keywordsearch;

import com.google.common.base.CharMatcher;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.datamodel.CreditCards;
import static org.sleuthkit.autopsy.keywordsearch.KeywordSearchSettings.MODULE_NAME;
import static org.sleuthkit.autopsy.keywordsearch.TermsComponentQuery.CREDIT_CARD_NUM_PATTERN;
import static org.sleuthkit.autopsy.keywordsearch.TermsComponentQuery.CREDIT_CARD_TRACK2_PATTERN;
import static org.sleuthkit.autopsy.keywordsearch.TermsComponentQuery.KEYWORD_SEARCH_DOCUMENT_ID;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskException;

final class RegexQuery implements KeywordSearchQuery {

    public static final Logger LOGGER = Logger.getLogger(RegexQuery.class.getName());
    private final List<KeywordQueryFilter> filters = new ArrayList<>();

    private final KeywordList keywordList;
    private final Keyword keyword;
    private String field = "content_str";
    private final String keywordString;
    static final private int MAX_RESULTS = 20000;
    private boolean escaped;
    private String escapedQuery;

    /**
     * Constructor with query to process.
     *
     * @param keywordList
     * @param keyword
     */
    RegexQuery(KeywordList keywordList, Keyword keyword) {
        this.keywordList = keywordList;
        this.keyword = keyword;
        this.keywordString = keyword.getSearchTerm();
    }

    @Override
    public KeywordList getKeywordList() {
        return keywordList;
    }

    @Override
    public boolean validate() {
        return true;
    }

    @Override
    public QueryResults performQuery() throws NoOpenCoreException {
        QueryResults results = new QueryResults(this, keywordList);

        ListMultimap<Keyword, KeywordHit> hitsMultMap = ArrayListMultimap.create();

        final Server solrServer = KeywordSearch.getServer();
        SolrQuery solrQuery = new SolrQuery();
        solrQuery.setShowDebugInfo(true); //debug

        solrQuery.setQuery((field == null ? Server.Schema.CONTENT_STR : field) + ":/.*" + getQueryString() + ".*/");
        solrQuery.setRows(MAX_RESULTS);
        if (KeywordSearchSettings.getShowSnippets()) {
            solrQuery.setFields("content_str", Server.Schema.ID.toString());
        } else {
            solrQuery.setFields(Server.Schema.ID.toString());
        }
        filters.stream()
                .map(KeywordQueryFilter::toString)
                .forEach(solrQuery::addFilterQuery);

        int start = 0;
        SolrDocumentList resultList = null;
        // cycle through results in sets of MAX_RESULTS
        while (resultList == null || start < resultList.getNumFound()) {
            solrQuery.setStart(start);

            try {
                final QueryResponse response = solrServer.query(solrQuery, SolrRequest.METHOD.POST);
                resultList = response.getResults();

                for (SolrDocument resultDoc : resultList) {

                    try {
                        List<KeywordHit> keywordHits = createKeywordtHits(resultDoc);
                        for (KeywordHit hit : keywordHits) {
                            hitsMultMap.put(new Keyword(hit.getHit(), true), hit);
                        }
                    } catch (TskException ex) {
                        //
                    }
                }
            } catch (KeywordSearchModuleException ex) {
                LOGGER.log(Level.SEVERE, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS
                MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.query.exception.msg", keywordString), ex.getCause().getMessage());
            }

            start = start + MAX_RESULTS;
        }
        for (Keyword k : hitsMultMap.keySet()) {
            results.addResult(k, hitsMultMap.get(k));
        }

        return results;
    }

    /**
     * Create the minimum set of documents. Ignores chunk IDs. Only one hit per
     * file in results.
     *
     * @param resultList
     *
     * @return
     */
    private Set<SolrDocument> filterOneHitPerDocument(SolrDocumentList resultList) {
        // sort the list so that we consistently pick the same chunk each time.
        // note this sort is doing a string comparison and not an integer comparison, so
        // chunk 10 will be smaller than chunk 9.
        Collections.sort(resultList, (SolrDocument left, SolrDocument right) -> {
            // ID is in the form of ObjectId_Chunk
            String leftID = left.getFieldValue(Server.Schema.ID.toString()).toString();
            String rightID = right.getFieldValue(Server.Schema.ID.toString()).toString();
            return leftID.compareTo(rightID);
        });

        // NOTE: We could probably just iterate through the list and compare each ID with the
        // previous ID to get the unique documents faster than using this set now that the list
        // is sorted.
        Set<SolrDocument> solrDocumentsWithMatches = new TreeSet<>(new LuceneQuery.SolrDocumentComparatorIgnoresChunkId());
        solrDocumentsWithMatches.addAll(resultList);
        return solrDocumentsWithMatches;
    }

    private List<KeywordHit> createKeywordtHits(SolrDocument solrDoc) throws TskException {

        List<KeywordHit> hits = new ArrayList<>();
        /**
         * Get the first snippet from the document if keyword search is
         * configured to use snippets.
         */
        final String docId = solrDoc.getFieldValue(Server.Schema.ID.toString()).toString();

        Collection<Object> fieldValues = solrDoc.getFieldValues("content_str");

        for (Object value : fieldValues) {
            String content = value.toString();

            Matcher hitMatcher = Pattern.compile(keywordString).matcher(content);

            while (hitMatcher.find()) {
                String snippet = "";
                final String hit = hitMatcher.group();
                /*
                 * If searching for credit card account numbers, do a Luhn check
                 * on the term and discard it if it does not pass.
                 */
                if (keyword.getArtifactAttributeType() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
                    Matcher ccnMatcher = CREDIT_CARD_NUM_PATTERN.matcher(hit);
                    ccnMatcher.find();
                    final String ccn = CharMatcher.anyOf(" -").removeFrom(ccnMatcher.group("ccn"));
                    if (false == TermsComponentQuery.CREDIT_CARD_NUM_LUHN_CHECK.isValid(ccn)) {
                        continue;
                    }
                }
<commit_msg>Make pattern matching case insensitive.<commit_after>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011-2016 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.keywordsearch;

import com.google.common.base.CharMatcher;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.datamodel.CreditCards;
import static org.sleuthkit.autopsy.keywordsearch.KeywordSearchSettings.MODULE_NAME;
import static org.sleuthkit.autopsy.keywordsearch.TermsComponentQuery.CREDIT_CARD_NUM_PATTERN;
import static org.sleuthkit.autopsy.keywordsearch.TermsComponentQuery.CREDIT_CARD_TRACK2_PATTERN;
import static org.sleuthkit.autopsy.keywordsearch.TermsComponentQuery.KEYWORD_SEARCH_DOCUMENT_ID;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskException;

final class RegexQuery implements KeywordSearchQuery {

    public static final Logger LOGGER = Logger.getLogger(RegexQuery.class.getName());
    private final List<KeywordQueryFilter> filters = new ArrayList<>();

    private final KeywordList keywordList;
    private final Keyword keyword;
    private String field = "content_str";
    private final String keywordString;
    static final private int MAX_RESULTS = 20000;
    private boolean escaped;
    private String escapedQuery;

    /**
     * Constructor with query to process.
     *
     * @param keywordList
     * @param keyword
     */
    RegexQuery(KeywordList keywordList, Keyword keyword) {
        this.keywordList = keywordList;
        this.keyword = keyword;
        this.keywordString = keyword.getSearchTerm();
    }

    @Override
    public KeywordList getKeywordList() {
        return keywordList;
    }

    @Override
    public boolean validate() {
        return true;
    }

    @Override
    public QueryResults performQuery() throws NoOpenCoreException {
        QueryResults results = new QueryResults(this, keywordList);

        ListMultimap<Keyword, KeywordHit> hitsMultMap = ArrayListMultimap.create();

        final Server solrServer = KeywordSearch.getServer();
        SolrQuery solrQuery = new SolrQuery();
        solrQuery.setShowDebugInfo(true); //debug

        solrQuery.setQuery((field == null ? Server.Schema.CONTENT_STR : field) + ":/.*" + getQueryString() + ".*/");
        solrQuery.setRows(MAX_RESULTS);
        if (KeywordSearchSettings.getShowSnippets()) {
            solrQuery.setFields("content_str", Server.Schema.ID.toString());
        } else {
            solrQuery.setFields(Server.Schema.ID.toString());
        }
        filters.stream()
                .map(KeywordQueryFilter::toString)
                .forEach(solrQuery::addFilterQuery);

        int start = 0;
        SolrDocumentList resultList = null;
        // cycle through results in sets of MAX_RESULTS
        while (resultList == null || start < resultList.getNumFound()) {
            solrQuery.setStart(start);

            try {
                final QueryResponse response = solrServer.query(solrQuery, SolrRequest.METHOD.POST);
                resultList = response.getResults();

                for (SolrDocument resultDoc : resultList) {

                    try {
                        List<KeywordHit> keywordHits = createKeywordtHits(resultDoc);
                        for (KeywordHit hit : keywordHits) {
                            hitsMultMap.put(new Keyword(hit.getHit(), true), hit);
                        }
                    } catch (TskException ex) {
                        //
                    }
                }
            } catch (KeywordSearchModuleException ex) {
                LOGGER.log(Level.SEVERE, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS
                MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.query.exception.msg", keywordString), ex.getCause().getMessage());
            }

            start = start + MAX_RESULTS;
        }
        for (Keyword k : hitsMultMap.keySet()) {
            results.addResult(k, hitsMultMap.get(k));
        }

        return results;
    }

    /**
     * Create the minimum set of documents. Ignores chunk IDs. Only one hit per
     * file in results.
     *
     * @param resultList
     *
     * @return
     */
    private Set<SolrDocument> filterOneHitPerDocument(SolrDocumentList resultList) {
        // sort the list so that we consistently pick the same chunk each time.
        // note this sort is doing a string comparison and not an integer comparison, so
        // chunk 10 will be smaller than chunk 9.
        Collections.sort(resultList, (SolrDocument left, SolrDocument right) -> {
            // ID is in the form of ObjectId_Chunk
            String leftID = left.getFieldValue(Server.Schema.ID.toString()).toString();
            String rightID = right.getFieldValue(Server.Schema.ID.toString()).toString();
            return leftID.compareTo(rightID);
        });

        // NOTE: We could probably just iterate through the list and compare each ID with the
        // previous ID to get the unique documents faster than using this set now that the list
        // is sorted.
        Set<SolrDocument> solrDocumentsWithMatches = new TreeSet<>(new LuceneQuery.SolrDocumentComparatorIgnoresChunkId());
        solrDocumentsWithMatches.addAll(resultList);
        return solrDocumentsWithMatches;
    }

    private List<KeywordHit> createKeywordtHits(SolrDocument solrDoc) throws TskException {

        List<KeywordHit> hits = new ArrayList<>();
        /**
         * Get the first snippet from the document if keyword search is
         * configured to use snippets.
         */
        final String docId = solrDoc.getFieldValue(Server.Schema.ID.toString()).toString();

        Collection<Object> fieldValues = solrDoc.getFieldValues("content_str");

        for (Object value : fieldValues) {
            String content = value.toString();

            Matcher hitMatcher = Pattern.compile(keywordString, Pattern.CASE_INSENSITIVE).matcher(content);

            while (hitMatcher.find()) {
                String snippet = "";
                final String hit = hitMatcher.group();
                /*
                 * If searching for credit card account numbers, do a Luhn check
                 * on the term and discard it if it does not pass.
                 */
                if (keyword.getArtifactAttributeType() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
                    Matcher ccnMatcher = CREDIT_CARD_NUM_PATTERN.matcher(hit);
                    ccnMatcher.find();
                    final String ccn = CharMatcher.anyOf(" -").removeFrom(ccnMatcher.group("ccn"));
                    if (false == TermsComponentQuery.CREDIT_CARD_NUM_LUHN_CHECK.isValid(ccn)) {
                        continue;
                    }
                }

{'fraction_non_alphanumeric': 0.09001665873393622, 'fraction_numerical': 0.002617801047120419, 'mean_word_length': 2.6910408432147563, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1891234', 'n_tokens_mistral': 4717, 'n_tokens_neox': 4511, 'n_words': 1135}