<commit_before>package org.modmine.web;

/*
 * Copyright (C) 2002-2011 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.intermine.api.InterMineAPI;
import org.intermine.metadata.AttributeDescriptor;
import org.intermine.metadata.ClassDescriptor;
import org.intermine.metadata.Model;
import org.intermine.model.bio.Antibody;
import org.intermine.model.bio.Gene;
import org.intermine.model.bio.Strain;
import org.intermine.model.bio.Submission;
import org.intermine.model.bio.SubmissionProperty;
import org.intermine.objectstore.ObjectStore;
import org.intermine.util.DynamicUtil;
import org.intermine.util.TypeUtil;

/**
 * allows for full-text searches over all metadata using the apache lucene
 * engine
 * @author nils
 */
public class ModMineSearch
{
    // public static final String SEARCH_KEY = "modminesearch";

    /**
     * maximum number of hits returned
     */
    public static final int MAX_HITS = 500;

    private static final Logger LOG = Logger.getLogger(ModMineSearch.class);
    private static final String[] BOOLEAN_WORDS = {"AND", "OR", "NOT"};

    private static RAMDirectory ram = null;
    private static HashSet<String> fieldNames = new HashSet<String>();
    private static HashMap<String, Float> fieldBoosts = new HashMap<String, Float>();

    /**
     * index document metadata in preparation for first search
     * @param im
     *            API for accessing object store
     */
    public static synchronized void initModMineSearch(InterMineAPI im) {
        if (ram == null) {

            // Map<Integer, Set<String>> subProps =
            // readSubmissionProperties(im);
            Set<Document> docs = readSubmissionsFromCache(im.getObjectStore());
            indexMetadata(docs);

            LOG.debug("Field names: " + fieldNames.toString());
            LOG.debug("Boosts: " + fieldBoosts.toString());
        }
    }

    /**
     * perform a keyword search over all document metadata fields with lucene
     * @param searchString
     *            string to search for
     * @return map of document IDs with their respective scores
     */
    public static Map<Integer, Float> runLuceneSearch(String searchString) {
        LinkedHashMap<Integer, Float> matches = new LinkedHashMap<Integer, Float>();

        String queryString = parseQueryString(searchString).toLowerCase();

        long time = System.currentTimeMillis();

        try {
            IndexSearcher searcher = new IndexSearcher(ram);
            Analyzer analyzer = new WhitespaceAnalyzer();
            org.apache.lucene.search.Query query;

            // pass entire list of field names to the multi-field parser
            // => search through all fields
            String[] fieldNamesArray = new String[fieldNames.size()];
            fieldNames.toArray(fieldNamesArray);
            QueryParser queryParser = new MultiFieldQueryParser(Version.LUCENE_30, fieldNamesArray,
                    analyzer, fieldBoosts);
            query = queryParser.parse(queryString);

            // required to expand search terms
            query = query.rewrite(IndexReader.open(ram));
            LOG.debug("Actual query: " + query);

            TopDocs topDocs = searcher.search(query, 500);

            time = System.currentTimeMillis() - time;
            LOG.info("Found " + topDocs.totalHits + " document(s) that matched query '"
                    + queryString + "' in " + time + " milliseconds:");
<commit_msg>corrected bug (was lowercasing the boolean operators)<commit_after>package org.modmine.web;

/*
 * Copyright (C) 2002-2011 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.intermine.api.InterMineAPI;
import org.intermine.metadata.AttributeDescriptor;
import org.intermine.metadata.ClassDescriptor;
import org.intermine.metadata.Model;
import org.intermine.model.bio.Antibody;
import org.intermine.model.bio.Gene;
import org.intermine.model.bio.Strain;
import org.intermine.model.bio.Submission;
import org.intermine.model.bio.SubmissionProperty;
import org.intermine.objectstore.ObjectStore;
import org.intermine.util.DynamicUtil;
import org.intermine.util.TypeUtil;

/**
 * allows for full-text searches over all metadata using the apache lucene
 * engine
 * @author nils
 */
public class ModMineSearch
{
    // public static final String SEARCH_KEY = "modminesearch";

    /**
     * maximum number of hits returned
     */
    public static final int MAX_HITS = 500;

    private static final Logger LOG = Logger.getLogger(ModMineSearch.class);
    private static final String[] BOOLEAN_WORDS = {"AND", "OR", "NOT"};

    private static RAMDirectory ram = null;
    private static HashSet<String> fieldNames = new HashSet<String>();
    private static HashMap<String, Float> fieldBoosts = new HashMap<String, Float>();

    /**
     * index document metadata in preparation for first search
     * @param im
     *            API for accessing object store
     */
    public static synchronized void initModMineSearch(InterMineAPI im) {
        if (ram == null) {

            // Map<Integer, Set<String>> subProps =
            // readSubmissionProperties(im);
            Set<Document> docs = readSubmissionsFromCache(im.getObjectStore());
            indexMetadata(docs);

            LOG.debug("Field names: " + fieldNames.toString());
            LOG.debug("Boosts: " + fieldBoosts.toString());
        }
    }

    
    private static String prepareQueryString(String formInput)
    {
        // to lowercase the search string terms but not the operators
        // TODO it should probably go into the parseQueryString method
        String[] result = formInput.split("\\s");
        StringBuffer newString = new StringBuffer();
        for (String token : result) {
            if (token.equalsIgnoreCase("AND") ||
                    token.equalsIgnoreCase("OR") ||
                    token.equalsIgnoreCase("NOT"))
            {
                newString.append(token.toUpperCase() + " ");
            }
            else {
                newString.append(token.toLowerCase() + " ");
            }
        }
        LOG.debug("QUERYSTRING " + newString.toString());        
        return newString.toString();
    }

    
    /**
     * perform a keyword search over all document metadata fields with lucene
     * @param searchString
     *            string to search for
     * @return map of document IDs with their respective scores
     */
    public static Map<Integer, Float> runLuceneSearch(String searchString) {
        LinkedHashMap<Integer, Float> matches = new LinkedHashMap<Integer, Float>();        
        String queryString = parseQueryString(prepareQueryString(searchString));

        long time = System.currentTimeMillis();

        try {
            IndexSearcher searcher = new IndexSearcher(ram);
            Analyzer analyzer = new WhitespaceAnalyzer();
            org.apache.lucene.search.Query query;

            // pass entire list of field names to the multi-field parser
            // => search through all fields
            String[] fieldNamesArray = new String[fieldNames.size()];
            fieldNames.toArray(fieldNamesArray);
            QueryParser queryParser = new MultiFieldQueryParser(Version.LUCENE_30, fieldNamesArray,
                    analyzer, fieldBoosts);
            query = queryParser.parse(queryString);

            // required to expand search terms
            query = query.rewrite(IndexReader.open(ram));
            LOG.debug("Actual query: " + query);

            TopDocs topDocs = searcher.search(query, 500);

            time = System.currentTimeMillis() - time;
            LOG.info("Found " + topDocs.totalHits + " document(s) that matched query '"
                    + queryString + "' in " + time + " milliseconds:");

{'fraction_non_alphanumeric': 0.09967042844302407, 'fraction_numerical': 0.003395585738539898, 'mean_word_length': 3.0056, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6982627', 'n_tokens_mistral': 2811, 'n_tokens_neox': 2715, 'n_words': 700}