<commit_before>/* $This file is distributed under the terms of the license in /doc/license.txt$ */

package edu.cornell.mannlib.vitro.webapp.search.controller;

import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.Arrays;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import dk.dtu.adm.rap.search.RAPSearchFacets;
import dk.dtu.adm.rap.search.SearchFacet;
import dk.dtu.adm.rap.search.SearchFacetCategory;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder.ParamMap;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupsForRequest;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache;
import edu.cornell.mannlib.vitro.webapp.i18n.I18n;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult;
import edu.ucsf.vitro.opensocial.OpenSocialManager;

/**
 * Paged search controller that uses the search engine
 */

public class PagedSearchController extends FreemarkerHttpServlet {

    private static final long serialVersionUID = 1L;
    private static final Log log = LogFactory.getLog(PagedSearchController.class);

    protected static final int DEFAULT_HITS_PER_PAGE = 25;
    protected static final int DEFAULT_MAX_HIT_COUNT = 1000;

    private static final String PARAM_XML_REQUEST = "xml";
    private static final String PARAM_CSV_REQUEST = "csv";
    private static final String PARAM_START_INDEX = "startIndex";
    private static final String PARAM_HITS_PER_PAGE = "hitsPerPage";
    private static final String PARAM_CLASSGROUP = "classgroup";
    private static final String PARAM_RDFTYPE = "type";
    // RAP make this field public
    public static final String PARAM_QUERY_TEXT = "querytext";
    public static final String FACET_FIELD_PREFIX = "facet_";
    public static final String PARAM_FACET_AS_TEXT = "facetAsText";
    public static final String PARAM_FACET_TEXT_VALUE = "facetTextValue";

    protected static final Map<Format,Map<Result,String>> templateTable;
    // RAP
    //protected static final Map<String, String> facetPublicNameTable;

    protected enum Format {
        HTML, XML, CSV;
    }

    protected enum Result {
        PAGED, ERROR, BAD_QUERY
    }

    static{
        templateTable = setupTemplateTable();
    }

    /**
     * Overriding doGet from FreemarkerHttpController to do a page template (as
     * opposed to body template) style output for XML requests.
     *
     * This follows the pattern in AutocompleteController.java.
     */
    @Override
    public void doGet(HttpServletRequest request, HttpServletResponse response)
            throws IOException, ServletException {
        VitroRequest vreq = new VitroRequest(request);
        boolean wasXmlRequested = isRequestedFormatXml(vreq);
        boolean wasCSVRequested = isRequestedFormatCSV(vreq);
        if( !wasXmlRequested && !wasCSVRequested){
            super.doGet(vreq,response);
        }else if (wasXmlRequested){
            try {
                ResponseValues rvalues = processRequest(vreq);

                response.setCharacterEncoding("UTF-8");
                response.setContentType("text/xml;charset=UTF-8");
                response.setHeader("Content-Disposition", "attachment; filename=search.xml");
                writeTemplate(rvalues.getTemplateName(), rvalues.getMap(), request, response);
            } catch (Exception e) {
                log.error(e, e);
            }
        }else if (wasCSVRequested){
        	try {
                ResponseValues rvalues = processRequest(vreq);

                response.setCharacterEncoding("UTF-8");
                response.setContentType("text/csv;charset=UTF-8");
                response.setHeader("Content-Disposition", "attachment; filename=search.csv");
                writeTemplate(rvalues.getTemplateName(), rvalues.getMap(), request, response);
            } catch (Exception e) {
                log.error(e, e);
            }
        }
    }

    @Override
    protected ResponseValues processRequest(VitroRequest vreq) {    	    	
    	
        //There may be other non-html formats in the future
        Format format = getFormat(vreq);
        boolean wasXmlRequested = Format.XML == format;
        boolean wasCSVRequested = Format.CSV == format;
        log.debug("Requested format was " + (wasXmlRequested ? "xml" : "html"));
        boolean wasHtmlRequested = ! (wasXmlRequested || wasCSVRequested);

        try {

            //make sure an IndividualDao is available
            if( vreq.getWebappDaoFactory() == null
                    || vreq.getWebappDaoFactory().getIndividualDao() == null ){
                log.error("Could not get webappDaoFactory or IndividualDao");
                throw new Exception("Could not access model.");
            }
            IndividualDao iDao = vreq.getWebappDaoFactory().getIndividualDao();
            VClassGroupDao grpDao = vreq.getWebappDaoFactory().getVClassGroupDao();
            VClassDao vclassDao = vreq.getWebappDaoFactory().getVClassDao();

            ApplicationBean appBean = vreq.getAppBean();

            log.debug("IndividualDao is " + iDao.toString() + " Public classes in the classgroup are " + grpDao.getPublicGroupsWithVClasses().toString());
            log.debug("VClassDao is "+ vclassDao.toString() );

            int startIndex = getStartIndex(vreq);
            int hitsPerPage = getHitsPerPage( vreq );

            String queryText = vreq.getParameter(PARAM_QUERY_TEXT);
            log.debug("Query text is \""+ queryText + "\"");

// RAP: allow empty searches
            if(queryText == null) {
                queryText = "";
            }
//            String badQueryMsg = badQueryText( queryText, vreq );
//            if( badQueryMsg != null ){
//                return doFailedSearch(badQueryMsg, queryText, format, vreq);
//            }

            SearchQuery query = getQuery(queryText, hitsPerPage, startIndex, vreq);
            SearchEngine search = ApplicationUtils.instance().getSearchEngine();
            SearchResponse response = null;

            try {
                response = search.query(query);
            } catch (Exception ex) {
                String msg = makeBadSearchMessage(queryText, ex.getMessage(), vreq);
                log.error("could not run search query",ex);
                return doFailedSearch(msg, queryText, format, vreq);
            }

            if (response == null) {
                log.error("Search response was null");
                return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText, format, vreq);
            }

            SearchResultDocumentList docs = response.getResults();
            if (docs == null) {
                log.error("Document list for a search was null");
                return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText,format, vreq);
            }

            long hitCount = docs.getNumFound();
            log.debug("Number of hits = " + hitCount);
            if ( hitCount < 1 ) {
                return doNoHits(queryText, format, vreq);
            }

            List<Individual> individuals = new ArrayList<Individual>(docs.size());
            Iterator<SearchResultDocument> docIter = docs.iterator();
            while( docIter.hasNext() ){
                try {
                    SearchResultDocument doc = docIter.next();
                    String uri = doc.getStringValue(VitroSearchTermNames.URI);
                    Individual ind = iDao.getIndividualByURI(uri);
                    if(ind != null) {
                      ind.setSearchSnippet( getSnippet(doc, response) );
                      individuals.add(ind);
                    }
                } catch(Exception e) {
                    log.error("Problem getting usable individuals from search hits. ",e);
                }
            }

            ParamMap pagingLinkParams = new ParamMap();
            pagingLinkParams.put(PARAM_QUERY_TEXT, queryText);
            pagingLinkParams.put(PARAM_HITS_PER_PAGE, String.valueOf(hitsPerPage));

            if( wasXmlRequested ){
                pagingLinkParams.put(PARAM_XML_REQUEST,"1");
            }

            /* Compile the data for the templates */

            Map<String, Object> body = new HashMap<String, Object>();

            String classGroupParam = vreq.getParameter(PARAM_CLASSGROUP);
            log.debug("ClassGroupParam is \""+ classGroupParam + "\"");<commit_msg>Changed the load of parameters for the prev/next page so we do not loose facets.<commit_after>/* $This file is distributed under the terms of the license in /doc/license.txt$ */

package edu.cornell.mannlib.vitro.webapp.search.controller;

import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.Arrays;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import dk.dtu.adm.rap.search.RAPSearchFacets;
import dk.dtu.adm.rap.search.SearchFacet;
import dk.dtu.adm.rap.search.SearchFacetCategory;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder.ParamMap;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupsForRequest;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache;
import edu.cornell.mannlib.vitro.webapp.i18n.I18n;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult;
import edu.ucsf.vitro.opensocial.OpenSocialManager;

/**
 * Paged search controller that uses the search engine
 */

public class PagedSearchController extends FreemarkerHttpServlet {

    private static final long serialVersionUID = 1L;
    private static final Log log = LogFactory.getLog(PagedSearchController.class);

    protected static final int DEFAULT_HITS_PER_PAGE = 25;
    protected static final int DEFAULT_MAX_HIT_COUNT = 1000;

    private static final String PARAM_XML_REQUEST = "xml";
    private static final String PARAM_CSV_REQUEST = "csv";
    private static final String PARAM_START_INDEX = "startIndex";
    private static final String PARAM_HITS_PER_PAGE = "hitsPerPage";
    private static final String PARAM_CLASSGROUP = "classgroup";
    private static final String PARAM_RDFTYPE = "type";
    // RAP make this field public
    public static final String PARAM_QUERY_TEXT = "querytext";
    public static final String FACET_FIELD_PREFIX = "facet_";
    public static final String PARAM_FACET_AS_TEXT = "facetAsText";
    public static final String PARAM_FACET_TEXT_VALUE = "facetTextValue";

    protected static final Map<Format,Map<Result,String>> templateTable;
    // RAP
    //protected static final Map<String, String> facetPublicNameTable;

    protected enum Format {
        HTML, XML, CSV;
    }

    protected enum Result {
        PAGED, ERROR, BAD_QUERY
    }

    static{
        templateTable = setupTemplateTable();
    }

    /**
     * Overriding doGet from FreemarkerHttpController to do a page template (as
     * opposed to body template) style output for XML requests.
     *
     * This follows the pattern in AutocompleteController.java.
     */
    @Override
    public void doGet(HttpServletRequest request, HttpServletResponse response)
            throws IOException, ServletException {
        VitroRequest vreq = new VitroRequest(request);
        boolean wasXmlRequested = isRequestedFormatXml(vreq);
        boolean wasCSVRequested = isRequestedFormatCSV(vreq);
        if( !wasXmlRequested && !wasCSVRequested){
            super.doGet(vreq,response);
        }else if (wasXmlRequested){
            try {
                ResponseValues rvalues = processRequest(vreq);

                response.setCharacterEncoding("UTF-8");
                response.setContentType("text/xml;charset=UTF-8");
                response.setHeader("Content-Disposition", "attachment; filename=search.xml");
                writeTemplate(rvalues.getTemplateName(), rvalues.getMap(), request, response);
            } catch (Exception e) {
                log.error(e, e);
            }
        }else if (wasCSVRequested){
        	try {
                ResponseValues rvalues = processRequest(vreq);

                response.setCharacterEncoding("UTF-8");
                response.setContentType("text/csv;charset=UTF-8");
                response.setHeader("Content-Disposition", "attachment; filename=search.csv");
                writeTemplate(rvalues.getTemplateName(), rvalues.getMap(), request, response);
            } catch (Exception e) {
                log.error(e, e);
            }
        }
    }

    @Override
    protected ResponseValues processRequest(VitroRequest vreq) {    	    	
    	
        //There may be other non-html formats in the future
        Format format = getFormat(vreq);
        boolean wasXmlRequested = Format.XML == format;
        boolean wasCSVRequested = Format.CSV == format;
        log.debug("Requested format was " + (wasXmlRequested ? "xml" : "html"));
        boolean wasHtmlRequested = ! (wasXmlRequested || wasCSVRequested);

        try {

            //make sure an IndividualDao is available
            if( vreq.getWebappDaoFactory() == null
                    || vreq.getWebappDaoFactory().getIndividualDao() == null ){
                log.error("Could not get webappDaoFactory or IndividualDao");
                throw new Exception("Could not access model.");
            }
            IndividualDao iDao = vreq.getWebappDaoFactory().getIndividualDao();
            VClassGroupDao grpDao = vreq.getWebappDaoFactory().getVClassGroupDao();
            VClassDao vclassDao = vreq.getWebappDaoFactory().getVClassDao();

            ApplicationBean appBean = vreq.getAppBean();

            log.debug("IndividualDao is " + iDao.toString() + " Public classes in the classgroup are " + grpDao.getPublicGroupsWithVClasses().toString());
            log.debug("VClassDao is "+ vclassDao.toString() );

            int startIndex = getStartIndex(vreq);
            int hitsPerPage = getHitsPerPage( vreq );

            String queryText = vreq.getParameter(PARAM_QUERY_TEXT);
            log.debug("Query text is \""+ queryText + "\"");

// RAP: allow empty searches
            if(queryText == null) {
                queryText = "";
            }
//            String badQueryMsg = badQueryText( queryText, vreq );
//            if( badQueryMsg != null ){
//                return doFailedSearch(badQueryMsg, queryText, format, vreq);
//            }

            SearchQuery query = getQuery(queryText, hitsPerPage, startIndex, vreq);
            SearchEngine search = ApplicationUtils.instance().getSearchEngine();
            SearchResponse response = null;

            try {
                response = search.query(query);
            } catch (Exception ex) {
                String msg = makeBadSearchMessage(queryText, ex.getMessage(), vreq);
                log.error("could not run search query",ex);
                return doFailedSearch(msg, queryText, format, vreq);
            }

            if (response == null) {
                log.error("Search response was null");
                return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText, format, vreq);
            }

            SearchResultDocumentList docs = response.getResults();
            if (docs == null) {
                log.error("Document list for a search was null");
                return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText,format, vreq);
            }

            long hitCount = docs.getNumFound();
            log.debug("Number of hits = " + hitCount);
            if ( hitCount < 1 ) {
                return doNoHits(queryText, format, vreq);
            }

            List<Individual> individuals = new ArrayList<Individual>(docs.size());
            Iterator<SearchResultDocument> docIter = docs.iterator();
            while( docIter.hasNext() ){
                try {
                    SearchResultDocument doc = docIter.next();
                    String uri = doc.getStringValue(VitroSearchTermNames.URI);
                    Individual ind = iDao.getIndividualByURI(uri);
                    if(ind != null) {
                      ind.setSearchSnippet( getSnippet(doc, response) );
                      individuals.add(ind);
                    }
                } catch(Exception e) {
                    log.error("Problem getting usable individuals from search hits. ",e);
                }
            }

//          ParamMap pagingLinkParams = new ParamMap();
            ParamMap pagingLinkParams = RAPgetQueryParamMap(vreq);
            pagingLinkParams.put(PARAM_QUERY_TEXT, queryText);
            pagingLinkParams.put(PARAM_HITS_PER_PAGE, String.valueOf(hitsPerPage));

            if( wasXmlRequested ){
                pagingLinkParams.put(PARAM_XML_REQUEST,"1");
            }

            /* Compile the data for the templates */

            Map<String, Object> body = new HashMap<String, Object>();

            String classGroupParam = vreq.getParameter(PARAM_CLASSGROUP);
            log.debug("ClassGroupParam is \""+ classGroupParam + "\"");
{'fraction_non_alphanumeric': 0.09810243183492999, 'fraction_numerical': 0.0019344141488577746, 'mean_word_length': 2.894708520179372, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '685934', 'n_tokens_mistral': 6404, 'n_tokens_neox': 6135, 'n_words': 1227}