<commit_before>/**  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */
package dk.statsbiblioteket.summa.support.summon.search;

import dk.statsbiblioteket.summa.common.configuration.Configuration;
import dk.statsbiblioteket.summa.common.lucene.index.IndexUtils;
import dk.statsbiblioteket.summa.common.unittest.ExtraAsserts;
import dk.statsbiblioteket.summa.common.util.SimplePair;
import dk.statsbiblioteket.summa.search.SearchNode;
import dk.statsbiblioteket.summa.search.SearchNodeFactory;
import dk.statsbiblioteket.summa.search.api.Request;
import dk.statsbiblioteket.summa.search.api.Response;
import dk.statsbiblioteket.summa.search.api.ResponseCollection;
import dk.statsbiblioteket.summa.search.api.document.DocumentKeys;
import dk.statsbiblioteket.summa.search.api.document.DocumentResponse;
import dk.statsbiblioteket.summa.search.tools.QueryRewriter;
import dk.statsbiblioteket.summa.support.api.LuceneKeys;
import dk.statsbiblioteket.summa.support.harmonise.AdjustingSearchNode;
import dk.statsbiblioteket.summa.support.harmonise.HarmoniseTestHelper;
import dk.statsbiblioteket.summa.support.harmonise.InteractionAdjuster;
import dk.statsbiblioteket.summa.support.solr.SolrSearchNode;
import dk.statsbiblioteket.util.Strings;
import dk.statsbiblioteket.util.qa.QAInfo;
import dk.statsbiblioteket.util.xml.DOM;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;

import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.rmi.RemoteException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@QAInfo(level = QAInfo.Level.NORMAL,
        state = QAInfo.State.IN_DEVELOPMENT,
        author = "te")
public class SummonSearchNodeTest extends TestCase {
    private static Log log = LogFactory.getLog(SummonSearchNodeTest.class);

    public SummonSearchNodeTest(String name) {
        super(name);
    }

    @Override
    public void setUp() throws Exception {
        super.setUp();
    }

    @Override
    public void tearDown() throws Exception {
        super.tearDown();
    }

    public static Test suite() {
        return new TestSuite(SummonSearchNodeTest.class);
    }

    public void testMoreLikeThis() throws RemoteException {
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        long standard = getHits(summon, DocumentKeys.SEARCH_QUERY, "foo");
        assertTrue("A search for 'foo' should give hits", standard > 0);

        long mlt = getHits(summon, DocumentKeys.SEARCH_QUERY, "foo", LuceneKeys.SEARCH_MORELIKETHIS_RECORDID, "bar");
        assertEquals("A search with a MoreLikeThis ID should not give hits", 0, mlt);
    }

    public void testPageFault() throws RemoteException {
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        try {
            summon.search(new Request(
                DocumentKeys.SEARCH_QUERY, "book", DocumentKeys.SEARCH_START_INDEX, 10000), responses);
        } catch (RemoteException e) {
            log.debug("Received RemoteException as expected");
        }
        fail("Search with large page number was expected to fail. Received response:\n" + responses.toXML());
    }

    public void testRecordBaseQueryRewrite() {
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        assertNull("Queries for recordBase:summon should be reduced to null (~match all)",
                   summon.convertQuery("recordBase:summon", null));
        assertEquals("recordBase:summon should be removed with implicit AND",
                     "\"foo\"", summon.convertQuery("recordBase:summon foo", null));
        assertEquals("recordBase:summon should be removed with explicit AND",
                     "\"foo\"", summon.convertQuery("recordBase:summon AND foo", null));
        assertEquals("OR with recordBase:summon and another recordBase should match",
                     null, summon.convertQuery("recordBase:summon OR recordBase:blah", null));
        assertEquals("OR with recordBase:summon should leave the rest of the query",
                     "\"foo\"", summon.convertQuery("recordBase:summon OR foo", null));
        assertEquals("recordBase:summon AND recordBase:nonexisting should not match anything",
                     SummonSearchNode.DEFAULT_NONMATCHING_QUERY.replace(":", ":\"") + "\"",
                     summon.convertQuery("recordBase:summon AND recordBase:nonexisting", null));
    }

    public void testIDResponse() throws IOException, TransformerException {
        String QUERY = "gene and protein evolution";

        log.debug("Creating SummonSearchNode");
        SearchNode summon = SummonTestHelper.createSummonSearchNode(true);
        Request req = new Request(
            DocumentKeys.SEARCH_QUERY, QUERY,
            DocumentKeys.SEARCH_COLLECT_DOCIDS, false);
        List<String> ids = getAttributes(summon, req, "id", false);
        assertTrue("There should be at least 1 result", ids.size() >= 1);

        final Pattern EMBEDDED_ID_PATTERN = Pattern.compile("<field name=\"recordID\">(.+?)</field>", Pattern.DOTALL);
        List<String> embeddedIDs = getPattern(summon, req, EMBEDDED_ID_PATTERN, false);
        ExtraAsserts.assertEquals("The embedded IDs should match the plain IDs", ids, embeddedIDs);
        System.out.println("Received IDs: " + Strings.join(ids, ", "));

        summon.close();
    }

    public void testNegativeFacet() throws RemoteException {
        final String JSON =
            "{\"search.document.query\":\"<NAME>\",\"search.document.collectdocids\":\"true\","
            + "\"solr.filterisfacet\":\"true\",\"solrparam.s.ho\":\"true\","
            + "\"search.document.filter\":\" NOT ContentType:\\\"Newspaper Article\\\"\","
            + "\"search.document.filter.purenegative\":\"true\"}";
        SearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.addJSON(JSON);
        summon.search(request, responses);
        assertFalse("The response should not have Newspaper Article as facet. total response was\n" + responses.toXML(),
                    responses.toXML().contains("<tag name=\"Newspaper Article\""));
        summon.close();
    }

    public void testRecordBaseFacet() throws RemoteException {
        final String JSON =
            "{\"search.document.query\":\"<NAME>\",\"search.document.collectdocids\":\"true\","
            + "\"solr.filterisfacet\":\"true\",\"solrparam.s.ho\":\"true\","
            + "\"search.document.filter\":\" recordBase:summon\"}";
        SearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.addJSON(JSON);
        summon.search(request, responses);
        assertTrue("The response should contain a summon tag from faceting\n" + responses.toXML(),
                   responses.toXML().contains("<tag name=\"summon\" addedobjects=\""));
        summon.close();
    }

    public void testRecordBaseFacetWithOR() throws RemoteException {
        final String JSON =
            "{\"search.document.query\":\"<NAME>\",\"search.document.collectdocids\":\"true\","
            + "\"solr.filterisfacet\":\"true\",\"solrparam.s.ho\":\"true\","
            + "\"search.document.filter\":\" recordBase:summon OR recordBase:sb_aleph\"}";
        SearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.addJSON(JSON);
        summon.search(request, responses);
        assertTrue("The response should contain a summon tag from faceting\n" + responses.toXML(),
                   responses.toXML().contains("<tag name=\"summon\" addedobjects=\""));
        summon.close();
    }

    public void testBasicSearch() throws RemoteException {
        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
//        summon.open(""); // Fake open for setting permits
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
        //System.out.println(responses.toXML());
        assertTrue("The result should contain at least one record", responses.toXML().contains("<record score"));
        assertTrue("The result should contain at least one tag", responses.toXML().contains("<tag name"));
    }

    public void testMultiID() throws RemoteException {
        List<String> IDs = Arrays.asList(
            "FETCH-proquest_dll_11531932811",
            "FETCH-proquest_dll_6357072911",
            "FETCH-proquest_dll_15622214411"
        );
        SummonSearchNode searcher = SummonTestHelper.createSummonSearchNode(true);

        for (String id: IDs) {
            assertEquals("The number of hits for ID '" + id + "' should match", 1, getAttributes(searcher, new Request(
                DocumentKeys.SEARCH_QUERY, "ID:\"" + id + "\"",
                SummonSearchNode.SEARCH_PASSTHROUGH_QUERY, true
            ), "id", false).size());
        }

        String IDS_QUERY = "(ID:\"" + Strings.join(IDs, "\" OR ID:\"") + "\")";

        Request req = new Request(
            DocumentKeys.SEARCH_QUERY, IDS_QUERY,
            SummonSearchNode.SEARCH_PASSTHROUGH_QUERY, true
        );
        List<String> returnedIDs = getAttributes(searcher, req, "id", false);
        if (IDs.size() != returnedIDs.size()) {
            ResponseCollection responses = new ResponseCollection();
            searcher.search(req, responses);
            System.out.println("Returned IDs: " + Strings.join(returnedIDs, ", "));
//            System.out.println(responses.toXML());
        }

        assertEquals("There should be a result for each id from search '" + IDS_QUERY + "'",
                     IDs.size(), returnedIDs.size());
    }

    public void testShortFormat() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(SummonResponseBuilder.CONF_SHORT_DATE, true);

        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = new SummonSearchNode(conf);
//        summon.open(""); // Fake open for setting permits
        final Pattern DATEPATTERN = Pattern.compile("<dc:date>(.+?)</dc:date>", Pattern.DOTALL);
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_RESULT_FIELDS, "shortformat");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        List<String> dates= getPattern(summon, request, DATEPATTERN, false);
        assertTrue("There should be at least 1 extracted date",
                   dates.size() > 0);
        for (String date: dates) {
            assertTrue("the returned dates should be of length 4 or less, got '" + date + "'", date.length() <= 4);
        }
//        System.out.println("Got dates:\n" + Strings.join(dates, ", "));
    }

    public void testIDSearch() throws IOException, TransformerException {
        String ID = "summon_FETCH-gale_primary_2105957371";

        log.debug("Creating SummonSearchNode");
        SearchNode summon = SummonTestHelper.createSummonSearchNode(true);
        Request req = new Request(
            DocumentKeys.SEARCH_QUERY, "recordID:\"" + ID + "\"",
            DocumentKeys.SEARCH_MAX_RECORDS, 1,
            DocumentKeys.SEARCH_COLLECT_DOCIDS, false);
        List<String> ids = getAttributes(summon, req, "id", false);
        assertTrue("There should be at least 1 result", ids.size() >= 1);
    }

    public void testGetField() throws IOException, TransformerException {
        String ID = "summon_FETCH-gale_primary_2105957371";

        log.debug("Creating SummonSearchNode");
        SearchNode summon = SummonTestHelper.createSummonSearchNode(true);
        String fieldName = "shortformat";
        String field = getField(summon, ID, fieldName);
        assertTrue("The field '" + fieldName + "' from ID '" + ID + "' should have content",
                   field != null && !"".equals(field));
//        System.out.println("'" + field + "'");
    }

    /* This is equivalent to SearchWS#getField */
    private String getField(SearchNode searcher, String id, String fieldName) throws IOException, TransformerException {
        String retXML;

        Request req = new Request();
        req.put(DocumentKeys.SEARCH_QUERY, "ID:\"" + id + "\"");
        req.put(DocumentKeys.SEARCH_MAX_RECORDS, 1);
        req.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, false);

        ResponseCollection res = new ResponseCollection();
        searcher.search(req, res);
//        System.out.println(res.toXML());
        Document dom = DOM.stringToDOM(res.toXML());
        Node subDom = DOM.selectNode(
            dom, "/responsecollection/response/documentresult/record/field[@name='" + fieldName + "']");
        retXML = DOM.domToString(subDom);
        return retXML;
    }

    public void testColonSearch() throws RemoteException {
        final String OK = "FETCH-proquest_dll_14482952011";
        final String PROBLEM = "FETCH-doaj_primary_oai:doaj-articles:932b6445ce452a2b2a544189863c472e1";
        performSearch("ID:\"" + OK + "\"");
        performSearch("ID:\"" + PROBLEM + "\"");
    }

    private void performSearch(String query) throws RemoteException {
        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, query);
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
//        System.out.println(responses.toXML());
        assertTrue("The result should contain at least one record for query '" + query + "'",
                   responses.toXML().contains("<record score"));
    }

    public void testFacetOrder() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
            //SummonSearchNode.CONF_SOLR_FACETS, ""

        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = new SummonSearchNode(conf);
//        summon.open(""); // Fake open for setting permits
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
        List<String> facets = getFacetNames(responses);
        List<String> expected = new ArrayList<String>(Arrays.asList(SummonSearchNode.DEFAULT_SUMMON_FACETS.split(" ?, ?")));
        expected.add("recordBase"); // We always add this when we're doing faceting
        for (int i = expected.size()-1 ; i >= 0 ; i--) {
            if (!facets.contains(expected.get(i))) {
                expected.remove(i);
            }
        }
        assertEquals("The order of the facets should be correct",
                     Strings.join(expected, ", "), Strings.join(facets, ", "));

//        System.out.println(responses.toXML());
//        System.out.println(Strings.join(facets, ", "));
    }

    public void testSpecificFacets() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        conf.set(SummonSearchNode.CONF_SOLR_FACETS, "SubjectTerms");

        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = new SummonSearchNode(conf);
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
        List<String> facets = getFacetNames(responses);
        assertEquals("The number of facets should be correct", 2, facets.size()); // 2 because of recordBase
        assertEquals("The returned facet should be correct",
                     "SubjectTerms, recordBase", Strings.join(facets, ", "));
    }

    public void testNegativeFacets() throws RemoteException {
        final String QUERY = "foo fighters NOT limits NOT (boo OR bam)";
        final String FACET = "SubjectTerms:\"united states\"";
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        assertHits("There should be at least one hit for positive faceting",
                   summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER, FACET);
        assertHits("There should be at least one hit for parenthesized positive faceting", summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER, "(" + FACET + ")");
        assertHits("There should be at least one hit for filter with pure negative faceting", summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER_PURE_NEGATIVE, "true",
                   DocumentKeys.SEARCH_FILTER, "NOT " + FACET);
        summon.close();
    }

    public void testFilterFacets() throws RemoteException {
        final String QUERY = "foo fighters";
        final String FACET = "SubjectTerms:\"united states\"";
        final String FACET_NEG = "-SubjectTerms:\"united states\"";
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        assertHits("There should be at least one hit for standard positive faceting",
                   summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER, FACET);
        assertHits("There should be at least one hit for facet filter positive faceting",
                   summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,<commit_msg>Testing: Added alpha-sort test for Summon facets<commit_after>/**  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */
package dk.statsbiblioteket.summa.support.summon.search;

import dk.statsbiblioteket.summa.common.configuration.Configuration;
import dk.statsbiblioteket.summa.common.lucene.index.IndexUtils;
import dk.statsbiblioteket.summa.common.unittest.ExtraAsserts;
import dk.statsbiblioteket.summa.common.util.SimplePair;
import dk.statsbiblioteket.summa.facetbrowser.api.FacetResultExternal;
import dk.statsbiblioteket.summa.facetbrowser.api.FacetResultImpl;
import dk.statsbiblioteket.summa.search.SearchNode;
import dk.statsbiblioteket.summa.search.SearchNodeFactory;
import dk.statsbiblioteket.summa.search.api.Request;
import dk.statsbiblioteket.summa.search.api.Response;
import dk.statsbiblioteket.summa.search.api.ResponseCollection;
import dk.statsbiblioteket.summa.search.api.document.DocumentKeys;
import dk.statsbiblioteket.summa.search.api.document.DocumentResponse;
import dk.statsbiblioteket.summa.search.tools.QueryRewriter;
import dk.statsbiblioteket.summa.support.api.LuceneKeys;
import dk.statsbiblioteket.summa.support.harmonise.AdjustingSearchNode;
import dk.statsbiblioteket.summa.support.harmonise.HarmoniseTestHelper;
import dk.statsbiblioteket.summa.support.harmonise.InteractionAdjuster;
import dk.statsbiblioteket.summa.support.solr.SolrSearchNode;
import dk.statsbiblioteket.util.Strings;
import dk.statsbiblioteket.util.qa.QAInfo;
import dk.statsbiblioteket.util.xml.DOM;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;

import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.rmi.RemoteException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@QAInfo(level = QAInfo.Level.NORMAL,
        state = QAInfo.State.IN_DEVELOPMENT,
        author = "te")
public class SummonSearchNodeTest extends TestCase {
    private static Log log = LogFactory.getLog(SummonSearchNodeTest.class);

    public SummonSearchNodeTest(String name) {
        super(name);
    }

    @Override
    public void setUp() throws Exception {
        super.setUp();
    }

    @Override
    public void tearDown() throws Exception {
        super.tearDown();
    }

    public static Test suite() {
        return new TestSuite(SummonSearchNodeTest.class);
    }

    public void testMoreLikeThis() throws RemoteException {
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        long standard = getHits(summon, DocumentKeys.SEARCH_QUERY, "foo");
        assertTrue("A search for 'foo' should give hits", standard > 0);

        long mlt = getHits(summon, DocumentKeys.SEARCH_QUERY, "foo", LuceneKeys.SEARCH_MORELIKETHIS_RECORDID, "bar");
        assertEquals("A search with a MoreLikeThis ID should not give hits", 0, mlt);
    }

    public void testPageFault() throws RemoteException {
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        try {
            summon.search(new Request(
                DocumentKeys.SEARCH_QUERY, "book", DocumentKeys.SEARCH_START_INDEX, 10000), responses);
        } catch (RemoteException e) {
            log.debug("Received RemoteException as expected");
        }
        fail("Search with large page number was expected to fail. Received response:\n" + responses.toXML());
    }

    public void testRecordBaseQueryRewrite() {
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        assertNull("Queries for recordBase:summon should be reduced to null (~match all)",
                   summon.convertQuery("recordBase:summon", null));
        assertEquals("recordBase:summon should be removed with implicit AND",
                     "\"foo\"", summon.convertQuery("recordBase:summon foo", null));
        assertEquals("recordBase:summon should be removed with explicit AND",
                     "\"foo\"", summon.convertQuery("recordBase:summon AND foo", null));
        assertEquals("OR with recordBase:summon and another recordBase should match",
                     null, summon.convertQuery("recordBase:summon OR recordBase:blah", null));
        assertEquals("OR with recordBase:summon should leave the rest of the query",
                     "\"foo\"", summon.convertQuery("recordBase:summon OR foo", null));
        assertEquals("recordBase:summon AND recordBase:nonexisting should not match anything",
                     SummonSearchNode.DEFAULT_NONMATCHING_QUERY.replace(":", ":\"") + "\"",
                     summon.convertQuery("recordBase:summon AND recordBase:nonexisting", null));
    }

    public void testIDResponse() throws IOException, TransformerException {
        String QUERY = "gene and protein evolution";

        log.debug("Creating SummonSearchNode");
        SearchNode summon = SummonTestHelper.createSummonSearchNode(true);
        Request req = new Request(
            DocumentKeys.SEARCH_QUERY, QUERY,
            DocumentKeys.SEARCH_COLLECT_DOCIDS, false);
        List<String> ids = getAttributes(summon, req, "id", false);
        assertTrue("There should be at least 1 result", ids.size() >= 1);

        final Pattern EMBEDDED_ID_PATTERN = Pattern.compile("<field name=\"recordID\">(.+?)</field>", Pattern.DOTALL);
        List<String> embeddedIDs = getPattern(summon, req, EMBEDDED_ID_PATTERN, false);
        ExtraAsserts.assertEquals("The embedded IDs should match the plain IDs", ids, embeddedIDs);
        System.out.println("Received IDs: " + Strings.join(ids, ", "));

        summon.close();
    }

    public void testNegativeFacet() throws RemoteException {
        final String JSON =
            "{\"search.document.query\":\"dark<NAME>\",\"search.document.collectdocids\":\"true\","
            + "\"solr.filterisfacet\":\"true\",\"solrparam.s.ho\":\"true\","
            + "\"search.document.filter\":\" NOT ContentType:\\\"Newspaper Article\\\"\","
            + "\"search.document.filter.purenegative\":\"true\"}";
        SearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.addJSON(JSON);
        summon.search(request, responses);
        assertFalse("The response should not have Newspaper Article as facet. total response was\n" + responses.toXML(),
                    responses.toXML().contains("<tag name=\"Newspaper Article\""));
        summon.close();
    }

    public void testRecordBaseFacet() throws RemoteException {
        final String JSON =
            "{\"search.document.query\":\"darkmans barker\",\"search.document.collectdocids\":\"true\","
            + "\"solr.filterisfacet\":\"true\",\"solrparam.s.ho\":\"true\","
            + "\"search.document.filter\":\" recordBase:summon\"}";
        SearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.addJSON(JSON);
        summon.search(request, responses);
        assertTrue("The response should contain a summon tag from faceting\n" + responses.toXML(),
                   responses.toXML().contains("<tag name=\"summon\" addedobjects=\""));
        summon.close();
    }

    public void testRecordBaseFacetWithOR() throws RemoteException {
        final String JSON =
            "{\"search.document.query\":\"darkmans barker\",\"search.document.collectdocids\":\"true\","
            + "\"solr.filterisfacet\":\"true\",\"solrparam.s.ho\":\"true\","
            + "\"search.document.filter\":\" recordBase:summon OR recordBase:sb_aleph\"}";
        SearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.addJSON(JSON);
        summon.search(request, responses);
        assertTrue("The response should contain a summon tag from faceting\n" + responses.toXML(),
                   responses.toXML().contains("<tag name=\"summon\" addedobjects=\""));
        summon.close();
    }

    public void testBasicSearch() throws RemoteException {
        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
//        summon.open(""); // Fake open for setting permits
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
        //System.out.println(responses.toXML());
        assertTrue("The result should contain at least one record", responses.toXML().contains("<record score"));
        assertTrue("The result should contain at least one tag", responses.toXML().contains("<tag name"));
    }

    public void testMultiID() throws RemoteException {
        List<String> IDs = Arrays.asList(
            "FETCH-proquest_dll_11531932811",
            "FETCH-proquest_dll_6357072911",
            "FETCH-proquest_dll_15622214411"
        );
        SummonSearchNode searcher = SummonTestHelper.createSummonSearchNode(true);

        for (String id: IDs) {
            assertEquals("The number of hits for ID '" + id + "' should match", 1, getAttributes(searcher, new Request(
                DocumentKeys.SEARCH_QUERY, "ID:\"" + id + "\"",
                SummonSearchNode.SEARCH_PASSTHROUGH_QUERY, true
            ), "id", false).size());
        }

        String IDS_QUERY = "(ID:\"" + Strings.join(IDs, "\" OR ID:\"") + "\")";

        Request req = new Request(
            DocumentKeys.SEARCH_QUERY, IDS_QUERY,
            SummonSearchNode.SEARCH_PASSTHROUGH_QUERY, true
        );
        List<String> returnedIDs = getAttributes(searcher, req, "id", false);
        if (IDs.size() != returnedIDs.size()) {
            ResponseCollection responses = new ResponseCollection();
            searcher.search(req, responses);
            System.out.println("Returned IDs: " + Strings.join(returnedIDs, ", "));
//            System.out.println(responses.toXML());
        }

        assertEquals("There should be a result for each id from search '" + IDS_QUERY + "'",
                     IDs.size(), returnedIDs.size());
    }

    public void testShortFormat() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(SummonResponseBuilder.CONF_SHORT_DATE, true);

        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = new SummonSearchNode(conf);
//        summon.open(""); // Fake open for setting permits
        final Pattern DATEPATTERN = Pattern.compile("<dc:date>(.+?)</dc:date>", Pattern.DOTALL);
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_RESULT_FIELDS, "shortformat");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        List<String> dates= getPattern(summon, request, DATEPATTERN, false);
        assertTrue("There should be at least 1 extracted date",
                   dates.size() > 0);
        for (String date: dates) {
            assertTrue("the returned dates should be of length 4 or less, got '" + date + "'", date.length() <= 4);
        }
//        System.out.println("Got dates:\n" + Strings.join(dates, ", "));
    }

    public void testIDSearch() throws IOException, TransformerException {
        String ID = "summon_FETCH-gale_primary_2105957371";

        log.debug("Creating SummonSearchNode");
        SearchNode summon = SummonTestHelper.createSummonSearchNode(true);
        Request req = new Request(
            DocumentKeys.SEARCH_QUERY, "recordID:\"" + ID + "\"",
            DocumentKeys.SEARCH_MAX_RECORDS, 1,
            DocumentKeys.SEARCH_COLLECT_DOCIDS, false);
        List<String> ids = getAttributes(summon, req, "id", false);
        assertTrue("There should be at least 1 result", ids.size() >= 1);
    }

    public void testGetField() throws IOException, TransformerException {
        String ID = "summon_FETCH-gale_primary_2105957371";

        log.debug("Creating SummonSearchNode");
        SearchNode summon = SummonTestHelper.createSummonSearchNode(true);
        String fieldName = "shortformat";
        String field = getField(summon, ID, fieldName);
        assertTrue("The field '" + fieldName + "' from ID '" + ID + "' should have content",
                   field != null && !"".equals(field));
//        System.out.println("'" + field + "'");
    }

    /* This is equivalent to SearchWS#getField */
    private String getField(SearchNode searcher, String id, String fieldName) throws IOException, TransformerException {
        String retXML;

        Request req = new Request();
        req.put(DocumentKeys.SEARCH_QUERY, "ID:\"" + id + "\"");
        req.put(DocumentKeys.SEARCH_MAX_RECORDS, 1);
        req.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, false);

        ResponseCollection res = new ResponseCollection();
        searcher.search(req, res);
//        System.out.println(res.toXML());
        Document dom = DOM.stringToDOM(res.toXML());
        Node subDom = DOM.selectNode(
            dom, "/responsecollection/response/documentresult/record/field[@name='" + fieldName + "']");
        retXML = DOM.domToString(subDom);
        return retXML;
    }

    public void testColonSearch() throws RemoteException {
        final String OK = "FETCH-proquest_dll_14482952011";
        final String PROBLEM = "FETCH-doaj_primary_oai:doaj-articles:932b6445ce452a2b2a544189863c472e1";
        performSearch("ID:\"" + OK + "\"");
        performSearch("ID:\"" + PROBLEM + "\"");
    }

    private void performSearch(String query) throws RemoteException {
        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, query);
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
//        System.out.println(responses.toXML());
        assertTrue("The result should contain at least one record for query '" + query + "'",
                   responses.toXML().contains("<record score"));
    }

    public void testFacetOrder() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
            //SummonSearchNode.CONF_SOLR_FACETS, ""

        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = new SummonSearchNode(conf);
//        summon.open(""); // Fake open for setting permits
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
        List<String> facets = getFacetNames(responses);
        List<String> expected = new ArrayList<String>(Arrays.asList(SummonSearchNode.DEFAULT_SUMMON_FACETS.split(" ?, ?")));
        expected.add("recordBase"); // We always add this when we're doing faceting
        for (int i = expected.size()-1 ; i >= 0 ; i--) {
            if (!facets.contains(expected.get(i))) {
                expected.remove(i);
            }
        }
        assertEquals("The order of the facets should be correct",
                     Strings.join(expected, ", "), Strings.join(facets, ", "));

//        System.out.println(responses.toXML());
//        System.out.println(Strings.join(facets, ", "));
    }

    public void testSpecificFacets() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        conf.set(SummonSearchNode.CONF_SOLR_FACETS, "SubjectTerms");

        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = new SummonSearchNode(conf);
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
        List<String> facets = getFacetNames(responses);
        assertEquals("The number of facets should be correct", 2, facets.size()); // 2 because of recordBase
        assertEquals("The returned facet should be correct",
                     "SubjectTerms, recordBase", Strings.join(facets, ", "));
    }

    public void testFacetSortingCount() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        conf.set(SummonSearchNode.CONF_SOLR_FACETS, "SubjectTerms");
        assertFacetOrder(conf, false);
    }

    // Summon does not support index ordering of facets so we must cheat by over-requesting and post-processing
    public void testFacetSortingAlpha() throws RemoteException {
        Configuration conf = SummonTestHelper.getDefaultSummonConfiguration();
        conf.set(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        conf.set(SummonSearchNode.CONF_SOLR_FACETS, "SubjectTerms(ALPHA)");
        assertFacetOrder(conf, true);
    }

    private void assertFacetOrder(Configuration summonConf, boolean alpha) throws RemoteException {
        log.debug("Creating SummonSearchNode");
        SummonSearchNode summon = new SummonSearchNode(summonConf);
        ResponseCollection responses = new ResponseCollection();
        Request request = new Request();
        request.put(DocumentKeys.SEARCH_QUERY, "foo");
        request.put(DocumentKeys.SEARCH_COLLECT_DOCIDS, true);
        log.debug("Searching");
        summon.search(request, responses);
        log.debug("Finished searching");
        List<String> tags = getTags(responses, "SubjectTerms");
        List<String> tagsAlpha = new ArrayList<String>(tags);
        Collections.sort(tagsAlpha);
        if (alpha) {
            ExtraAsserts.assertEquals(
                "The order should be alphanumeric\nExp: "
                + Strings.join(tagsAlpha, " ") + "\nAct: " + Strings.join(tags, " "),
                tagsAlpha, tags);
        } else {
            boolean misMatch = false;
            for (int i = 0 ; i < tags.size() ; i++) {
                if (!tags.get(i).equals(tagsAlpha.get(i))) {
                    misMatch = true;
                    break;
                }
            }
            if (!misMatch) {
                fail("The order should not be alphanumeric but it was");
            }
        }
        log.debug("Received facets with alpha=" + alpha + ": " + Strings.join(tags, ", "));
    }

    private List<String> getTags(ResponseCollection responses, String facet) {
        List<String> result = new ArrayList<String>();
        for (Response response: responses) {
            if (response instanceof FacetResultExternal) {
                FacetResultExternal facetResult = (FacetResultExternal)response;
                List<FacetResultImpl.Tag<String>> tags = facetResult.getMap().get(facet);
                for (FacetResultImpl.Tag<String> tag: tags) {
                    result.add(tag.getKey() + "(" + tag.getCount() + ")");
                }
                return result;
            }
        }
        fail("Unable to locate a FacetResponse in the ResponseCollection");
        return null;
    }

    public void testNegativeFacets() throws RemoteException {
        final String QUERY = "foo fighters NOT limits NOT (boo OR bam)";
        final String FACET = "SubjectTerms:\"united states\"";
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        assertHits("There should be at least one hit for positive faceting",
                   summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER, FACET);
        assertHits("There should be at least one hit for parenthesized positive faceting", summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER, "(" + FACET + ")");
        assertHits("There should be at least one hit for filter with pure negative faceting", summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER_PURE_NEGATIVE, "true",
                   DocumentKeys.SEARCH_FILTER, "NOT " + FACET);
        summon.close();
    }

    public void testFilterFacets() throws RemoteException {
        final String QUERY = "foo fighters";
        final String FACET = "SubjectTerms:\"united states\"";
        final String FACET_NEG = "-SubjectTerms:\"united states\"";
        SummonSearchNode summon = SummonTestHelper.createSummonSearchNode();
        assertHits("There should be at least one hit for standard positive faceting",
                   summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
                   DocumentKeys.SEARCH_FILTER, FACET);
        assertHits("There should be at least one hit for facet filter positive faceting",
                   summon,
                   DocumentKeys.SEARCH_QUERY, QUERY,
{'fraction_non_alphanumeric': 0.11574449801880937, 'fraction_numerical': 0.005561655607008164, 'mean_word_length': 2.9722195885085805, 'pattern_counts': {'":': 37, '<': 59, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7492366', 'n_tokens_mistral': 11846, 'n_tokens_neox': 11451, 'n_words': 2599}