<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.handler;

import org.apache.solr.client.solrj.request.DocumentAnalysisRequest;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

import java.util.ArrayList;
import java.util.List;

/**
 * A test for {@link DocumentAnalysisRequestHandler}.
 *
 * @version $Id$
 * @since solr 1.4
 */
public class DocumentAnalysisRequestHandlerTest extends AnalysisRequestHandlerTestBase {

  private DocumentAnalysisRequestHandler handler;

  @BeforeClass
  public static void beforeClass() throws Exception {
    initCore("solrconfig.xml", "schema.xml");
  }

  @Override
  @Before
  public void setUp() throws Exception {
    super.setUp();
    handler = new DocumentAnalysisRequestHandler();
    handler.init(new NamedList());
  }

  /**
   * Tests the {@link DocumentAnalysisRequestHandler#resolveAnalysisRequest(org.apache.solr.request.SolrQueryRequest)}
   */
  @Test
  public void testResolveAnalysisRequest() throws Exception {

    String docsInput =
            "<docs>" +
                    "<doc>" +
                    "<field name=\"id\">1</field>" +
                    "<field name=\"whitetok\">The Whitetok</field>" +
                    "<field name=\"text\">The Text</field>" +
                    "</doc>" +
                    "</docs>";

    final List<ContentStream> contentStreams = new ArrayList<ContentStream>(1);
    contentStreams.add(new ContentStreamBase.StringStream(docsInput));
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.add("analysis.query", "The Query String");
    params.add("analysis.showmatch", "true");
    SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), params) {
      @Override
      public Iterable<ContentStream> getContentStreams() {
        return contentStreams;
      }
    };

    DocumentAnalysisRequest request = handler.resolveAnalysisRequest(req);

    assertNotNull(request);
    assertTrue(request.isShowMatch());
    assertNotNull(request.getQuery());
    assertEquals("The Query String", request.getQuery());
    List<SolrInputDocument> documents = request.getDocuments();
    assertNotNull(documents);
    assertEquals(1, documents.size());
    SolrInputDocument document = documents.get(0);
    SolrInputField field = document.getField("id");
    assertNotNull(field);
    assertEquals("1", field.getFirstValue());
    field = document.getField("whitetok");
    assertNotNull(field);
    assertEquals("The Whitetok", field.getFirstValue());
    field = document.getField("text");
    assertNotNull(field);
    assertEquals("The Text", field.getFirstValue());

    req.close();
  }

  /**
   * Tests the {@link DocumentAnalysisRequestHandler#handleAnalysisRequest(org.apache.solr.client.solrj.request.DocumentAnalysisRequest,
   * org.apache.solr.schema.IndexSchema)}
   */
  @Test
  public void testHandleAnalysisRequest() throws Exception {

    SolrInputDocument document = new SolrInputDocument();
    document.addField("id", 1);
    document.addField("whitetok", "Jumping Jack");
    document.addField("text", "The Fox Jumped Over The Dogs");

    DocumentAnalysisRequest request = new DocumentAnalysisRequest()
            .setQuery("JUMPING")
            .setShowMatch(true)
            .addDocument(document);

    NamedList<Object> result = handler.handleAnalysisRequest(request, h.getCore().getSchema());
    assertNotNull("result is null and it shouldn't be", result);
    NamedList<NamedList<NamedList<Object>>> documentResult = (NamedList<NamedList<NamedList<Object>>>) result.get("1");<commit_msg>SOLR-96: Add test case<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.handler;

import org.apache.solr.client.solrj.request.DocumentAnalysisRequest;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

import java.util.Collections;
import java.util.List;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.Reader;

/**
 * A test for {@link DocumentAnalysisRequestHandler}.
 *
 * @version $Id$
 * @since solr 1.4
 */
public class DocumentAnalysisRequestHandlerTest extends AnalysisRequestHandlerTestBase {

  private DocumentAnalysisRequestHandler handler;

  @BeforeClass
  public static void beforeClass() throws Exception {
    initCore("solrconfig.xml", "schema.xml");
  }

  @Override
  @Before
  public void setUp() throws Exception {
    super.setUp();
    handler = new DocumentAnalysisRequestHandler();
    handler.init(new NamedList());
  }

  /**
   * Tests the {@link DocumentAnalysisRequestHandler#resolveAnalysisRequest(org.apache.solr.request.SolrQueryRequest)}
   */
  @Test
  public void testResolveAnalysisRequest() throws Exception {

    String docsInput =
            "<docs>" +
                    "<doc>" +
                    "<field name=\"id\">1</field>" +
                    "<field name=\"whitetok\">The Whitetok</field>" +
                    "<field name=\"text\">The Text</field>" +
                    "</doc>" +
                    "</docs>";

    final ContentStream cs = new ContentStreamBase.StringStream(docsInput);
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.add("analysis.query", "The Query String");
    params.add("analysis.showmatch", "true");
    SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), params) {
      @Override
      public Iterable<ContentStream> getContentStreams() {
        return Collections.singleton(cs);
      }
    };

    DocumentAnalysisRequest request = handler.resolveAnalysisRequest(req);

    assertNotNull(request);
    assertTrue(request.isShowMatch());
    assertNotNull(request.getQuery());
    assertEquals("The Query String", request.getQuery());
    List<SolrInputDocument> documents = request.getDocuments();
    assertNotNull(documents);
    assertEquals(1, documents.size());
    SolrInputDocument document = documents.get(0);
    SolrInputField field = document.getField("id");
    assertNotNull(field);
    assertEquals("1", field.getFirstValue());
    field = document.getField("whitetok");
    assertNotNull(field);
    assertEquals("The Whitetok", field.getFirstValue());
    field = document.getField("text");
    assertNotNull(field);
    assertEquals("The Text", field.getFirstValue());

    req.close();
  }

  /** A binary-only ContentStream */
  static class ByteStream extends ContentStreamBase {
    private final byte[] bytes;
    
    public ByteStream(byte[] bytes, String contentType) {
      this.bytes = bytes; 
      this.contentType = contentType;
      name = null;
      size = Long.valueOf(bytes.length);
      sourceInfo = "rawBytes";
    }

    public InputStream getStream() throws IOException {
      return new ByteArrayInputStream(bytes);
    }

    @Override
    public Reader getReader() throws IOException {
      throw new IOException("This is a byte stream, Readers are not supported.");
    }
  }

  
  // This test should also test charset detection in UpdateRequestHandler,
  // but the DocumentAnalysisRequestHandler is simplier to use/check.
  @Test
  public void testCharsetInDocument() throws Exception {
    final byte[] xmlBytes = (
      "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>\r\n" +
      "<docs>\r\n" +
      " <doc>\r\n" +
      "  <field name=\"id\">Müller</field>\r\n" +
      " </doc>" +
      "</docs>"
    ).getBytes("ISO-8859-1");
    
    // we declare a content stream without charset:
    final ContentStream cs = new ByteStream(xmlBytes, "application/xml");
    
    ModifiableSolrParams params = new ModifiableSolrParams();
    SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), params) {
      @Override
      public Iterable<ContentStream> getContentStreams() {
        return Collections.singleton(cs);
      }
    };

    DocumentAnalysisRequest request = handler.resolveAnalysisRequest(req);
    assertNotNull(request);
    final List<SolrInputDocument> documents = request.getDocuments();
    assertNotNull(documents);
    assertEquals(1, documents.size());
    SolrInputDocument doc = documents.get(0);
    assertEquals("Müller", doc.getField("id").getValue());
  }

  // This test should also test charset detection in UpdateRequestHandler,
  // but the DocumentAnalysisRequestHandler is simplier to use/check.
  @Test
  public void testCharsetOutsideDocument() throws Exception {
    final byte[] xmlBytes = (
      "<docs>\r\n" +
      " <doc>\r\n" +
      "  <field name=\"id\">Müller</field>\r\n" +
      " </doc>" +
      "</docs>"
    ).getBytes("ISO-8859-1");
    
    // we declare a content stream without charset:
    final ContentStream cs = new ByteStream(xmlBytes, "application/xml; charset=ISO-8859-1");
    
    ModifiableSolrParams params = new ModifiableSolrParams();
    SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), params) {
      @Override
      public Iterable<ContentStream> getContentStreams() {
        return Collections.singleton(cs);
      }
    };

    DocumentAnalysisRequest request = handler.resolveAnalysisRequest(req);
    assertNotNull(request);
    final List<SolrInputDocument> documents = request.getDocuments();
    assertNotNull(documents);
    assertEquals(1, documents.size());
    SolrInputDocument doc = documents.get(0);
    assertEquals("Müller", doc.getField("id").getValue());
  }

  /**
   * Tests the {@link DocumentAnalysisRequestHandler#handleAnalysisRequest(org.apache.solr.client.solrj.request.DocumentAnalysisRequest,
   * org.apache.solr.schema.IndexSchema)}
   */
  @Test
  public void testHandleAnalysisRequest() throws Exception {

    SolrInputDocument document = new SolrInputDocument();
    document.addField("id", 1);
    document.addField("whitetok", "<NAME>");
    document.addField("text", "The Fox Jumped Over The Dogs");

    DocumentAnalysisRequest request = new DocumentAnalysisRequest()
            .setQuery("JUMPING")
            .setShowMatch(true)
            .addDocument(document);

    NamedList<Object> result = handler.handleAnalysisRequest(request, h.getCore().getSchema());
    assertNotNull("result is null and it shouldn't be", result);
    NamedList<NamedList<NamedList<Object>>> documentResult = (NamedList<NamedList<NamedList<Object>>>) result.get("1");
{'fraction_non_alphanumeric': 0.11585704096996592, 'fraction_numerical': 0.004200015849116411, 'mean_word_length': 3.7532956685499057, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 1, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5046715', 'n_tokens_mistral': 3501, 'n_tokens_neox': 3402, 'n_words': 861}