<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.spatial.test;


import junit.framework.Assert;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.spatial.base.context.SpatialContext;
import org.apache.lucene.spatial.base.io.sample.SampleData;
import org.apache.lucene.spatial.base.io.sample.SampleDataReader;
import org.apache.lucene.spatial.base.query.SpatialArgsParser;
import org.apache.lucene.spatial.base.shape.Shape;
import org.apache.lucene.spatial.strategy.SpatialFieldInfo;
import org.apache.lucene.spatial.strategy.SpatialStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;


public abstract class StrategyTestCase<T extends SpatialFieldInfo> extends SpatialTestCase {

  public static final String DATA_STATES_POLY = "states-poly.txt";
  public static final String DATA_STATES_BBOX = "states-bbox.txt";
  public static final String DATA_COUNTRIES_POLY = "countries-poly.txt";
  public static final String DATA_COUNTRIES_BBOX = "countries-bbox.txt";
  public static final String DATA_WORLD_CITIES_POINTS = "world-cities-points.txt";

  public static final String QTEST_States_IsWithin_BBox   = "states-IsWithin-BBox.txt";
  public static final String QTEST_States_Intersects_BBox = "states-Intersects-BBox.txt";

  public static final String QTEST_Cities_IsWithin_BBox = "cities-IsWithin-BBox.txt";

  protected final Logger log = LoggerFactory.getLogger(getClass());

  protected final SpatialArgsParser argsParser = new SpatialArgsParser();

  protected SpatialStrategy<T> strategy;
  protected SpatialContext ctx;
  protected T fieldInfo;

  protected void executeQueries(SpatialMatchConcern concern, String... testQueryFile) throws IOException {
    log.info("testing queried for strategy "+strategy);
    for( String path : testQueryFile ) {
      Iterator<SpatialTestQuery> testQueryIterator = getTestQueries(path, ctx);
      runTestQueries(testQueryIterator, concern);
    }
  }

  protected void getAddAndVerifyIndexedDocuments(String testDataFile) throws IOException {
    List<Document> testDocuments = getDocuments(testDataFile);
    addDocuments(testDocuments);
    verifyDocumentsIndexed(testDocuments.size());
  }

  protected List<Document> getDocuments(String testDataFile) throws IOException {
    Iterator<SampleData> sampleData = getSampleData(testDataFile);
    List<Document> documents = new ArrayList<Document>();
    while (sampleData.hasNext()) {
      SampleData data = sampleData.next();
      Document document = new Document();
      document.add(new Field("id", data.id, Store.YES, Index.ANALYZED));
      document.add(new Field("name", data.name, Store.YES, Index.ANALYZED));
      Shape shape = ctx.readShape(data.shape);
      for (Fieldable f : strategy.createFields(fieldInfo, shape, true, true)) {
        if( f != null ) { // null if incompatibleGeometry && ignore
          document.add(f);
        }
      }
      documents.add(document);
    }
    return documents;
  }

  protected Iterator<SampleData> getSampleData(String testDataFile) throws IOException {
    return new SampleDataReader(
        getClass().getClassLoader().getResourceAsStream("data/"+testDataFile) );
  }<commit_msg>Make option to store the shape a configurable test option.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.spatial.test;


import junit.framework.Assert;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.spatial.base.context.SpatialContext;
import org.apache.lucene.spatial.base.io.sample.SampleData;
import org.apache.lucene.spatial.base.io.sample.SampleDataReader;
import org.apache.lucene.spatial.base.query.SpatialArgsParser;
import org.apache.lucene.spatial.base.shape.Shape;
import org.apache.lucene.spatial.strategy.SpatialFieldInfo;
import org.apache.lucene.spatial.strategy.SpatialStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;


public abstract class StrategyTestCase<T extends SpatialFieldInfo> extends SpatialTestCase {

  public static final String DATA_STATES_POLY = "states-poly.txt";
  public static final String DATA_STATES_BBOX = "states-bbox.txt";
  public static final String DATA_COUNTRIES_POLY = "countries-poly.txt";
  public static final String DATA_COUNTRIES_BBOX = "countries-bbox.txt";
  public static final String DATA_WORLD_CITIES_POINTS = "world-cities-points.txt";

  public static final String QTEST_States_IsWithin_BBox   = "states-IsWithin-BBox.txt";
  public static final String QTEST_States_Intersects_BBox = "states-Intersects-BBox.txt";

  public static final String QTEST_Cities_IsWithin_BBox = "cities-IsWithin-BBox.txt";

  protected final Logger log = LoggerFactory.getLogger(getClass());

  protected final SpatialArgsParser argsParser = new SpatialArgsParser();

  protected SpatialStrategy<T> strategy;
  protected SpatialContext ctx;
  protected T fieldInfo;
  protected boolean storeShape = true;

  protected void executeQueries(SpatialMatchConcern concern, String... testQueryFile) throws IOException {
    log.info("testing queried for strategy "+strategy);
    for( String path : testQueryFile ) {
      Iterator<SpatialTestQuery> testQueryIterator = getTestQueries(path, ctx);
      runTestQueries(testQueryIterator, concern);
    }
  }

  protected void getAddAndVerifyIndexedDocuments(String testDataFile) throws IOException {
    List<Document> testDocuments = getDocuments(testDataFile);
    addDocuments(testDocuments);
    verifyDocumentsIndexed(testDocuments.size());
  }

  protected List<Document> getDocuments(String testDataFile) throws IOException {
    Iterator<SampleData> sampleData = getSampleData(testDataFile);
    List<Document> documents = new ArrayList<Document>();
    while (sampleData.hasNext()) {
      SampleData data = sampleData.next();
      Document document = new Document();
      document.add(new Field("id", data.id, Store.YES, Index.ANALYZED));
      document.add(new Field("name", data.name, Store.YES, Index.ANALYZED));
      Shape shape = ctx.readShape(data.shape);
      for (Fieldable f : strategy.createFields(fieldInfo, shape, true, storeShape)) {
        if( f != null ) { // null if incompatibleGeometry && ignore
          document.add(f);
        }
      }
      documents.add(document);
    }
    return documents;
  }

  protected Iterator<SampleData> getSampleData(String testDataFile) throws IOException {
    return new SampleDataReader(
        getClass().getClassLoader().getResourceAsStream("data/"+testDataFile) );
  }
{'fraction_non_alphanumeric': 0.09213298993210021, 'fraction_numerical': 0.0014048232264106766, 'mean_word_length': 4.665119363395226, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2896082', 'n_tokens_mistral': 2515, 'n_tokens_neox': 2389, 'n_words': 661}