<commit_before>package com.senseidb.search.query.filters;

import java.io.IOException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermRangeFilter;
import org.json.JSONObject;

import com.browseengine.bobo.api.BoboIndexReader;
import com.browseengine.bobo.facets.FacetHandler;
import com.browseengine.bobo.facets.data.FacetDataCache;
import com.browseengine.bobo.facets.filter.FacetRangeFilter;
import com.browseengine.bobo.facets.filter.RandomAccessFilter;
import com.browseengine.bobo.query.MatchAllDocIdSetIterator;
import com.senseidb.indexing.DefaultSenseiInterpreter;
import com.senseidb.indexing.MetaType;
import com.senseidb.indexing.activity.facet.ActivityRangeFacetHandler;

public class RangeFilterConstructor extends FilterConstructor
{
  public static final String FILTER_TYPE = "range";

  @Override
  protected Filter doConstructFilter(Object obj) throws Exception
  {
    final JSONObject json = (JSONObject)obj;

    Iterator<String> iter = json.keys();
    if (!iter.hasNext())
      return null;

    final String field = iter.next();

    JSONObject jsonObj = json.getJSONObject(field);

    final String  from, to;
    final boolean include_lower, include_upper;
    final boolean noOptimize = jsonObj.optBoolean(NOOPTIMIZE_PARAM, false);
    final String type;
    final String dateFormat;
    
    String gt  = jsonObj.optString(GT_PARAM, null);
    String gte = jsonObj.optString(GTE_PARAM, null);
    String lt  = jsonObj.optString(LT_PARAM, null);
    String lte = jsonObj.optString(LTE_PARAM, null);
    
    type = jsonObj.optString(RANGE_FIELD_TYPE, null);
    dateFormat = jsonObj.optString(RANGE_DATE_FORMAT, null);
    
    if (gt != null && gt.length() != 0)
    {
      from          = gt;
      include_lower = false;
    }
    else if (gte != null && gte.length() != 0)
    {
      from          = gte;
      include_lower = true;
    }
    else
    {
      from          = jsonObj.optString(FROM_PARAM, null);
      include_lower = jsonObj.optBoolean(INCLUDE_LOWER_PARAM, true);
    }

    if (lt != null && lt.length() != 0)
    {
      to = lt;
      include_upper = false;
    }
    else if (lte != null && lte.length() != 0)
    {
      to = lte;
      include_upper = true;
    }
    else
    {
      to            = jsonObj.optString(TO_PARAM, null);
      include_upper = jsonObj.optBoolean(INCLUDE_UPPER_PARAM, true);
    }

    return new Filter()
    {
      @Override
      public DocIdSet getDocIdSet(final IndexReader reader) throws IOException
      {
        String fromPadded = from, toPadded = to;
        if (!noOptimize)
        {
          if (reader instanceof BoboIndexReader)
          {
            BoboIndexReader boboReader = (BoboIndexReader)reader;
            FacetHandler facetHandler = boboReader.getFacetHandler(field);
            if (facetHandler != null)
            {
              StringBuilder sb = new StringBuilder();
              if (include_lower && from != null && from.length() != 0)
                sb.append("[");
              else
                sb.append("(");

              if (from == null || from.length() == 0)
                sb.append("*");
              else
                sb.append(from);
              sb.append(" TO ");
              if (to == null || to.length() == 0)
                sb.append("*");
              else
                sb.append(to);

              if (include_upper && to != null && to.length() != 0)
                sb.append("]");
              else
                sb.append(")");
              RandomAccessFilter filter = null;;
              Object facetData = boboReader.getFacetData(field);
              //If it's Sensei's standard facet handler
              if (facetData instanceof FacetDataCache) {
              	filter = new FacetRangeFilter(facetHandler, sb.toString());
              } else {
                filter = facetHandler.buildRandomAccessFilter(sb.toString(), null);
              }
              return filter.getDocIdSet(reader);
            }
          }
        }
        
        if(type == null)
          throw new IllegalArgumentException("need to specify the type of field in filter json: " + json);
        
        if ("int".equals(type)) {
          MetaType metaType = DefaultSenseiInterpreter.CLASS_METATYPE_MAP.get(int.class);
          String formatString = DefaultSenseiInterpreter.DEFAULT_FORMAT_STRING_MAP.get(metaType);
          DecimalFormat formatter = new DecimalFormat(formatString, new DecimalFormatSymbols(Locale.US));
          fromPadded = formatter.format(Integer.parseInt(from));
          toPadded = formatter.format(Integer.parseInt(to));
        } 
        else if ("short".equals(type)) {
          MetaType metaType = DefaultSenseiInterpreter.CLASS_METATYPE_MAP.get(short.class);<commit_msg>Added some better exception handling for range queries<commit_after>package com.senseidb.search.query.filters;

import java.io.IOException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermRangeFilter;
import org.json.JSONObject;

import com.browseengine.bobo.api.BoboIndexReader;
import com.browseengine.bobo.facets.FacetHandler;
import com.browseengine.bobo.facets.data.FacetDataCache;
import com.browseengine.bobo.facets.filter.EmptyFilter;
import com.browseengine.bobo.facets.filter.FacetRangeFilter;
import com.browseengine.bobo.facets.filter.RandomAccessFilter;
import com.browseengine.bobo.query.MatchAllDocIdSetIterator;
import com.senseidb.indexing.DefaultSenseiInterpreter;
import com.senseidb.indexing.MetaType;
import com.senseidb.indexing.activity.facet.ActivityRangeFacetHandler;

public class RangeFilterConstructor extends FilterConstructor
{
  public static final String FILTER_TYPE = "range";

  @Override
  protected Filter doConstructFilter(Object obj) throws Exception
  {
    final JSONObject json = (JSONObject)obj;

    Iterator<String> iter = json.keys();
    if (!iter.hasNext())
      return null;

    final String field = iter.next();

    JSONObject jsonObj = json.getJSONObject(field);

    final String  from, to;
    final boolean include_lower, include_upper;
    final boolean noOptimize = jsonObj.optBoolean(NOOPTIMIZE_PARAM, false);
    final String type;
    final String dateFormat;
    
    String gt  = jsonObj.optString(GT_PARAM, null);
    String gte = jsonObj.optString(GTE_PARAM, null);
    String lt  = jsonObj.optString(LT_PARAM, null);
    String lte = jsonObj.optString(LTE_PARAM, null);
    
    type = jsonObj.optString(RANGE_FIELD_TYPE, null);
    dateFormat = jsonObj.optString(RANGE_DATE_FORMAT, null);
    
    if (gt != null && gt.length() != 0)
    {
      from          = gt;
      include_lower = false;
    }
    else if (gte != null && gte.length() != 0)
    {
      from          = gte;
      include_lower = true;
    }
    else
    {
      from          = jsonObj.optString(FROM_PARAM, null);
      include_lower = jsonObj.optBoolean(INCLUDE_LOWER_PARAM, true);
    }

    if (lt != null && lt.length() != 0)
    {
      to = lt;
      include_upper = false;
    }
    else if (lte != null && lte.length() != 0)
    {
      to = lte;
      include_upper = true;
    }
    else
    {
      to            = jsonObj.optString(TO_PARAM, null);
      include_upper = jsonObj.optBoolean(INCLUDE_UPPER_PARAM, true);
    }

    return new Filter()
    {
      @Override
      public DocIdSet getDocIdSet(final IndexReader reader) throws IOException
      {
        String fromPadded = from, toPadded = to;
        if (!noOptimize)
        {
          if (reader instanceof BoboIndexReader)
          {
            BoboIndexReader boboReader = (BoboIndexReader)reader;
            FacetHandler facetHandler = boboReader.getFacetHandler(field);
            if (facetHandler != null)
            {
              StringBuilder sb = new StringBuilder();
              if (include_lower && from != null && from.length() != 0)
                sb.append("[");
              else
                sb.append("(");

              if (from == null || from.length() == 0)
                sb.append("*");
              else
                sb.append(from);
              sb.append(" TO ");
              if (to == null || to.length() == 0)
                sb.append("*");
              else
                sb.append(to);

              if (include_upper && to != null && to.length() != 0)
                sb.append("]");
              else
                sb.append(")");
              RandomAccessFilter filter = null;;
              Object facetData = boboReader.getFacetData(field);
              //If it's Sensei's standard facet handler
              if (facetData instanceof FacetDataCache) {
              	filter = new FacetRangeFilter(facetHandler, sb.toString());
              } else {
                filter = facetHandler.buildRandomAccessFilter(sb.toString(), null);
              }
              return filter.getDocIdSet(reader);
            }
          }
        }
        
        if(type == null)
          return EmptyFilter.getInstance().getDocIdSet(reader);
        
        if ("int".equals(type)) {
          MetaType metaType = DefaultSenseiInterpreter.CLASS_METATYPE_MAP.get(int.class);
          String formatString = DefaultSenseiInterpreter.DEFAULT_FORMAT_STRING_MAP.get(metaType);
          DecimalFormat formatter = new DecimalFormat(formatString, new DecimalFormatSymbols(Locale.US));
          fromPadded = formatter.format(Integer.parseInt(from));
          toPadded = formatter.format(Integer.parseInt(to));
        } 
        else if ("short".equals(type)) {
          MetaType metaType = DefaultSenseiInterpreter.CLASS_METATYPE_MAP.get(short.class);
{'fraction_non_alphanumeric': 0.10532452390205985, 'fraction_numerical': 0.001554605518849592, 'mean_word_length': 2.4287141905396403, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6820632', 'n_tokens_mistral': 3111, 'n_tokens_neox': 2989, 'n_words': 619}