<commit_before>/*
 * BSD License (http://lemurproject.org/galago-license)

 */
package org.lemurproject.galago.core.retrieval.iterator;

import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import org.lemurproject.galago.core.index.disk.FieldIndexReader;
import org.lemurproject.galago.core.retrieval.query.AnnotatedNode;
import org.lemurproject.galago.core.retrieval.query.NodeParameters;

/**
 * Abstract superclass for comparing fields to values given.
 * The values to match against may involve more than one (i.e. in
 * the between operator), so that extraction is not done here.
 * All navigation functions are taken care of here though.
 *
 * @author irmarc
 */
public abstract class FieldComparisonIterator extends TransformIterator implements MovableIndicatorIterator {
  NodeParameters p;
  FieldIndexReader.ListIterator fieldIterator;
  String format;
  String strValue;
  int intValue;
  long longValue;
  float floatValue;
  double doubleValue;
  long dateValue;

  public FieldComparisonIterator(NodeParameters p, FieldIndexReader.ListIterator fieldIterator) {
    super(fieldIterator);
    this.p = p;
    this.fieldIterator = fieldIterator;
    this.format = fieldIterator.getFormat();
  }

  protected void parseField(NodeParameters p) {
    try {
      if (format.equals("string")) {
        strValue = p.getString("0");
      } else if (format.equals("int")) {
        intValue = Integer.parseInt(p.getString("0"));
      } else if (format.equals("long")) {
        longValue = Long.parseLong(p.getString("0"));
      } else if (format.equals("float")) {
        floatValue = Float.parseFloat(p.getString("0"));
      } else if (format.equals("double")) {
        doubleValue = Double.parseDouble(p.getString("0"));
      } else if (format.equals("date")) {
        DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT);
        dateValue = df.parse(p.getString("0")).getTime();
      } else {
        throw new RuntimeException(String.format("Don't have any plausible format for tag %s\n",
                format));
      }
    } catch (ParseException pe) {
      throw new RuntimeException(pe);
    }
  }

  @Override
  public AnnotatedNode getAnnotatedNode() throws IOException {
    String type = "indicator";
    String className = this.getClass().getSimpleName();
    String parameters = p.toString();
    int document = currentCandidate();
    boolean atCandidate = hasMatch(this.context.document);
    String returnValue = Boolean.toString( this.indicator( this.context.document ) );
    List<AnnotatedNode> children = Collections.singletonList( this.iterator.getAnnotatedNode() );
    
    return new AnnotatedNode(type, className, parameters, document, atCandidate, returnValue, children);
  }
}
<commit_msg>Get the typed objects directly from the parameters, rather than getting a string to parse (addresses issue with numeric parameters to field comparison query operators).<commit_after>/*
 * BSD License (http://lemurproject.org/galago-license)

 */
package org.lemurproject.galago.core.retrieval.iterator;

import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import org.lemurproject.galago.core.index.disk.FieldIndexReader;
import org.lemurproject.galago.core.retrieval.query.AnnotatedNode;
import org.lemurproject.galago.core.retrieval.query.NodeParameters;

/**
 * Abstract superclass for comparing fields to values given.
 * The values to match against may involve more than one (i.e. in
 * the between operator), so that extraction is not done here.
 * All navigation functions are taken care of here though.
 *
 * @author irmarc
 */
public abstract class FieldComparisonIterator extends TransformIterator implements MovableIndicatorIterator {
  NodeParameters p;
  FieldIndexReader.ListIterator fieldIterator;
  String format;
  String strValue;
  int intValue;
  long longValue;
  float floatValue;
  double doubleValue;
  long dateValue;

  public FieldComparisonIterator(NodeParameters p, FieldIndexReader.ListIterator fieldIterator) {
    super(fieldIterator);
    this.p = p;
    this.fieldIterator = fieldIterator;
    this.format = fieldIterator.getFormat();
  }

  protected void parseField(NodeParameters p) {
    try {
      if (format.equals("string")) {
        strValue = p.getString("0");
      } else if (format.equals("int")) {
          intValue = (int) p.getLong("0");
      } else if (format.equals("long")) {
          longValue = p.getLong("0");
      } else if (format.equals("float")) {
          floatValue = (float) p.getDouble("0");
      } else if (format.equals("double")) {
        doubleValue = p.getDouble("0");
      } else if (format.equals("date")) {
        DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT);
        dateValue = df.parse(p.getString("0")).getTime();
      } else {
        throw new RuntimeException(String.format("Don't have any plausible format for tag %s\n",
                format));
      }
    } catch (ParseException pe) {
      throw new RuntimeException(pe);
    }
  }

  @Override
  public AnnotatedNode getAnnotatedNode() throws IOException {
    String type = "indicator";
    String className = this.getClass().getSimpleName();
    String parameters = p.toString();
    int document = currentCandidate();
    boolean atCandidate = hasMatch(this.context.document);
    String returnValue = Boolean.toString( this.indicator( this.context.document ) );
    List<AnnotatedNode> children = Collections.singletonList( this.iterator.getAnnotatedNode() );
    
    return new AnnotatedNode(type, className, parameters, document, atCandidate, returnValue, children);
  }
}

{'fraction_non_alphanumeric': 0.11028264262181377, 'fraction_numerical': 0.0020808045777700713, 'mean_word_length': 3.8146911519198663, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4828705', 'n_tokens_mistral': 1617, 'n_tokens_neox': 1598, 'n_words': 416}