<commit_before>// BSD License (http://lemurproject.org/galago-license)
package org.lemurproject.galago.core.retrieval;

import java.io.Serializable;
import org.lemurproject.galago.core.retrieval.query.AnnotatedNode;

/**
 * Basic retrieval unit. The results returned by the Retrieval.runQuery typically return
 * a ranked list of these.
 *
 * @author trevor, irmarc, sjh
 */
public class ScoredDocument implements Comparable<ScoredDocument>, Serializable {

  public String documentName;
  public String source; // lets us know where this scored doc came from
  public int document;
  public int rank;
  public double score;
  public AnnotatedNode annotation = null;

  public ScoredDocument() {
    this(0, 0);
  }

  public ScoredDocument(int document, double score) {
    this.document = document;
    this.score = score;
  }

  public ScoredDocument(String documentName, int rank, double score) {
    this.documentName = documentName;
    this.rank = rank;
    this.score = score;
    this.document = -1;
  }

  @Override
  public int compareTo(ScoredDocument other) {
    if (score != other.score) {
      return Double.compare(score, other.score);
    }
    if ((source != null) && (other.source != null)
            && (!source.equals(other.source))) {
      return source.compareTo(other.source);
    }
    return other.document - document;
  }

  @Override
  public String toString() {
    return String.format("%s %d %s galago", documentName, rank, formatScore(score));
  }

  public String toString(String qid) {
    return String.format("%s Q0 %s %d %s galago", qid, documentName, rank,  formatScore(score));
  }

  public String toTRECformat(String qid) {
    return String.format("%s Q0 %s %d %s galago", qid, documentName, rank, formatScore(score));
  }

  protected static String formatScore(double score) {
    double difference = Math.abs(score - (int) score);

    if (difference < 0.00001) {
      return Integer.toString((int) score);
    }
    return String.format("%10.8f", score);
  }
}
<commit_msg>modified scoreddocument comparator to be faster<commit_after>// BSD License (http://lemurproject.org/galago-license)
package org.lemurproject.galago.core.retrieval;

import java.io.Serializable;
import org.lemurproject.galago.core.retrieval.query.AnnotatedNode;
import org.lemurproject.galago.tupleflow.Utility;

/**
 * Basic retrieval unit. The results returned by the Retrieval.runQuery typically return
 * a ranked list of these.
 *
 * @author trevor, irmarc, sjh
 */
public class ScoredDocument implements Comparable<ScoredDocument>, Serializable {

  public String documentName;
  public String source; // lets us know where this scored doc came from
  public int document;
  public int rank;
  public double score;
  public AnnotatedNode annotation = null;

  public ScoredDocument() {
    this(0, 0);
  }

  public ScoredDocument(int document, double score) {
    this.document = document;
    this.score = score;
  }

  public ScoredDocument(String documentName, int rank, double score) {
    this.documentName = documentName;
    this.rank = rank;
    this.score = score;
    this.document = -1;
  }

  @Override
  public int compareTo(ScoredDocument other) {
    if (score != other.score) {
      return Utility.compare(score, other.score);
    }
    if ((source != null) && (other.source != null)
            && (!source.equals(other.source))) {
      return source.compareTo(other.source);
    }
    return other.document - document;
  }

  @Override
  public String toString() {
    return String.format("%s %d %s galago", documentName, rank, formatScore(score));
  }

  public String toString(String qid) {
    return String.format("%s Q0 %s %d %s galago", qid, documentName, rank,  formatScore(score));
  }

  public String toTRECformat(String qid) {
    return String.format("%s Q0 %s %d %s galago", qid, documentName, rank, formatScore(score));
  }

  protected static String formatScore(double score) {
    double difference = Math.abs(score - (int) score);

    if (difference < 0.00001) {
      return Integer.toString((int) score);
    }
    return String.format("%10.8f", score);
  }
}

{'fraction_non_alphanumeric': 0.11595255386105059, 'fraction_numerical': 0.006778019849915275, 'mean_word_length': 3.5707964601769913, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1795805', 'n_tokens_mistral': 1282, 'n_tokens_neox': 1244, 'n_words': 357}