<commit_before>package opennlp.textgrounder.gridlocate

import math.log

import opennlp.textgrounder.worddist.WordDist._
import opennlp.textgrounder.worddist.UnigramWordDist
import opennlp.textgrounder.util.printutil._
import opennlp.textgrounder.perceptron._

import GridLocateDriver.Debug._

/**
 * A basic ranker.  Given a test item, return a list of ranked answers from
 * best to worst, with a score for each.  The score must not increase from
 * any answer to the next one.
 */
trait Ranker[TestItem, Answer] {
  /**
   * Evaluate a test item, returning a list of ranked answers from best to
   * worst, with a score for each.  The score must not increase from any
   * answer to the next one.  Any answers mentioned in `include` must be
   * included in the returned list.
   */
  def evaluate(item: TestItem, include: Iterable[Answer]):
    Iterable[(Answer, Double)]
}

/**
 * A ranker for ranking cells in a grid as possible matches for a given
 * document.
 *
 * @param strategy Object encapsulating the strategy used for performing
 *   ranking.
 */
class GridRanker[Co](
  strategy: GridLocateDocumentStrategy[Co]
 ) extends Ranker[GeoDoc[Co], GeoCell[Co]] {
  def evaluate(item: GeoDoc[Co], include: Iterable[GeoCell[Co]]) =
    strategy.return_ranked_cells(item.dist, include)
}

/**
 * A reranker.  This is a particular type of ranker that involves two
 * steps: One to compute an initial ranking, and a second to do a more
 * accurate reranking of some subset of the highest-ranked answers.  The
 * idea is that a more accurate value can be computed when there are fewer
 * possible answers to distinguish -- assuming that the initial ranker
 * is able to include the correct answer near the top significantly more
 * often than actually at the top.
 */
trait Reranker[TestItem, Answer] extends Ranker[TestItem, Answer] {
  /** Ranker for generating initial ranking. */
  protected def initial_ranker: Ranker[TestItem, Answer]

  /**
   * Number of top-ranked items to submit to reranking.
   */
  protected val top_n: Int

  /**
   * Rerank the given answers, based on an initial ranking.
   */
  def rerank_answers(item: TestItem,
    initial_ranking: Iterable[(Answer, Double)]): Iterable[(Answer, Double)]

  def evaluate_with_initial_ranking(item: TestItem,
      include: Iterable[Answer]) = {
    val initial_ranking = initial_ranker.evaluate(item, include)
    val (to_rerank, others) = initial_ranking.splitAt(top_n)
    val reranking = rerank_answers(item, to_rerank) ++ others
    (initial_ranking, reranking)
  }

  def evaluate(item: TestItem, include: Iterable[Answer]) = {
    val (initial_ranking, reranking) =
      evaluate_with_initial_ranking(item, include)
    reranking
  }
}

/**
 * A pointwise reranker that uses a scoring classifier to assign a score
 * to each possible answer to be reranked.  The idea is that, for each
 * possible answer, we create test instances based on a combination of the
 * test item and answer and score the instances to determine the ranking.
 */
trait PointwiseClassifyingReranker[TestItem, RerankInstance, Answer]
    extends Reranker[TestItem, Answer] {
  /** Scoring classifier for use in reranking. */
  protected def rerank_classifier: ScoringBinaryClassifier[RerankInstance]

  /**
   * Create a reranking training instance to feed to the classifier, given
   * a test item, a potential answer from the ranker, and whether the answer
   * is correct.  These training instances will be used to train the
   * classifier.
   */
  protected val create_rerank_instance:
    (TestItem, Answer, Double, Boolean) => RerankInstance

  /**
   * Generate rerank training instances for a given ranker
   * training instance.
   */
  protected def get_rerank_training_instances(item: TestItem,
      true_answer: Answer) = {
    val initial_answers = initial_ranker.evaluate(item, Iterable(true_answer))
    val top_answers = initial_answers.take(top_n)
    val answers =
      if (top_answers.find(_._1 == true_answer) != None)
        top_answers
      else
        top_answers ++ Iterable(initial_answers.find(_._1 == true_answer).get)
    for {(possible_answer, score) <- answers
         is_correct = possible_answer == true_answer
        }
      yield (
        create_rerank_instance(item, possible_answer, score, true), is_correct)
  }

  def rerank_answers(item: TestItem,
      answers: Iterable[(Answer, Double)]) = {
    val new_scores =
      for {(answer, score) <- answers
           instance = create_rerank_instance(item, answer, score, false)
           new_score = rerank_classifier.score_item(instance)
          }
        yield (answer, new_score)
    new_scores.toIndexedSeq sortWith (_._2 > _._2)
  }
}

/**
 * A pointwise classifying reranker that uses training data to train the
 * classifier.  The training data consists of test items and correct
 * answers.  From each data item, a series of training instances are
 * generated as follows: (1) rank the data item to produce a set of
 * possible answers; (2) if necessary, augment the possible answers to
 * include the correct one; (3) create a training instance for each
 * combination of test item and answer, with "correct" or "incorrect"
 * indicated.
 */
trait PointwiseClassifyingRerankerWithTrainingData[
    TestItem, RerankInstance, Answer] extends
  PointwiseClassifyingReranker[TestItem, RerankInstance, Answer] {
  /**
   * Training data used to create the reranker.
   */
  protected val training_data: Iterable[(TestItem, Answer)]

  /**
   * Create the classifier used for reranking, given a set of training data
   * (in the form of pairs of reranking instances and whether they represent
   * correct answers).
   */
  protected val create_rerank_classifier:
    Iterable[(RerankInstance, Boolean)] => ScoringBinaryClassifier[RerankInstance]

  lazy protected val rerank_classifier = {
    val rerank_training_data = training_data.flatMap {
      case (item, true_answer) =>
        get_rerank_training_instances(item, true_answer)
    }
    create_rerank_classifier(rerank_training_data)
  }
}

/**
 * A scoring binary classifier.  Given a test item, return a score, with
 * higher numbers indicating greater likelihood of being "positive".
 */
trait ScoringBinaryClassifier[TestItem] {
  /**
   * The value of `minimum_positive` indicates the dividing line between values
   * that should be considered positive and those that should be considered<commit_msg>Add some info for debugging generation of rerank training instances<commit_after>package opennlp.textgrounder.gridlocate

import math.log

import opennlp.textgrounder.worddist.WordDist._
import opennlp.textgrounder.worddist.UnigramWordDist
import opennlp.textgrounder.util.printutil._
import opennlp.textgrounder.perceptron._

import GridLocateDriver.Debug._

/**
 * A basic ranker.  Given a test item, return a list of ranked answers from
 * best to worst, with a score for each.  The score must not increase from
 * any answer to the next one.
 */
trait Ranker[TestItem, Answer] {
  /**
   * Evaluate a test item, returning a list of ranked answers from best to
   * worst, with a score for each.  The score must not increase from any
   * answer to the next one.  Any answers mentioned in `include` must be
   * included in the returned list.
   */
  def evaluate(item: TestItem, include: Iterable[Answer]):
    Iterable[(Answer, Double)]
}

/**
 * A ranker for ranking cells in a grid as possible matches for a given
 * document.
 *
 * @param strategy Object encapsulating the strategy used for performing
 *   ranking.
 */
class GridRanker[Co](
  strategy: GridLocateDocumentStrategy[Co]
 ) extends Ranker[GeoDoc[Co], GeoCell[Co]] {
  def evaluate(item: GeoDoc[Co], include: Iterable[GeoCell[Co]]) =
    strategy.return_ranked_cells(item.dist, include)
}

/**
 * A reranker.  This is a particular type of ranker that involves two
 * steps: One to compute an initial ranking, and a second to do a more
 * accurate reranking of some subset of the highest-ranked answers.  The
 * idea is that a more accurate value can be computed when there are fewer
 * possible answers to distinguish -- assuming that the initial ranker
 * is able to include the correct answer near the top significantly more
 * often than actually at the top.
 */
trait Reranker[TestItem, Answer] extends Ranker[TestItem, Answer] {
  /** Ranker for generating initial ranking. */
  protected def initial_ranker: Ranker[TestItem, Answer]

  /**
   * Number of top-ranked items to submit to reranking.
   */
  protected val top_n: Int

  /**
   * Rerank the given answers, based on an initial ranking.
   */
  def rerank_answers(item: TestItem,
    initial_ranking: Iterable[(Answer, Double)]): Iterable[(Answer, Double)]

  def evaluate_with_initial_ranking(item: TestItem,
      include: Iterable[Answer]) = {
    val initial_ranking = initial_ranker.evaluate(item, include)
    val (to_rerank, others) = initial_ranking.splitAt(top_n)
    val reranking = rerank_answers(item, to_rerank) ++ others
    (initial_ranking, reranking)
  }

  def evaluate(item: TestItem, include: Iterable[Answer]) = {
    val (initial_ranking, reranking) =
      evaluate_with_initial_ranking(item, include)
    reranking
  }
}

/**
 * A pointwise reranker that uses a scoring classifier to assign a score
 * to each possible answer to be reranked.  The idea is that, for each
 * possible answer, we create test instances based on a combination of the
 * test item and answer and score the instances to determine the ranking.
 */
trait PointwiseClassifyingReranker[TestItem, RerankInstance, Answer]
    extends Reranker[TestItem, Answer] {
  /** Scoring classifier for use in reranking. */
  protected def rerank_classifier: ScoringBinaryClassifier[RerankInstance]

  /**
   * Create a reranking training instance to feed to the classifier, given
   * a test item, a potential answer from the ranker, and whether the answer
   * is correct.  These training instances will be used to train the
   * classifier.
   */
  protected val create_rerank_instance:
    (TestItem, Answer, Double, Boolean) => RerankInstance

  /**
   * Generate rerank training instances for a given ranker
   * training instance.
   */
  protected def get_rerank_training_instances(item: TestItem,
      true_answer: Answer) = {
    val initial_answers = initial_ranker.evaluate(item, Iterable(true_answer))
    val top_answers = initial_answers.take(top_n)
    val answers =
      if (top_answers.find(_._1 == true_answer) != None)
        top_answers
      else
        top_answers ++ Iterable(initial_answers.find(_._1 == true_answer).get)
    for {(possible_answer, score) <- answers
         is_correct = possible_answer == true_answer
        }
      yield (
        create_rerank_instance(item, possible_answer, score, true), is_correct)
  }

  def rerank_answers(item: TestItem,
      answers: Iterable[(Answer, Double)]) = {
    val new_scores =
      for {(answer, score) <- answers
           instance = create_rerank_instance(item, answer, score, false)
           new_score = rerank_classifier.score_item(instance)
          }
        yield (answer, new_score)
    new_scores.toIndexedSeq sortWith (_._2 > _._2)
  }
}

/**
 * A pointwise classifying reranker that uses training data to train the
 * classifier.  The training data consists of test items and correct
 * answers.  From each data item, a series of training instances are
 * generated as follows: (1) rank the data item to produce a set of
 * possible answers; (2) if necessary, augment the possible answers to
 * include the correct one; (3) create a training instance for each
 * combination of test item and answer, with "correct" or "incorrect"
 * indicated.
 */
trait PointwiseClassifyingRerankerWithTrainingData[
    TestItem, RerankInstance, Answer] extends
  PointwiseClassifyingReranker[TestItem, RerankInstance, Answer] {
  /**
   * Training data used to create the reranker.
   */
  protected val training_data: Iterable[(TestItem, Answer)]

  /**
   * Create the classifier used for reranking, given a set of training data
   * (in the form of pairs of reranking instances and whether they represent
   * correct answers).
   */
  protected val create_rerank_classifier:
    Iterable[(RerankInstance, Boolean)] => ScoringBinaryClassifier[RerankInstance]

  lazy protected val rerank_classifier = {
    val rerank_training_data =
      if (debug("rerank-training")) {
        training_data.zipWithIndex.flatMap {
          case ((item, true_answer), index) => {
            val prefix = "#%d: " format (index + 1)
            errprint("%sTraining item: %s", prefix, item)
            errprint("%sTrue answer: %s", prefix, true_answer)
            val training_insts =
              get_rerank_training_instances(item, true_answer)
            for (((featvec, correct), instind) <- training_insts.zipWithIndex) {
              val instpref = "%s#%d: " format (prefix, instind + 1)
              val correctstr =
                if (correct) "CORRECT" else "INCORRECT"
              errprint("%s%s: %s", instpref, correctstr, featvec)
            }
            training_insts
          }
        }
      } else {
        training_data.flatMap {
          case (item, true_answer) =>
            get_rerank_training_instances(item, true_answer)
        }
      }
    create_rerank_classifier(rerank_training_data)
  }
}

/**
 * A scoring binary classifier.  Given a test item, return a score, with
 * higher numbers indicating greater likelihood of being "positive".
 */
trait ScoringBinaryClassifier[TestItem] {
  /**
   * The value of `minimum_positive` indicates the dividing line between values
   * that should be considered positive and those that should be considered
{'fraction_non_alphanumeric': 0.0817219992703393, 'fraction_numerical': 0.0011674571324334184, 'mean_word_length': 3.5993288590604027, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '936028', 'n_tokens_mistral': 4066, 'n_tokens_neox': 3894, 'n_words': 1385}