<commit_before>package tarski

import scala.annotation.tailrec
import scala.math._
import tarski.JavaScores._
import ambiguity.Utility._
import scala.reflect.ClassTag

object Scores {
  /* For now, we choose among options using frequentist statistics.  That is, we score A based on the probability
   *
   *   p = Pr(user chooses B | user thinks of A)
   *
   * where B is what we see as input.
   */

  // High probabilities compare first
  abstract class HasProb extends Comparable[HasProb] {
    def p: Double
    def compareTo(o: HasProb): Int = {
      val x = p
      val y = o.p
      if (x > y) -1 else if (x < y) 1 else 0
    }
  }

  // Probabilities are either simple doubles (normally), or structures declared in Java (for debugging with names)
  type Prob = Double
  @inline def Prob(name: => String, p: Prob): Prob = p
  //type Prob = DebugProb
  //def Prob(name: String, p: Double): Prob = new NameProb(name,p)

  // Probabilities
  case class Alt[+A](dp: Prob, x: A) extends HasProb { // Explicit class to avoid boxing the probability
    def p = pp(dp)
  }

  sealed abstract class Scored[+A] extends HasProb {
    // Invariant: p >= probability of any option
    def p: Double

    // These force some evaluation
    private final def strict: StrictScored[A] = {
      @tailrec def loop(x: Scored[A]): StrictScored[A] = x match {
        case x:StrictScored[A] => x
        case x:LazyScored[A] => loop(x force 0)
      }
      loop(this)
    }
    final def best: Either[Error,A] = strict match {
      case Best(_,x,_) => Right(x)
      case x:EmptyOrBad => Left(x.error)
    }
    final def all: Either[Error,Stream[Alt[A]]] = strict match {
      case x:Best[A] => Right(x.stream)
      case x:EmptyOrBad => Left(x.error)
    }
    final def stream: Stream[Alt[A]] = strict match {
      case Best(p,x,r) => Alt(p,x) #:: r.stream
      case x:EmptyOrBad => Stream.Empty
    }
    final def isEmpty: Boolean = strict match {
      case x:Best[_] => false
      case _:EmptyOrBad => true
    }
    final def isSingle: Boolean = strict match {
      case Best(_,_,r) => r.isEmpty
      case _:EmptyOrBad => false
    }

    // Multiply all probabilities by p.  For internal use only: users should call biased(p,s).
    def _bias(p: Prob): Scored[A]

    // Apply f to every alternative
    def map[B](f: A => B): Scored[B] = new LazyMap(this,f)

    // Either this or s
    def ++[B >: A](s: Scored[B]): Scored[B]

    // f is assumed to generate conditional probabilities
    def flatMap[B](f: A => Scored[B]): Scored[B] = new Extractor[B](new FlatMapState(this,f))

    // We are assumed independent of t
    def productWith[B,C](s: Scored[B])(f: (A,B) => C): Scored[C] = new LazyProductWith(this,s,f)

    // Filter, turning Empty into given error
    final def filter(f: A => Boolean, error: => String): Scored[A] = _filter(f, if (trackErrors) () => error else null)
    def _filter(f: A => Boolean, error: () => String): Scored[A] = new LazyFilter(this,f,error)
  }

  // Scored with at least one option evaluated (if any exists)
  sealed abstract class StrictScored[+A] extends Scored[A]

  // A lazy version of Scored
  abstract class LazyScored[+A] extends Scored[A] {
    // May return another LazyScored, usually with lower probability.  Optionally continue until prob <= p.
    def force(hi: Double): Scored[A]

    def _bias(p: Prob): Scored[A] = new LazyBias(this,p)
    def ++[B >: A](s: Scored[B]): Scored[B] =
      if (p >= s.p) new LazyPlus(this,s)
      else s match {
        case s:LazyScored[B] => new LazyPlus(s,this)
        case s:Best[B] => Best(s.dp,s.x,s.r ++ this)
        case _:EmptyOrBad => impossible
      }
  }

  // No options
  private val zeroProb = Prob("zero",0)
  sealed abstract class EmptyOrBad extends StrictScored[Nothing] {
    def p = 0
    def error: Error
    override def map[B](f: Nothing => B) = this
    override def flatMap[B](f: Nothing => Scored[B]) = this
    override def _bias(p: Prob) = this
    override def productWith[B,C](s: Scored[B])(f: (Nothing,B) => C) = this
    override def _filter(f: Nothing => Boolean, error: () => String) = this
  }
  // Failure
  final class Bad(_error: => Error) extends EmptyOrBad {
    lazy val error = _error
    def ++[B](s: Scored[B]) = s match {
      case s:LazyScored[B] => new LazyPlus(s,this)
      case s:EmptyOrBad => new Bad(NestError("++ failed",List(error,s.error)))
      case _:Best[_] => this
    }
  }
  // No options, but not Bad
  object Empty extends EmptyOrBad {<commit_msg>Remove unused declaration of zeroProb<commit_after>package tarski

import scala.annotation.tailrec
import scala.math._
import tarski.JavaScores._
import ambiguity.Utility._
import scala.reflect.ClassTag

object Scores {
  /* For now, we choose among options using frequentist statistics.  That is, we score A based on the probability
   *
   *   p = Pr(user chooses B | user thinks of A)
   *
   * where B is what we see as input.
   */

  // High probabilities compare first
  abstract class HasProb extends Comparable[HasProb] {
    def p: Double
    def compareTo(o: HasProb): Int = {
      val x = p
      val y = o.p
      if (x > y) -1 else if (x < y) 1 else 0
    }
  }

  // Probabilities are either simple doubles (normally), or structures declared in Java (for debugging with names)
  type Prob = Double
  @inline def Prob(name: => String, p: Prob): Prob = p
  //type Prob = DebugProb
  //def Prob(name: String, p: Double): Prob = new NameProb(name,p)

  // Probabilities
  case class Alt[+A](dp: Prob, x: A) extends HasProb { // Explicit class to avoid boxing the probability
    def p = pp(dp)
  }

  sealed abstract class Scored[+A] extends HasProb {
    // Invariant: p >= probability of any option
    def p: Double

    // These force some evaluation
    private final def strict: StrictScored[A] = {
      @tailrec def loop(x: Scored[A]): StrictScored[A] = x match {
        case x:StrictScored[A] => x
        case x:LazyScored[A] => loop(x force 0)
      }
      loop(this)
    }
    final def best: Either[Error,A] = strict match {
      case Best(_,x,_) => Right(x)
      case x:EmptyOrBad => Left(x.error)
    }
    final def all: Either[Error,Stream[Alt[A]]] = strict match {
      case x:Best[A] => Right(x.stream)
      case x:EmptyOrBad => Left(x.error)
    }
    final def stream: Stream[Alt[A]] = strict match {
      case Best(p,x,r) => Alt(p,x) #:: r.stream
      case x:EmptyOrBad => Stream.Empty
    }
    final def isEmpty: Boolean = strict match {
      case x:Best[_] => false
      case _:EmptyOrBad => true
    }
    final def isSingle: Boolean = strict match {
      case Best(_,_,r) => r.isEmpty
      case _:EmptyOrBad => false
    }

    // Multiply all probabilities by p.  For internal use only: users should call biased(p,s).
    def _bias(p: Prob): Scored[A]

    // Apply f to every alternative
    def map[B](f: A => B): Scored[B] = new LazyMap(this,f)

    // Either this or s
    def ++[B >: A](s: Scored[B]): Scored[B]

    // f is assumed to generate conditional probabilities
    def flatMap[B](f: A => Scored[B]): Scored[B] = new Extractor[B](new FlatMapState(this,f))

    // We are assumed independent of t
    def productWith[B,C](s: Scored[B])(f: (A,B) => C): Scored[C] = new LazyProductWith(this,s,f)

    // Filter, turning Empty into given error
    final def filter(f: A => Boolean, error: => String): Scored[A] = _filter(f, if (trackErrors) () => error else null)
    def _filter(f: A => Boolean, error: () => String): Scored[A] = new LazyFilter(this,f,error)
  }

  // Scored with at least one option evaluated (if any exists)
  sealed abstract class StrictScored[+A] extends Scored[A]

  // A lazy version of Scored
  abstract class LazyScored[+A] extends Scored[A] {
    // May return another LazyScored, usually with lower probability.  Optionally continue until prob <= p.
    def force(hi: Double): Scored[A]

    def _bias(p: Prob): Scored[A] = new LazyBias(this,p)
    def ++[B >: A](s: Scored[B]): Scored[B] =
      if (p >= s.p) new LazyPlus(this,s)
      else s match {
        case s:LazyScored[B] => new LazyPlus(s,this)
        case s:Best[B] => Best(s.dp,s.x,s.r ++ this)
        case _:EmptyOrBad => impossible
      }
  }

  // No options
  sealed abstract class EmptyOrBad extends StrictScored[Nothing] {
    def p = 0
    def error: Error
    override def map[B](f: Nothing => B) = this
    override def flatMap[B](f: Nothing => Scored[B]) = this
    override def _bias(p: Prob) = this
    override def productWith[B,C](s: Scored[B])(f: (Nothing,B) => C) = this
    override def _filter(f: Nothing => Boolean, error: () => String) = this
  }
  // Failure
  final class Bad(_error: => Error) extends EmptyOrBad {
    lazy val error = _error
    def ++[B](s: Scored[B]) = s match {
      case s:LazyScored[B] => new LazyPlus(s,this)
      case s:EmptyOrBad => new Bad(NestError("++ failed",List(error,s.error)))
      case _:Best[_] => this
    }
  }
  // No options, but not Bad
  object Empty extends EmptyOrBad {
{'fraction_non_alphanumeric': 0.13149711239449133, 'fraction_numerical': 0.0012216792536650378, 'mean_word_length': 2.9220383275261326, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5620130', 'n_tokens_mistral': 3134, 'n_tokens_neox': 3012, 'n_words': 988}