<commit_before>package provingground.experiments

import spire.math._
import spire.algebra._
import spire.implicits._

class FieldGeomDist[F](N: Int = 100, p: Double = 0.5)(implicit field: Field[F]){
  val minusOne : F = field.negate(field.one)

  val q = 1 - p

  def sumErr(prob: Vector[F]) = (prob.foldRight[F](minusOne)(_ + _)).pow(2)

  def recErrTerms(prob: Vector[F]) : Vector[F] =
    {
      val prevMul = prob.init.map{(x) => x * q}
      prob.tail.zip(prevMul).map{
        case (a, b) => (a - b).pow(2)
      }
    }

  def totalError(prob: Vector[F]) =
    recErrTerms(prob).fold(sumErr(prob))(_ + _)

}

class JetGeomDist(N: Int = 100, p: Double =  0.5){
  implicit val dim = JetDim(N)

  object DistErr extends FieldGeomDist[Jet[Double]](N, p){
    def tangent(prob: Vector[Double]) =
      prob.zipWithIndex.map{
        case (x, n) => x + Jet.h[Double](n)
      }
  }

}

object JetGeomDist{
  def distErr(N: Int = 2, p: Double = 0.5) = new JetGeomDist(N, p).DistErr
}<commit_msg>more experiments; fine in simple case, scales poorly with additive errors.<commit_after>package provingground.experiments

import spire.math._
import spire.algebra._
import spire.implicits._

class FieldGeomDist[F](N: Int = 100, p: Double = 0.5)(implicit field: Field[F]){
  val minusOne : F = field.negate(field.one)

  val q = 1 - p

  def sumErr(prob: Vector[F]) = (prob.foldRight[F](minusOne)(_ + _)).pow(2)

  def recErrTerms(prob: Vector[F]) : Vector[F] =
    {
      val prevMul = prob.init.map{(x) => x * q}
      prob.tail.zip(prevMul).map{
        case (a, b) => (a - b).pow(2)
      }
    }

  def totalError(prob: Vector[F]) =
    recErrTerms(prob).fold(sumErr(prob))(_ + _)

}

class JetGeomDist(N: Int = 100, p: Double =  0.5){
  implicit val dim = JetDim(N)

  object DistErr extends FieldGeomDist[Jet[Double]](N, p){
    def tangent(prob: Vector[Double]) =
      prob.zipWithIndex.map{
        case (x, n) => x + Jet.h[Double](n)
      }

    def shifted(prob: Vector[Double], epsilon: Double = 0.1) =
      {
        val shift =
          totalError(tangent(prob)).infinitesimal.toVector
        prob.zip(shift).map{
          case (x, t) => x - (epsilon * t)
        }
      }

    @annotation.tailrec
    def flowed(prob: Vector[Double], steps: Int = 10, epsilon: Double = 0.1) : Vector[Double] =
      if (steps < 1) prob
      else flowed(shifted(prob, epsilon), steps - 1, epsilon)
  }

}

object JetGeomDist{
  def distErr(N: Int = 2, p: Double = 0.5) = new JetGeomDist(N, p).DistErr
}
{'fraction_non_alphanumeric': 0.1596806387225549, 'fraction_numerical': 0.015968063872255488, 'mean_word_length': 2.9589257503949447, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5693514', 'n_tokens_mistral': 995, 'n_tokens_neox': 944, 'n_words': 219}