<commit_before>package tuner.gp

import net.liftweb.json.JsonParser._
import net.liftweb.json.JsonAST._
import net.liftweb.json.JsonDSL._

import numberz.Matrix
import numberz.Vector

import org.rosuda.JRI.RList

import tuner.Config
import tuner.DimRanges
import tuner.Grid2D
import tuner.Sampler
import tuner.Table
import tuner.util.Util

case class GpSpecification(
  responseDim:String,
  dimNames:List[String],
  thetas:List[Double],
  alphas:List[Double],
  mean:Double,
  sigma2:Double,
  designMatrix:List[List[Double]],
  responses:List[Double],
  invCorMtx:List[List[Double]]
)

object GpModel {
  def fromJson(json:GpSpecification) = {
    new GpModel(Vector(json.thetas), Vector(json.alphas), 
                json.mean, json.sigma2,
                Matrix.fromColumnMajor(json.designMatrix), 
                Vector(json.responses),
                Matrix.fromColumnMajor(json.invCorMtx),
                json.dimNames, json.responseDim, Config.errorField)
  }
}

// A gp model takes a sampling density and returns 
// a filename from which to read the sampled data
//type Model = Int => String
class GpModel(val thetas:Vector, val alphas:Vector, 
              val mean:Double, val sig2:Double, 
              val design:Matrix, val responses:Vector, 
              val rInverse:Matrix, 
              val dims:List[String], val respDim:String, val errDim:String) {

  def this(thetas:Vector, alphas:Vector, mean:Double, 
           design:Matrix, responses:Vector, 
           rInverse:Matrix, 
           dims:List[String], respDim:String, errDim:String) =
    this(thetas, alphas, mean, 
         (responses.map {_ - mean} dot (rInverse dot responses.map {_ - mean})) / responses.size,
         design, responses, rInverse, dims, respDim, errDim)

  def this(thetas:Vector, alphas:Vector, 
           design:Matrix, responses:Vector, 
           rInverse:Matrix, 
           dims:List[String], respDim:String, errDim:String) =
    this(thetas, alphas, 
         (Vector.ones(responses.size) dot (rInverse dot responses)) /
           (Vector.ones(responses.size) dot (rInverse dot Vector.ones(responses.size))),
         design, responses, rInverse, dims, respDim, errDim)

  // Also precompute rInverse . (responses - mean)
  val corrResponses = rInverse dot (responses map {_ - mean})

  def toJson = {
    GpSpecification(
      respDim, dims, thetas.toList, alphas.toList, mean, sig2,
      design.toColumnMajorList, responses.toList, rInverse.toColumnMajorList)
  }

  def maxGain(range:DimRanges):Float = {
    var mx = Double.MinValue
    Sampler.lhc(range, Config.respHistogramSampleDensity, pt => {
      val (est, err) = runSample(pt)
      val expgain = calcExpectedGain(est, err)
      mx = math.max(mx, expgain)
    })
    mx.toFloat
  }

  // Store the most recently seen max value for the function
  def funcMax:Float = responses.max.toFloat
  def funcMin:Float = responses.min.toFloat

  def theta(dim:String) = thetas(dims.indexOf(dim))

  def sampleSlice(rowDim:(String,(Float,Float)), 
                  colDim:(String,(Float,Float)),
                  slices:List[(String,Float)], 
                  numSamples:Int)
      : ((String, Grid2D), (String, Grid2D), (String, Grid2D)) = {
    
    val arrSlice = Array.fill(dims.length)(0.0)
    val sliceMap = slices.toMap
    dims.zipWithIndex.foreach {case (fld, i) =>
      arrSlice(i) = sliceMap(fld)
    }
    val xDim = dims.indexOf(rowDim._1)
    val yDim = dims.indexOf(colDim._1)

    // generate one matrix from scratch and then copy the rest
    val startTime = System.currentTimeMillis
    val response = Sampler.regularSlice(rowDim, colDim, numSamples)
    val errors = new Grid2D(response.rowIds, response.colIds)
    val gains = new Grid2D(response.rowIds, response.colIds)

    response.rowIds.zipWithIndex.foreach {tmpx =>
      val (xval,x) = tmpx
      response.colIds.zipWithIndex.foreach {tmpy =>
        val (yval, y) = tmpy
        arrSlice(xDim) = xval
        arrSlice(yDim) = yval
        val (est, err) = estimatePoint(Vector(arrSlice))
        response.set(x, y, est.toFloat)
        errors.set(x, y, err.toFloat)
        val expgain = calcExpectedGain(est, err)
        if(!expgain.isNaN) {
          gains.set(x, y, expgain.toFloat)
        } else {
          gains.set(x, y, 0f)
        }
      }
    }

    val endTime = System.currentTimeMillis
    ((respDim, response), 
     (Config.errorField, errors), 
     (Config.gainField, gains))
  }

  def sampleTable(samples:Table) : Table = {
    // verify that the input table has the proper fields
    val sampleFields:Set[String] = samples.fieldNames.toSet
    if(!(dims.toSet subsetOf sampleFields)) {
      val missingFields:Set[String] = dims.toSet -- sampleFields
      throw new tuner.error.NonMatchingParameterException(missingFields toList)
    }

    val outTbl = new Table
    for(r <- 0 until samples.numRows) {
      val tpl = samples.tuple(r)
      val (est, err) = runSample(tpl.toList)
      outTbl.addRow((respDim, est.toFloat)::(errDim, err.toFloat)::tpl.toList)
    }
    outTbl
  }

  def runSample(pt:List[(String, Float)]) : (Double, Double) = {
    val mapx = pt.toMap
    val xx = dims.map({mapx.get(_)}).flatten.map({_.toDouble})
    val (est, err) = estimatePoint(Vector(xx))
    //curFuncMax = math.max(est.toFloat, curFuncMax)
    (est, err)
  }

  // Compute the correlation wrt each design point
  private def estimatePoint(point:Vector) : (Double,Double) = {<commit_msg>faster way of predicting points<commit_after>package tuner.gp

import net.liftweb.json.JsonParser._
import net.liftweb.json.JsonAST._
import net.liftweb.json.JsonDSL._

import numberz.Matrix
import numberz.Vector

import org.rosuda.JRI.RList

import tuner.Config
import tuner.DimRanges
import tuner.Grid2D
import tuner.Sampler
import tuner.Table
import tuner.util.Util

case class GpSpecification(
  responseDim:String,
  dimNames:List[String],
  thetas:List[Double],
  alphas:List[Double],
  mean:Double,
  sigma2:Double,
  designMatrix:List[List[Double]],
  responses:List[Double],
  invCorMtx:List[List[Double]]
)

object GpModel {
  def fromJson(json:GpSpecification) = {
    new GpModel(Vector(json.thetas), Vector(json.alphas), 
                json.mean, json.sigma2,
                Matrix.fromColumnMajor(json.designMatrix), 
                Vector(json.responses),
                Matrix.fromColumnMajor(json.invCorMtx),
                json.dimNames, json.responseDim, Config.errorField)
  }
}

// A gp model takes a sampling density and returns 
// a filename from which to read the sampled data
//type Model = Int => String
class GpModel(val thetas:Vector, val alphas:Vector, 
              val mean:Double, val sig2:Double, 
              val design:Matrix, val responses:Vector, 
              val rInverse:Matrix, 
              val dims:List[String], val respDim:String, val errDim:String) {

  def this(thetas:Vector, alphas:Vector, mean:Double, 
           design:Matrix, responses:Vector, 
           rInverse:Matrix, 
           dims:List[String], respDim:String, errDim:String) =
    this(thetas, alphas, mean, 
         (responses.map {_ - mean} dot (rInverse dot responses.map {_ - mean})) / responses.size,
         design, responses, rInverse, dims, respDim, errDim)

  def this(thetas:Vector, alphas:Vector, 
           design:Matrix, responses:Vector, 
           rInverse:Matrix, 
           dims:List[String], respDim:String, errDim:String) =
    this(thetas, alphas, 
         (Vector.ones(responses.size) dot (rInverse dot responses)) /
           (Vector.ones(responses.size) dot (rInverse dot Vector.ones(responses.size))),
         design, responses, rInverse, dims, respDim, errDim)

  // Also precompute rInverse . (responses - mean)
  val corrResponses = rInverse dot (responses map {_ - mean})

  def toJson = {
    GpSpecification(
      respDim, dims, thetas.toList, alphas.toList, mean, sig2,
      design.toColumnMajorList, responses.toList, rInverse.toColumnMajorList)
  }

  def maxGain(range:DimRanges):Float = {
    var mx = Double.MinValue
    Sampler.lhc(range, Config.respHistogramSampleDensity, pt => {
      val (est, err) = runSample(pt)
      val expgain = calcExpectedGain(est, err)
      mx = math.max(mx, expgain)
    })
    mx.toFloat
  }

  // Store the most recently seen max value for the function
  def funcMax:Float = responses.max.toFloat
  def funcMin:Float = responses.min.toFloat

  def theta(dim:String) = thetas(dims.indexOf(dim))

  def sampleSlice(rowDim:(String,(Float,Float)), 
                  colDim:(String,(Float,Float)),
                  slices:List[(String,Float)], 
                  numSamples:Int)
      : ((String, Grid2D), (String, Grid2D), (String, Grid2D)) = {
    
    val arrSlice = Array.fill(dims.length)(0.0)
    val sliceMap = slices.toMap
    dims.zipWithIndex.foreach {case (fld, i) =>
      arrSlice(i) = sliceMap(fld)
    }
    val xDim = dims.indexOf(rowDim._1)
    val yDim = dims.indexOf(colDim._1)

    // generate one matrix from scratch and then copy the rest
    val startTime = System.currentTimeMillis
    val response = Sampler.regularSlice(rowDim, colDim, numSamples)
    val errors = new Grid2D(response.rowIds, response.colIds)
    val gains = new Grid2D(response.rowIds, response.colIds)

    response.rowIds.zipWithIndex.foreach {tmpx =>
      val (xval,x) = tmpx
      response.colIds.zipWithIndex.foreach {tmpy =>
        val (yval, y) = tmpy
        arrSlice(xDim) = xval
        arrSlice(yDim) = yval
        val (est, err) = estimatePoint(Vector(arrSlice))
        response.set(x, y, est.toFloat)
        errors.set(x, y, err.toFloat)
        val expgain = calcExpectedGain(est, err)
        if(!expgain.isNaN) {
          gains.set(x, y, expgain.toFloat)
        } else {
          gains.set(x, y, 0f)
        }
      }
    }

    val endTime = System.currentTimeMillis
    ((respDim, response), 
     (Config.errorField, errors), 
     (Config.gainField, gains))
  }

  def sampleTable(samples:Table) : Table = {
    // verify that the input table has the proper fields
    val sampleFields:Set[String] = samples.fieldNames.toSet
    if(!(dims.toSet subsetOf sampleFields)) {
      val missingFields:Set[String] = dims.toSet -- sampleFields
      throw new tuner.error.NonMatchingParameterException(missingFields toList)
    }

    val outTbl = new Table
    for(r <- 0 until samples.numRows) {
      val tpl = samples.tuple(r)
      val (est, err) = runSample(tpl)
      outTbl.addRow((respDim, est.toFloat)::(errDim, err.toFloat)::tpl.toList)
    }
    outTbl
  }

  def runSample(pt:List[(String, Float)]) : (Double, Double) = 
    runSample(pt.toMap)

  def runSample(pt:Table.Tuple) : (Double, Double) = {
    val xx = dims.map(pt.get(_)).flatten.map(_.toDouble)
    val (est, err) = estimatePoint(Vector(xx))
    //curFuncMax = math.max(est.toFloat, curFuncMax)
    (est, err)
  }

  // Compute the correlation wrt each design point
  private def estimatePoint(point:Vector) : (Double,Double) = {
{'fraction_non_alphanumeric': 0.11662418062636562, 'fraction_numerical': 0.0029133284777858705, 'mean_word_length': 2.93445558739255, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2749213', 'n_tokens_mistral': 3699, 'n_tokens_neox': 3600, 'n_words': 790}