<commit_before>package ch.ethz.dalab.dissolve.diagnostics

import java.nio.file.Paths
import org.scalatest.FlatSpec
import org.scalatest.Inside
import org.scalatest.Inspectors
import org.scalatest.Matchers
import org.scalatest.OptionValues
import breeze.linalg.DenseVector
import breeze.linalg.Matrix
import breeze.linalg.Vector
import ch.ethz.dalab.dissolve.classification.StructSVMModel
import ch.ethz.dalab.dissolve.examples.chain.ChainDemo
import ch.ethz.dalab.dissolve.examples.imageseg.ImageSegmentationAdv
import ch.ethz.dalab.dissolve.examples.imageseg.ImageSegmentationAdvUtils
import ch.ethz.dalab.dissolve.examples.imageseg.QuantizedImage
import ch.ethz.dalab.dissolve.examples.imageseg.QuantizedLabel
import ch.ethz.dalab.dissolve.optimization.DissolveFunctions
import ch.ethz.dalab.dissolve.regression.LabeledObject
import breeze.linalg.max

object ImageTestAdapter {
  type X = QuantizedImage
  type Y = QuantizedLabel

  /**
   * Dissolve Functions
   */
  val dissolveFunctions: DissolveFunctions[X, Y] = ImageSegmentationAdv
  /**
   * Some Data
   */
  val data = {
    val dataDir = "../data/generated/msrc"
    val trainFilePath = Paths.get(dataDir, "Train.txt")
    val trainDataSeq = ImageSegmentationAdvUtils.loadData(dataDir, trainFilePath, limit = 50)

    trainDataSeq
  }
  /**
   * A dummy model
   */
  val lo = data(0)
  val numd = ImageSegmentationAdv.featureFn(lo.pattern, lo.label).size
  val model: StructSVMModel[X, Y] =
    new StructSVMModel[X, Y](DenseVector.zeros(numd), 0.0,
      DenseVector.zeros(numd), dissolveFunctions, 1)

  def perturb(y: Y, degree: Double = 0.1): Y = {
    val d = y.labels.size
    val numSwaps = max(1, (degree * d).toInt)

    for (swapNo <- 0 until numSwaps) {
      // Swap two random values in y
      val (i, j) = (scala.util.Random.nextInt(d), scala.util.Random.nextInt(d))
      val temp = y.labels(i)
      y.labels(i) = y.labels(j)
      y.labels(j) = temp
    }

    y

  }
}

object ChainTestAdapter {
  type X = Matrix[Double]
  type Y = Vector[Double]

  /**
   * Dissolve Functions
   */
  val dissolveFunctions: DissolveFunctions[X, Y] = ChainDemo
  /**
   * Some Data
   */
  val data = {
    val dataDir = "../data/generated"
    val trainDataSeq: Vector[LabeledObject[Matrix[Double], Vector[Double]]] =
      ChainDemo.loadData(dataDir + "/patterns_train.csv",
        dataDir + "/labels_train.csv",
        dataDir + "/folds_train.csv")

    trainDataSeq.toArray
  }
  /**
   * A dummy model
   */
  val lo = data(0)
  val numd = ChainDemo.featureFn(lo.pattern, lo.label).size
  val model: StructSVMModel[X, Y] =
    new StructSVMModel[X, Y](DenseVector.zeros(numd), 0.0,
      DenseVector.zeros(numd), dissolveFunctions, 1)

  /**
   * Perturb
   * Return a compatible perturbed Y
   * Higher the degree, more perturbed y is
   *
   * This function perturbs `degree` of the values by swapping
   */
  def perturb(y: Y, degree: Double = 0.1): Y = {
    val d = y.size
    val numSwaps = max(1, (degree * d).toInt)

    for (swapNo <- 0 until numSwaps) {
      // Swap two random values in y
      val (i, j) = (scala.util.Random.nextInt(d), scala.util.Random.nextInt(d))
      val temp = y(i)
      y(i) = y(j)
      y(j) = temp
    }

    y

  }
}

/**
 * @author torekond
 */
abstract class UnitSpec extends FlatSpec with Matchers with OptionValues with Inside with Inspectors {

  val DissolveAdapter = ChainTestAdapter

  type X = DissolveAdapter.X
  type Y = DissolveAdapter.Y

  val dissolveFunctions = DissolveAdapter.dissolveFunctions
  val data = DissolveAdapter.data
  val model = DissolveAdapter.model

  /**
   * Helper functions
   */
  def perturb = DissolveAdapter.perturb _

  // Joint Feature Map
  def phi = dissolveFunctions.featureFn _
  def delta = dissolveFunctions.lossFn _
  def maxoracle = dissolveFunctions.oracleFn _

  def psi(lo: LabeledObject[X, Y], ymap: Y) =
    phi(lo.pattern, lo.label) - phi(lo.pattern, ymap)

  def F(x: X, y: Y, w: Vector[Double]) =
    w dot phi(x, y)
  def deltaF(lo: LabeledObject[X, Y], ystar: Y, w: Vector[Double]) =
    w dot psi(lo, ystar)

}<commit_msg>Updating UnitSpec to ImageSeg<commit_after>package ch.ethz.dalab.dissolve.diagnostics

import java.nio.file.Paths
import org.scalatest.FlatSpec
import org.scalatest.Inside
import org.scalatest.Inspectors
import org.scalatest.Matchers
import org.scalatest.OptionValues
import breeze.linalg.DenseVector
import breeze.linalg.Matrix
import breeze.linalg.Vector
import ch.ethz.dalab.dissolve.classification.StructSVMModel
import ch.ethz.dalab.dissolve.examples.chain.ChainDemo
import ch.ethz.dalab.dissolve.examples.imageseg.ImageSeg
import ch.ethz.dalab.dissolve.examples.imageseg.ImageSegUtils
import ch.ethz.dalab.dissolve.examples.imageseg.QuantizedImage
import ch.ethz.dalab.dissolve.examples.imageseg.QuantizedLabel
import ch.ethz.dalab.dissolve.optimization.DissolveFunctions
import ch.ethz.dalab.dissolve.regression.LabeledObject
import breeze.linalg.max

object ImageTestAdapter {
  type X = QuantizedImage
  type Y = QuantizedLabel

  /**
   * Dissolve Functions
   */
  val dissolveFunctions: DissolveFunctions[X, Y] = ImageSeg
  /**
   * Some Data
   */
  val data = {
    val dataDir = "../data/generated/msrc"
    val trainFilePath = Paths.get(dataDir, "Train.txt")
    val trainDataSeq = ImageSegUtils.loadData(dataDir, trainFilePath, limit = 50)

    trainDataSeq
  }
  /**
   * A dummy model
   */
  val lo = data(0)
  val numd = ImageSeg.featureFn(lo.pattern, lo.label).size
  val model: StructSVMModel[X, Y] =
    new StructSVMModel[X, Y](DenseVector.zeros(numd), 0.0,
      DenseVector.zeros(numd), dissolveFunctions, 1)

  def perturb(y: Y, degree: Double = 0.1): Y = {
    val d = y.labels.size
    val numSwaps = max(1, (degree * d).toInt)

    for (swapNo <- 0 until numSwaps) {
      // Swap two random values in y
      val (i, j) = (scala.util.Random.nextInt(d), scala.util.Random.nextInt(d))
      val temp = y.labels(i)
      y.labels(i) = y.labels(j)
      y.labels(j) = temp
    }

    y

  }
}

object ChainTestAdapter {
  type X = Matrix[Double]
  type Y = Vector[Double]

  /**
   * Dissolve Functions
   */
  val dissolveFunctions: DissolveFunctions[X, Y] = ChainDemo
  /**
   * Some Data
   */
  val data = {
    val dataDir = "../data/generated"
    val trainDataSeq: Vector[LabeledObject[Matrix[Double], Vector[Double]]] =
      ChainDemo.loadData(dataDir + "/patterns_train.csv",
        dataDir + "/labels_train.csv",
        dataDir + "/folds_train.csv")

    trainDataSeq.toArray
  }
  /**
   * A dummy model
   */
  val lo = data(0)
  val numd = ChainDemo.featureFn(lo.pattern, lo.label).size
  val model: StructSVMModel[X, Y] =
    new StructSVMModel[X, Y](DenseVector.zeros(numd), 0.0,
      DenseVector.zeros(numd), dissolveFunctions, 1)

  /**
   * Perturb
   * Return a compatible perturbed Y
   * Higher the degree, more perturbed y is
   *
   * This function perturbs `degree` of the values by swapping
   */
  def perturb(y: Y, degree: Double = 0.1): Y = {
    val d = y.size
    val numSwaps = max(1, (degree * d).toInt)

    for (swapNo <- 0 until numSwaps) {
      // Swap two random values in y
      val (i, j) = (scala.util.Random.nextInt(d), scala.util.Random.nextInt(d))
      val temp = y(i)
      y(i) = y(j)
      y(j) = temp
    }

    y

  }
}

/**
 * @author torekond
 */
abstract class UnitSpec extends FlatSpec with Matchers with OptionValues with Inside with Inspectors {

  val DissolveAdapter = ImageTestAdapter

  type X = DissolveAdapter.X
  type Y = DissolveAdapter.Y

  val dissolveFunctions = DissolveAdapter.dissolveFunctions
  val data = DissolveAdapter.data
  val model = DissolveAdapter.model

  /**
   * Helper functions
   */
  def perturb = DissolveAdapter.perturb _

  // Joint Feature Map
  def phi = dissolveFunctions.featureFn _
  def delta = dissolveFunctions.lossFn _
  def maxoracle = dissolveFunctions.oracleFn _

  def psi(lo: LabeledObject[X, Y], ymap: Y) =
    phi(lo.pattern, lo.label) - phi(lo.pattern, ymap)

  def F(x: X, y: Y, w: Vector[Double]) =
    w dot phi(x, y)
  def deltaF(lo: LabeledObject[X, Y], ystar: Y, w: Vector[Double]) =
    w dot psi(lo, ystar)

}
{'fraction_non_alphanumeric': 0.11743074282912479, 'fraction_numerical': 0.004412846285854376, 'mean_word_length': 3.245057232049948, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4674457', 'n_tokens_mistral': 2989, 'n_tokens_neox': 2815, 'n_words': 661}