<commit_before>/*
 Copyright The MITRE Corporation 2009-2010.   All rights reserved.
 */

package org.mitre.jcarafe.crf
import collection.mutable.HashSet
import collection.mutable.HashMap
import org.mitre.jcarafe.util.Options

trait Trainable[T] {
  val lambdas: Array[Double]
  val numParams: Int

  def getGradient(instAccessor: AccessSeq[T]): Option[Double]

  def initialize(): Unit

  def getCoreModel(): CoreModel

  def train(accessSeq: AccessSeq[T], num: Int = 200, mi: Option[(CoreModel, Int) => Unit] = None): CoreModel

}

trait SequenceTrainable extends Trainable[AbstractInstance]

trait DenseTrainable[T] extends Trainable[T] {
  val gradient: Array[Double]
}

trait SparseTrainable[T] extends Trainable[T] {
  class DoubleCell(var g: Double, var e: Double)
  val maxEpochs: Int
  val batchSize: Int
  val C: Double
  val initialLearningRate: Double
  val momentum: Double
  val eta: Double
  val etas: Array[Double]
  var curPos: Int
  val gradient: HashMap[Int, DoubleCell]
  val quiet: Boolean
  val periodSize: Int
  var numGradIssues: Int
  val pAlpha: Double
}

class CoreModel(val params: Array[Double], val nfs: Int, val nls: Int, val nNfs: Int = 0, val nGates: Int = 0) {
  def print() = {
    println("NLS: => " + nls)
    println("NFS: => " + nfs)
    println("Parameters")
    for (i <- 0 until params.length) {
      println("  " + i + " => " + params(i))
    }
  }
}

trait AccessSeq[T] {
  def apply(i: Int): Seq[T]
  def length: Int
  def repermute(): Unit
  def splitAccessor(n: Int): Seq[AccessSeq[T]]
  def getAllSplits(n: Int) : Seq[(AccessSeq[T],AccessSeq[T])]
  def accessSingleInstance(i: Int): T
}

//class CachedAccessSeq(fs: Array[String]) extends AccessSeq
class MemoryAccessSeq(iseqs: Seq[InstanceSequence], seed: Option[Int] = None) extends AccessSeq[AbstractInstance] {
  var seqs = iseqs //permuteSeq(iseqs.filter{s => s.length > 0})
  def getSeqs = iseqs
  def apply(i: Int) = seqs(i).iseq
  def length = seqs.length
  val (randomSelector) = {
    seed match {
      case Some(x) => new util.Random(x)
      case None => new util.Random()
    }
  }

  def accessSingleInstance(i: Int) = apply(i)(0)

  private def permuteArray[A](a: Array[A]) = {
    var i = a.length - 1
    while (i >= 0) {
      val j = randomSelector.nextInt(i + 1)
      val t = a(i)
      a(i) = a(j)
      a(j) = t
      i -= 1
    }
  }

  private def permuteSeq(s: Seq[InstanceSequence]): IndexedSeq[InstanceSequence] = {
    val nseqs = s.toArray
    permuteArray(nseqs)
    nseqs.toIndexedSeq
  }
  def repermute() = seqs = permuteSeq(seqs)

  def splitAccessor(n: Int): Seq[MemoryAccessSeq] = {
    val ns = if ((seqs.length % n) == 0) seqs.length / n else (seqs.length / n) + 1
    for (j <- 0 until n) yield {
      new MemoryAccessSeq(seqs.slice(j * ns, seqs.length min ((j + 1) * ns)))
    }
  }
  def getAllSplits(n: Int) : Seq[(MemoryAccessSeq, MemoryAccessSeq)] = {
    val folds = splitAccessor(n)
    for (i <- 0 until n) yield {
      val tst = folds(i)
      val instBuf = new collection.mutable.ArrayBuffer[InstanceSequence]
      for (j <- 0 until n; if (j != i)) {
        instBuf ++= folds(j).getSeqs
      }
      (new MemoryAccessSeq(instBuf.toSeq), tst)
    }
  }
}

/**
 * Top abstract class representing the core elements and functionality of a sequence-structured
 * Conditional Random Field.  A <code>Crf</code> object is created after feature extraction
 * has occurred.
 * @param lambdas   Parameter (lambda) vector
 * @param nls       Number of labels/states
 * @param nfs       Number of features
 * @param segSize   The size of segments.  Sizes greater than 1 indicate the model is a semi-CRF
 * @param gPrior    The Gaussian prior variance used as a regularizer
 * @param nNfs      Number of neural gate input features (for NeuralCrf)
 * @param nGates    Number of neural gates per label (for NeuralCrf)
 */
abstract class Crf(val lambdas: Array[Double], val nls: Int, val nfs: Int, val segSize: Int, val gPrior: Double, val nNfs: Int, val nGates: Int)
  extends Trainable[AbstractInstance] {

  val numParams = nfs

  def this(nls: Int, nfs: Int, gPrior: Double) = this(Array.fill(nfs)(0.0), nls, nfs, 1, gPrior, 0, 0)
  def this(nls: Int, nfs: Int, segSize: Int) = this(Array.fill(nfs)(0.0), nls, nfs, segSize, 10.0, 0, 0)
  def this(nls: Int, nfs: Int) = this(nls, nfs, 1)

  def getCoreModel() = new CoreModel(lambdas, numParams, nls, nNfs, nGates)
  
  def resetParameters() : Unit = {
    val l = lambdas.length
    var i = 0; while (i < l) {
      lambdas(i) = 0.0
      i += 1
    }
  }

  /**
   * When set to <code>true</code>, the Crf will allow the state-space to be dynamically
   * sized - i.e. the number of states is dependent on each sequence
   */
  var adjustible = false

  if ((nls < 2) || (nfs < 2)) {
    System.err.println("\n\nEnsure the tagset/label set is properly defined!!")
    throw new RuntimeException("Crf requires 2 or more states/features")
  }
  /**
   * A matrix type as an array of arrays of <code>Double</code>s
   */
  type Matrix = Array[Array[Double]]

  /**
   * A tensor object as an array of <code>Matrix</code> objects
   */
  type Tensor = Array[Matrix]

  /**
   * The value of the inverse square of the Gaussian prior
   */
  val invSigSqr = 1.0 / (gPrior * gPrior)
<commit_msg>Modified SPLIT tokenizer augmenter to do basic recursive greedy sub-token splitting<commit_after>/*
 Copyright The MITRE Corporation 2009-2010.   All rights reserved.
 */

package org.mitre.jcarafe.crf
import collection.mutable.HashSet
import collection.mutable.HashMap
import org.mitre.jcarafe.util.Options

trait Trainable[T] {
  val lambdas: Array[Double]
  val numParams: Int

  def getGradient(instAccessor: AccessSeq[T]): Option[Double]

  def initialize(): Unit

  def getCoreModel(): CoreModel

  def train(accessSeq: AccessSeq[T], num: Int = 200, mi: Option[(CoreModel, Int) => Unit] = None): CoreModel

}

trait SequenceTrainable extends Trainable[AbstractInstance]

trait DenseTrainable[T] extends Trainable[T] {
  val gradient: Array[Double]
}

trait SparseTrainable[T] extends Trainable[T] {
  class DoubleCell(var g: Double, var e: Double)
  val maxEpochs: Int
  val batchSize: Int
  val C: Double
  val initialLearningRate: Double
  val momentum: Double
  val eta: Double
  val etas: Array[Double]
  var curPos: Int
  val gradient: HashMap[Int, DoubleCell]
  val quiet: Boolean
  val periodSize: Int
  var numGradIssues: Int
  val pAlpha: Double
}

class CoreModel(val params: Array[Double], val nfs: Int, val nls: Int, val nNfs: Int = 0, val nGates: Int = 0) {
  def print() = {
    println("NLS: => " + nls)
    println("NFS: => " + nfs)
    println("Parameters")
    for (i <- 0 until params.length) {
      println("  " + i + " => " + params(i))
    }
  }
}

trait AccessSeq[T] {
  def apply(i: Int): Seq[T]
  def length: Int
  def repermute(): Unit
  def splitAccessor(n: Int): Seq[AccessSeq[T]]
  def getAllSplits(n: Int) : Seq[(AccessSeq[T],AccessSeq[T])]
  def accessSingleInstance(i: Int): T
}

//class CachedAccessSeq(fs: Array[String]) extends AccessSeq
class MemoryAccessSeq(iseqs: Seq[InstanceSequence], seed: Option[Int] = None) extends AccessSeq[AbstractInstance] {
  var seqs = iseqs //permuteSeq(iseqs.filter{s => s.length > 0})
  def getSeqs = iseqs
  def apply(i: Int) = seqs(i).iseq
  def length = seqs.length
  val (randomSelector) = {
    seed match {
      case Some(x) => new util.Random(x)
      case None => new util.Random()
    }
  }

  def accessSingleInstance(i: Int) = apply(i)(0)

  private def permuteArray[A](a: Array[A]) = {
    var i = a.length - 1
    while (i >= 0) {
      val j = randomSelector.nextInt(i + 1)
      val t = a(i)
      a(i) = a(j)
      a(j) = t
      i -= 1
    }
  }

  private def permuteSeq(s: Seq[InstanceSequence]): IndexedSeq[InstanceSequence] = {
    val nseqs = s.toArray
    permuteArray(nseqs)
    nseqs.toIndexedSeq
  }
  def repermute() = seqs = permuteSeq(seqs)

  def splitAccessor(n: Int): Seq[MemoryAccessSeq] = {
    val ns = if ((seqs.length % n) == 0) seqs.length / n else (seqs.length / n) + 1
    for (j <- 0 until n) yield {
      new MemoryAccessSeq(seqs.slice(j * ns, seqs.length min ((j + 1) * ns)))
    }
  }
  def getAllSplits(n: Int) : Seq[(MemoryAccessSeq, MemoryAccessSeq)] = {
    val folds = splitAccessor(n)
    for (i <- 0 until n) yield {
      val tst = folds(i)
      val instBuf = new collection.mutable.ArrayBuffer[InstanceSequence]
      for (j <- 0 until n; if (j != i)) {
        instBuf ++= folds(j).getSeqs
      }
      (new MemoryAccessSeq(instBuf.toSeq), tst)
    }
  }
}

/**
 * Top abstract class representing the core elements and functionality of a sequence-structured
 * Conditional Random Field.  A <code>Crf</code> object is created after feature extraction
 * has occurred.
 * @param lambdas   Parameter (lambda) vector
 * @param nls       Number of labels/states
 * @param nfs       Number of features
 * @param segSize   The size of segments.  Sizes greater than 1 indicate the model is a semi-CRF
 * @param gPrior    The Gaussian prior variance used as a regularizer
 * @param nNfs      Number of neural gate input features (for NeuralCrf)
 * @param nGates    Number of neural gates per label (for NeuralCrf)
 */
abstract class Crf(val lambdas: Array[Double], val nls: Int, val nfs: Int, val segSize: Int, val gPrior: Double, val nNfs: Int, val nGates: Int)
  extends Trainable[AbstractInstance] {

  val numParams = nfs

  def this(nls: Int, nfs: Int, gPrior: Double) = this(Array.fill(nfs)(0.0), nls, nfs, 1, gPrior, 0, 0)
  def this(nls: Int, nfs: Int, segSize: Int) = this(Array.fill(nfs)(0.0), nls, nfs, segSize, 10.0, 0, 0)
  def this(nls: Int, nfs: Int) = this(nls, nfs, 1)

  def getCoreModel() = new CoreModel(lambdas, numParams, nls, nNfs, nGates)
  
  def resetParameters() : Unit = {
    val l = lambdas.length
    var i = 0; while (i < l) {
      lambdas(i) = 0.0
      i += 1
    }
  }

  /**
   * When set to <code>true</code>, the Crf will allow the state-space to be dynamically
   * sized - i.e. the number of states is dependent on each sequence
   */
  var adjustible = false

  if ((nls < 2) || (nfs < 2) && segSize < 2) {
    System.err.println("\n\nEnsure the tagset/label set is properly defined!!")
    throw new RuntimeException("Crf requires 2 or more states/features")
  }
  /**
   * A matrix type as an array of arrays of <code>Double</code>s
   */
  type Matrix = Array[Array[Double]]

  /**
   * A tensor object as an array of <code>Matrix</code> objects
   */
  type Tensor = Array[Matrix]

  /**
   * The value of the inverse square of the Gaussian prior
   */
  val invSigSqr = 1.0 / (gPrior * gPrior)

{'fraction_non_alphanumeric': 0.12472997088381704, 'fraction_numerical': 0.009298393913778529, 'mean_word_length': 3.252396166134185, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1215767', 'n_tokens_mistral': 3846, 'n_tokens_neox': 3674, 'n_words': 1160}