<commit_before>/*
 Copyright The MITRE Corporation 2009-2010.   All rights reserved.
 */

package org.mitre.jcarafe.crf
import scala.collection.mutable.HashMap
import scala.collection.immutable.IntMap
import cern.colt.map.{OpenLongObjectHashMap,AbstractLongIntMap,OpenLongIntHashMap};
import org.mitre.jcarafe.util._

case class ModelMetaData(val ver: String, val date: String, val user: String)

case class ModelAuxiliaries(lex: Option[BloomLexicon],
  wdProps: Option[WordProperties],
  wdScores: Option[WordScores],
  inducedFs: Option[InducedFeatureMap])

abstract class Model(
  val fspec: String, // string format version of feature extraction specification
  val beg: Boolean,
  val lex: Option[BloomLexicon],
  val segSize: Int,
  val labelAlphabet: Alphabet[AbstractLabel],
  val crf: CoreModel) {

  def this(labelAlphabet: Alphabet[AbstractLabel], crf: CoreModel) = this("",false,None,0,labelAlphabet,crf)
  def print(f: java.io.File): Unit
  def fixAlphabet(t: Boolean) = labelAlphabet.fixed_=(t)
}

class MaxEntModel(
  labelAlphabet: Alphabet[AbstractLabel],
  crf: CoreModel,
  val fsetMap: LongAlphabet,
  val inducedMap: Option[InducedFeatureMap] = None) extends Model(labelAlphabet, crf) {
  def print(f: java.io.File) = println("--no printing available--")
}

class StdModel(
  fspec: String, // string format version of feature extraction specification
  beg: Boolean,
  val aux: ModelAuxiliaries,
  segSize: Int,
  labelAlphabet: Alphabet[AbstractLabel],
  crf: CoreModel,
  val fsetMap: OpenLongObjectHashMap) extends Model(fspec, beg, aux.lex, segSize, labelAlphabet, crf) {
  
  def getLabelAlphabet = labelAlphabet

  def this(fs: String, b: Boolean, l: Option[BloomLexicon], wp: Option[WordProperties], ws: Option[WordScores], ifs: Option[InducedFeatureMap],
    ss: Int, lalpha: Alphabet[AbstractLabel], crf: CoreModel, fsmap: OpenLongObjectHashMap) =
    this(fs, b, ModelAuxiliaries(l, wp, ws, ifs), ss, lalpha, crf, fsmap)

  def wdProps = aux.wdProps
  def wdScores = aux.wdScores
  def inducedFs = aux.inducedFs

  class GetFaMap(val a: LongAlphabet) extends cern.colt.function.LongObjectProcedure {
    def apply(fn: Long, v: java.lang.Object) = {
      v match {
        case ft: FeatureType =>
          ft.fdetail foreach { fea => a update (fea.prv, fea.cur, fn) }
        case _ =>
      }
      true
    }
  }

  def deriveFaMap: LongAlphabet = {
    val a = new LongAlphabet
    fsetMap.forEachPair(new GetFaMap(a))
    a
  }

  def adjustParameter(fname: Long, lab: AbstractLabel, fac: Double) = {
    val labId = labelAlphabet(lab)
    val ft =
      fsetMap.get(fname) match {
        case ft: FeatureType => ft.fdetail.foreach { f => if (f.cur == labId) crf.params(f.fid) += fac }
        case _ =>
      }
  }

  def adjustParameter(fname: String, lab: AbstractLabel, fac: Double): Unit = adjustParameter(fname.hashCode, lab, fac)

  def print(mfile: java.io.File) = {
    val f_out = new java.io.OutputStreamWriter(new java.io.BufferedOutputStream(new java.io.FileOutputStream(mfile)), "UTF-8")
    val invLa = labelAlphabet.getInvMap
    f_out.write("Using the following feature specification:\n")
    f_out.write(fspec)
    f_out.write("\n")
    f_out.write("State alphabet\n-------------\n")
    labelAlphabet.mp foreach { case (x, _) => f_out.write(x.toString); f_out.write("\n") }
    f_out.write("-------------\n")
    f_out.flush
    f_out.close
  }
}

class RandomStdModel(
  fspec: String, // string format version of feature extraction specification
  beg: Boolean,
  aux: ModelAuxiliaries,
  segSize: Int,
  labelAlphabet: Alphabet[AbstractLabel],
  crf: CoreModel,
  val faMap: RandomLongAlphabet) extends StdModel(fspec, beg, aux, segSize, labelAlphabet, crf, new OpenLongObjectHashMap) {

  def this(fs: String, b: Boolean, l: Option[BloomLexicon], wp: Option[WordProperties], ws: Option[WordScores], ifs: Option[InducedFeatureMap],
    ss: Int, lalpha: Alphabet[AbstractLabel], crf: CoreModel, faMap: RandomLongAlphabet) =
    this(fs, b, ModelAuxiliaries(l, wp, ws, ifs), ss, lalpha, crf, faMap)
  override def deriveFaMap = faMap
  
  override def adjustParameter(fname: Long, lab: AbstractLabel, fac: Double) = {
    val labelId = labelAlphabet.get(lab).getOrElse(0)
    val fid = faMap.update(labelId,fname)
    crf.params(fid) += fac
  }
}

class NonFactoredModel(fspec: String, lex: Option[BloomLexicon], val wp: Option[WordProperties], ss: Int, crf: CoreModel, val faMap: LongAlphabet, val numStates: Int)
  extends Model(fspec, true, lex, ss, (new Alphabet[AbstractLabel]), crf) {

  class PrintFaMap extends cern.colt.function.LongIntProcedure {
    def apply(f: Long, i: Int) = {
      println("f: " + f + " i: " + i)
      true
    }
  }

  def print(mfile: java.io.File) = {
    val f_out = new java.io.BufferedWriter(new java.io.FileWriter(mfile))
    val invLa = labelAlphabet.getInvMap
    f_out.write("Using the following feature specification:\n")
    f_out.write(fspec)
    f_out.write("\n")
    f_out.write("State alphabet\n")
    labelAlphabet.mp foreach { case (x, _) => f_out.write(x.toString); f_out.write("\n") }
    f_out.write("params:")
    faMap.getUnderlyingMap.forEachPair(new PrintFaMap)
    //faMap foreach {case (fn,ft) => f_out.write("'" + fn + "' [" + ft + "] => " + crf.params(ft) + "\n")}
    f_out.flush
    f_out.close
  }
}

class RandomNonFactoredModel(fspec: String, lex: Option[BloomLexicon], wp: Option[WordProperties], ss: Int, crf: CoreModel, faMap: RandomLongAlphabet, numStates: Int)
  extends NonFactoredModel(fspec, lex, wp, ss, crf, faMap, numStates)


abstract class CoreModelSerializer {
  import scala.collection.mutable.HashMap
  import com.esotericsoftware.kryo.io.{Input => KInput, Output => KOutput}
  import com.twitter.chill.{EmptyScalaKryoInstantiator, AllScalaRegistrar}

  val instantiator = new EmptyScalaKryoInstantiator
  val kryo = instantiator.newKryo
  new AllScalaRegistrar()(kryo)
  kryo.register(classOf[CrfInstance])
  kryo.register(classOf[Alphabet[Long]])
  kryo.register(classOf[MaxEntModel])
  kryo.register(classOf[WordProperties])
  kryo.register(classOf[RandomLongAlphabet])
  kryo.register(classOf[LongAlphabet])
  kryo.register(classOf[BloomLexicon])
  kryo.register(classOf[BloomFilter])  
  kryo.register(classOf[OpenLongIntHashMap])
  kryo.register(classOf[AbstractLongIntMap])

  protected def checkModel(m: Model) = {
    import FastLoops._
    val params = m.crf.params
    var ok = true
    forIndex(params.length) { i =>
      val k = params(i)
      if (k != params(i))
        ok = false
    }
    if (!ok) println("\n\n!!!!WARNING!!!! training resulted in model parameters having value NaN")
  }
}

object InducedFeatureMapProtocol {
  import com.esotericsoftware.kryo.io.{Input => KInput, Output => KOutput}
  import com.twitter.chill.{EmptyScalaKryoInstantiator, AllScalaRegistrar}
  import collection.mutable.HashMap

  val instantiator = new EmptyScalaKryoInstantiator
  val kryo = instantiator.newKryo
  
  def readFMap(kInput: KInput) : HashMap[Long,Array[Double]] = {
    val m = kryo.readObject(kInput, classOf[HashMap[Long,Array[Double]]])
    kInput.close()
    m
  }
  
  def readFMap(f: java.io.File): HashMap[Long,Array[Double]] = {
    val is = new java.io.BufferedInputStream(new java.io.FileInputStream(f))<commit_msg>Added kryo infor for java bitset<commit_after>/*
 Copyright The MITRE Corporation 2009-2010.   All rights reserved.
 */

package org.mitre.jcarafe.crf
import scala.collection.mutable.HashMap
import scala.collection.immutable.IntMap
import cern.colt.map.{OpenLongObjectHashMap,AbstractLongIntMap,OpenLongIntHashMap};
import org.mitre.jcarafe.util._

case class ModelMetaData(val ver: String, val date: String, val user: String)

case class ModelAuxiliaries(lex: Option[BloomLexicon],
  wdProps: Option[WordProperties],
  wdScores: Option[WordScores],
  inducedFs: Option[InducedFeatureMap])

abstract class Model(
  val fspec: String, // string format version of feature extraction specification
  val beg: Boolean,
  val lex: Option[BloomLexicon],
  val segSize: Int,
  val labelAlphabet: Alphabet[AbstractLabel],
  val crf: CoreModel) {

  def this(labelAlphabet: Alphabet[AbstractLabel], crf: CoreModel) = this("",false,None,0,labelAlphabet,crf)
  def print(f: java.io.File): Unit
  def fixAlphabet(t: Boolean) = labelAlphabet.fixed_=(t)
}

class MaxEntModel(
  labelAlphabet: Alphabet[AbstractLabel],
  crf: CoreModel,
  val fsetMap: LongAlphabet,
  val inducedMap: Option[InducedFeatureMap] = None) extends Model(labelAlphabet, crf) {
  def print(f: java.io.File) = println("--no printing available--")
}

class StdModel(
  fspec: String, // string format version of feature extraction specification
  beg: Boolean,
  val aux: ModelAuxiliaries,
  segSize: Int,
  labelAlphabet: Alphabet[AbstractLabel],
  crf: CoreModel,
  val fsetMap: OpenLongObjectHashMap) extends Model(fspec, beg, aux.lex, segSize, labelAlphabet, crf) {
  
  def getLabelAlphabet = labelAlphabet

  def this(fs: String, b: Boolean, l: Option[BloomLexicon], wp: Option[WordProperties], ws: Option[WordScores], ifs: Option[InducedFeatureMap],
    ss: Int, lalpha: Alphabet[AbstractLabel], crf: CoreModel, fsmap: OpenLongObjectHashMap) =
    this(fs, b, ModelAuxiliaries(l, wp, ws, ifs), ss, lalpha, crf, fsmap)

  def wdProps = aux.wdProps
  def wdScores = aux.wdScores
  def inducedFs = aux.inducedFs

  class GetFaMap(val a: LongAlphabet) extends cern.colt.function.LongObjectProcedure {
    def apply(fn: Long, v: java.lang.Object) = {
      v match {
        case ft: FeatureType =>
          ft.fdetail foreach { fea => a update (fea.prv, fea.cur, fn) }
        case _ =>
      }
      true
    }
  }

  def deriveFaMap: LongAlphabet = {
    val a = new LongAlphabet
    fsetMap.forEachPair(new GetFaMap(a))
    a
  }

  def adjustParameter(fname: Long, lab: AbstractLabel, fac: Double) = {
    val labId = labelAlphabet(lab)
    val ft =
      fsetMap.get(fname) match {
        case ft: FeatureType => ft.fdetail.foreach { f => if (f.cur == labId) crf.params(f.fid) += fac }
        case _ =>
      }
  }

  def adjustParameter(fname: String, lab: AbstractLabel, fac: Double): Unit = adjustParameter(fname.hashCode, lab, fac)

  def print(mfile: java.io.File) = {
    val f_out = new java.io.OutputStreamWriter(new java.io.BufferedOutputStream(new java.io.FileOutputStream(mfile)), "UTF-8")
    val invLa = labelAlphabet.getInvMap
    f_out.write("Using the following feature specification:\n")
    f_out.write(fspec)
    f_out.write("\n")
    f_out.write("State alphabet\n-------------\n")
    labelAlphabet.mp foreach { case (x, _) => f_out.write(x.toString); f_out.write("\n") }
    f_out.write("-------------\n")
    f_out.flush
    f_out.close
  }
}

class RandomStdModel(
  fspec: String, // string format version of feature extraction specification
  beg: Boolean,
  aux: ModelAuxiliaries,
  segSize: Int,
  labelAlphabet: Alphabet[AbstractLabel],
  crf: CoreModel,
  val faMap: RandomLongAlphabet) extends StdModel(fspec, beg, aux, segSize, labelAlphabet, crf, new OpenLongObjectHashMap) {

  def this(fs: String, b: Boolean, l: Option[BloomLexicon], wp: Option[WordProperties], ws: Option[WordScores], ifs: Option[InducedFeatureMap],
    ss: Int, lalpha: Alphabet[AbstractLabel], crf: CoreModel, faMap: RandomLongAlphabet) =
    this(fs, b, ModelAuxiliaries(l, wp, ws, ifs), ss, lalpha, crf, faMap)
  override def deriveFaMap = faMap
  
  override def adjustParameter(fname: Long, lab: AbstractLabel, fac: Double) = {
    val labelId = labelAlphabet.get(lab).getOrElse(0)
    val fid = faMap.update(labelId,fname)
    crf.params(fid) += fac
  }
}

class NonFactoredModel(fspec: String, lex: Option[BloomLexicon], val wp: Option[WordProperties], ss: Int, crf: CoreModel, val faMap: LongAlphabet, val numStates: Int)
  extends Model(fspec, true, lex, ss, (new Alphabet[AbstractLabel]), crf) {

  class PrintFaMap extends cern.colt.function.LongIntProcedure {
    def apply(f: Long, i: Int) = {
      println("f: " + f + " i: " + i)
      true
    }
  }

  def print(mfile: java.io.File) = {
    val f_out = new java.io.BufferedWriter(new java.io.FileWriter(mfile))
    val invLa = labelAlphabet.getInvMap
    f_out.write("Using the following feature specification:\n")
    f_out.write(fspec)
    f_out.write("\n")
    f_out.write("State alphabet\n")
    labelAlphabet.mp foreach { case (x, _) => f_out.write(x.toString); f_out.write("\n") }
    f_out.write("params:")
    faMap.getUnderlyingMap.forEachPair(new PrintFaMap)
    //faMap foreach {case (fn,ft) => f_out.write("'" + fn + "' [" + ft + "] => " + crf.params(ft) + "\n")}
    f_out.flush
    f_out.close
  }
}

class RandomNonFactoredModel(fspec: String, lex: Option[BloomLexicon], wp: Option[WordProperties], ss: Int, crf: CoreModel, faMap: RandomLongAlphabet, numStates: Int)
  extends NonFactoredModel(fspec, lex, wp, ss, crf, faMap, numStates)


abstract class CoreModelSerializer {
  import scala.collection.mutable.HashMap
  import com.esotericsoftware.kryo.io.{Input => KInput, Output => KOutput}
  import com.twitter.chill.{EmptyScalaKryoInstantiator, AllScalaRegistrar}

  val instantiator = new EmptyScalaKryoInstantiator
  val kryo = instantiator.newKryo
  new AllScalaRegistrar()(kryo)
  kryo.register(classOf[CrfInstance])
  kryo.register(classOf[Alphabet[Long]])
  kryo.register(classOf[MaxEntModel])
  kryo.register(classOf[WordProperties])
  kryo.register(classOf[RandomLongAlphabet])
  kryo.register(classOf[LongAlphabet])
  kryo.register(classOf[BloomLexicon])
  kryo.register(classOf[BloomFilter])  
  kryo.register(classOf[OpenLongIntHashMap])
  kryo.register(classOf[AbstractLongIntMap])
  kryo.register(classOf[java.util.BitSet])

  protected def checkModel(m: Model) = {
    import FastLoops._
    val params = m.crf.params
    var ok = true
    forIndex(params.length) { i =>
      val k = params(i)
      if (k != params(i))
        ok = false
    }
    if (!ok) println("\n\n!!!!WARNING!!!! training resulted in model parameters having value NaN")
  }
}

object InducedFeatureMapProtocol {
  import com.esotericsoftware.kryo.io.{Input => KInput, Output => KOutput}
  import com.twitter.chill.{EmptyScalaKryoInstantiator, AllScalaRegistrar}
  import collection.mutable.HashMap

  val instantiator = new EmptyScalaKryoInstantiator
  val kryo = instantiator.newKryo
  
  def readFMap(kInput: KInput) : HashMap[Long,Array[Double]] = {
    val m = kryo.readObject(kInput, classOf[HashMap[Long,Array[Double]]])
    kInput.close()
    m
  }
  
  def readFMap(f: java.io.File): HashMap[Long,Array[Double]] = {
    val is = new java.io.BufferedInputStream(new java.io.FileInputStream(f))
{'fraction_non_alphanumeric': 0.1256490844493031, 'fraction_numerical': 0.0015031429352282044, 'mean_word_length': 4.212606837606837, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6613053', 'n_tokens_mistral': 5194, 'n_tokens_neox': 5067, 'n_words': 1149}