<commit_before>/*
 Copyright The MITRE Corporation 2009-2010.   All rights reserved.
 */

package org.mitre.jcarafe.crf

import scala.collection.mutable.ListBuffer
import java.io.File
import org.mitre.jcarafe.util._
import sbinary._

abstract class InstanceSequence(val st: Int, val en: Int) {
  var seqPosteriorProbability = 0.0
  def iseq: Seq[AbstractInstance]
  def length: Int
  def print() =
    iseq foreach { ai =>
      println("l => " + ai.label + " vecs: ")
      ai.getCompVec foreach { v =>
        v foreach { x => println("fid=> " + x.fid + " v = " + x.value) }
      }
    }
}

class MemoryInstanceSequence(val internalSeq: Seq[AbstractInstance], st: Int, en: Int) extends InstanceSequence(st, en) {
  def this(iseq: Seq[AbstractInstance]) = this(iseq, -1, -1)
  def length = iseq.length
  def iseq = internalSeq
}

abstract class DiskInstanceSequence(val fp: java.io.File, st: Int, en: Int, val len: Int) extends InstanceSequence(st, en) {

  def iseq: Seq[AbstractInstance]
  def length: Int = len
}


class FactoredCachedSourceSequence[T](val sGen: TrainingSeqGen[T], val src: SourceSequence[T], st: Int, en: Int) extends InstanceSequence(st, en) {
  def iseq: Seq[AbstractInstance] = sGen.extractFeaturesDirect(src)
  override lazy val length = iseq.length
}

class NonFactoredCachedSourceSequence[T](val sGen: SeqGen[T], val src: SourceSequence[T], st: Int, en: Int) extends InstanceSequence(st, en) {

  def iseq: Seq[AbstractInstance] = sGen.extractFeatures(src).iseq
  def length: Int = iseq.length
}

object InstSeq {
  import InstanceSerializations._
  import com.twitter.chill.{ AllScalaRegistrar, EmptyScalaKryoInstantiator }
  import com.esotericsoftware.kryo.io.{ Output, Input }
  var icnt = 0

  val instantiator = new EmptyScalaKryoInstantiator
  val kryo = instantiator.newKryo
  kryo.register(classOf[CrfInstance])
  kryo.register(classOf[SourceSequence[String]])
  kryo.register(classOf[NonFactoredCrfInstance])
  
  class NonFactoredCrfDiskInstanceSequence(fp: java.io.File, st: Int, en: Int, ln: Int) extends DiskInstanceSequence(fp, st, en, ln) {
    val kInput = new Input(new java.io.BufferedInputStream(new java.io.FileInputStream(fp)))
    def iseq: Seq[AbstractInstance] = {
      kryo.readObject(kInput, classOf[Seq[NonFactoredCrfInstance]])
    }
  }

  class RawInstanceSequenceStringObs(val sGen: TrainingSeqGen[String], fp: java.io.File, st: Int, en: Int, ln: Int) extends DiskInstanceSequence(fp, st, en, ln) {
    // serialize just the plain 
    val kInput = new Input(new java.io.BufferedInputStream(new java.io.FileInputStream(fp)))
    def iseq: Seq[AbstractInstance] = {
      //val src = sbinary.Operations.fromFile[SourceSequence[String]](fp)
      val src = kryo.readObject(kInput, classOf[SourceSequence[String]])
      sGen.extractFeaturesDirect(src)
    }
    override lazy val length = iseq.length
  }

  class CrfDiskInstanceSequence(fp: java.io.File, st: Int, en: Int, ln: Int) extends DiskInstanceSequence(fp, st, en, ln) {
    val is = new java.io.BufferedInputStream(new java.io.FileInputStream(fp))
    val kInput = new Input(is)

    def iseq: Seq[AbstractInstance] = {
      //val ss = sbinary.Operations.fromFile[Seq[CrfInstance]](fp)
      val ss = kryo.readObject(kInput, classOf[Seq[CrfInstance]])
      ss

    }
  }

  def serializeSourceSeqToFile(ss: SourceSequence[String], f: java.io.File): Unit = {
    val os = new java.io.BufferedOutputStream(new java.io.FileOutputStream(f))
    val kOutput = new Output(os)
    kryo.writeObject(kOutput, ss)
    kOutput.close
    os.close
  }
  
  def serializeNonFactoredToFile(ss: Seq[NonFactoredCrfInstance], f: java.io.File) = {
    val os = new java.io.BufferedOutputStream(new java.io.FileOutputStream(f))
    val kOutput = new Output(os)
    kryo.writeObject(kOutput, ss)
    kOutput.close
    os.close
  }
  
  /*
   * This method creates a new InstanceSequence, RawInstanceSequenceStringObs that lazily re-computes the feature vector for the instance.
   * WARNING: This method only works for observation sequences parameterized with type java.lang.String since it 
   * must serialize the instance to disk with a specific type.
   */
  def apply[T](sg: TrainingSeqGen[T], ss: SourceSequence[T], st: Int, en: Int)(implicit m: Manifest[T]): InstanceSequence = {
    CrfInstance.diskCache match {
      case Some(filePath) =>
        val ofile = new java.io.File(filePath + "/" + icnt)
        icnt += 1
        if (m.toString equals "java.lang.String") { // should be a better way to set this up...
          val ssA = ss.asInstanceOf[SourceSequence[String]]
          val sgA = sg.asInstanceOf[TrainingSeqGen[String]]
          //sbinary.Operations.toFile[SourceSequence[String]](ssA)(ofile)
          serializeSourceSeqToFile(ssA, ofile)
          new RawInstanceSequenceStringObs(sgA, ofile, st, en, ss.length)
        } else throw new RuntimeException("Disk caching only allowed with SourceSequence[T] where T = String")
      case None => new FactoredCachedSourceSequence(sg, ss, st, en)
    }
  }
  def apply(s: Seq[AbstractInstance], st: Int, en: Int): InstanceSequence = {
    CrfInstance.diskCache match {
      case Some(filePath) =>
        val ofile = new java.io.File(filePath + "/" + icnt)
        icnt += 1
        s match {
          case s: Seq[NonFactoredCrfInstance] =>
            serializeNonFactoredToFile(s, ofile)
            new NonFactoredCrfDiskInstanceSequence(ofile, st, en, s.length)
        }
      case None => new MemoryInstanceSequence(s, st, en)
    }
  }
  def apply(s: Seq[AbstractInstance]): InstanceSequence = apply(s, -1, -1)
  def apply[T](sg: SeqGen[T], s: SourceSequence[T]) = {
    new NonFactoredCachedSourceSequence(sg, s, -1, -1)
  }
}

class SourceSequence[Obs](val seq: Seq[ObsSource[Obs]], val parentSeq: Option[SourceSequence[Obs]], val st: Int, val en: Int) {
  def this(s: Seq[ObsSource[Obs]], par: Option[SourceSequence[Obs]]) = this(s, par, -1, -1)
  def this(s: Seq[ObsSource[Obs]]) = this(s, None)
  def this(s: Seq[ObsSource[Obs]], st: Int, en: Int) = this(s, None, st, en)
  def this(s: Seq[ObsSource[Obs]], par: SourceSequence[Obs]) = this(s, Some(par))

  def getInfo(k: String): Option[String] = None // subclasses can store and retrieve info here
  def apply(i: Int) = seq(i)
  def length = seq.length<commit_msg>Fixed issue with too many open output streams for cached instances<commit_after>/*
 Copyright The MITRE Corporation 2009-2010.   All rights reserved.
 */

package org.mitre.jcarafe.crf

import scala.collection.mutable.ListBuffer
import java.io.File
import org.mitre.jcarafe.util._
import sbinary._

abstract class InstanceSequence(val st: Int, val en: Int) {
  var seqPosteriorProbability = 0.0
  def iseq: Seq[AbstractInstance]
  def length: Int
  def print() =
    iseq foreach { ai =>
      println("l => " + ai.label + " vecs: ")
      ai.getCompVec foreach { v =>
        v foreach { x => println("fid=> " + x.fid + " v = " + x.value) }
      }
    }
}

class MemoryInstanceSequence(val internalSeq: Seq[AbstractInstance], st: Int, en: Int) extends InstanceSequence(st, en) {
  def this(iseq: Seq[AbstractInstance]) = this(iseq, -1, -1)
  def length = iseq.length
  def iseq = internalSeq
}

abstract class DiskInstanceSequence(val fp: java.io.File, st: Int, en: Int, val len: Int) extends InstanceSequence(st, en) {

  def iseq: Seq[AbstractInstance]
  def length: Int = len
}


class FactoredCachedSourceSequence[T](val sGen: TrainingSeqGen[T], val src: SourceSequence[T], st: Int, en: Int) extends InstanceSequence(st, en) {
  def iseq: Seq[AbstractInstance] = sGen.extractFeaturesDirect(src)
  override lazy val length = iseq.length
}

class NonFactoredCachedSourceSequence[T](val sGen: SeqGen[T], val src: SourceSequence[T], st: Int, en: Int) extends InstanceSequence(st, en) {

  def iseq: Seq[AbstractInstance] = sGen.extractFeatures(src).iseq
  def length: Int = iseq.length
}

object InstSeq {
  import InstanceSerializations._
  import com.twitter.chill.{ AllScalaRegistrar, EmptyScalaKryoInstantiator }
  import com.esotericsoftware.kryo.io.{ Output, Input }
  var icnt = 0

  val instantiator = new EmptyScalaKryoInstantiator
  val kryo = instantiator.newKryo
  kryo.register(classOf[CrfInstance])
  kryo.register(classOf[SourceSequence[String]])
  kryo.register(classOf[NonFactoredCrfInstance])
  kryo.register(classOf[Seq[NonFactoredCrfInstance]])
  kryo.register(classOf[Seq[CrfInstance]])
  
  class NonFactoredCrfDiskInstanceSequence(fp: java.io.File, st: Int, en: Int, ln: Int) extends DiskInstanceSequence(fp, st, en, ln) {
    
    def iseq: Seq[AbstractInstance] = {
      val is = new java.io.BufferedInputStream(new java.io.FileInputStream(fp))
      val kInput = new Input(is)
      val o = kryo.readObject(kInput, classOf[Seq[NonFactoredCrfInstance]])
      is.close()
      kInput.close()
      o
    }
  }

  class RawInstanceSequenceStringObs(val sGen: TrainingSeqGen[String], fp: java.io.File, st: Int, en: Int, ln: Int) extends DiskInstanceSequence(fp, st, en, ln) {
    // serialize just the plain 
    def iseq: Seq[AbstractInstance] = {
      //val src = sbinary.Operations.fromFile[SourceSequence[String]](fp)
      val kInput = new Input(new java.io.BufferedInputStream(new java.io.FileInputStream(fp)))    
      val src = kryo.readObject(kInput, classOf[SourceSequence[String]])
      kInput.close
      sGen.extractFeaturesDirect(src)
    }
    override lazy val length = iseq.length
  }

  class CrfDiskInstanceSequence(fp: java.io.File, st: Int, en: Int, ln: Int) extends DiskInstanceSequence(fp, st, en, ln) {
    
    def iseq: Seq[AbstractInstance] = {
      val is = new java.io.BufferedInputStream(new java.io.FileInputStream(fp))
      //val ss = sbinary.Operations.fromFile[Seq[CrfInstance]](fp)
      val kInput = new Input(is)
      val ss = kryo.readObject(kInput, classOf[Seq[CrfInstance]])
      kInput.close()
      ss
    }
  }
    

  def serializeSourceSeqToFile(ss: SourceSequence[String], f: java.io.File): Unit = {
    val os = new java.io.BufferedOutputStream(new java.io.FileOutputStream(f))
    val kOutput = new Output(os)
    kryo.writeObject(kOutput, ss)
    kOutput.close
    os.close
  }
  
  def serializeNonFactoredToFile(ss: Seq[NonFactoredCrfInstance], f: java.io.File) = {
    val os = new java.io.BufferedOutputStream(new java.io.FileOutputStream(f))
    val kOutput = new Output(os)
    kryo.writeObject(kOutput, ss)
    kOutput.close
    os.close
  }

  def serializeCrfInstanceToFile(ss: Seq[CrfInstance], f: java.io.File) = {
    val os = new java.io.BufferedOutputStream(new java.io.FileOutputStream(f))
    val kOutput = new Output(os)
    kryo.writeObject(kOutput, ss)
    kOutput.close
    os.close
  }

  /*
   * This method creates a new InstanceSequence, RawInstanceSequenceStringObs that lazily re-computes the feature vector for the instance.
   * WARNING: This method only works for observation sequences parameterized with type java.lang.String since it 
   * must serialize the instance to disk with a specific type.
   */
  def apply[T](sg: TrainingSeqGen[T], ss: SourceSequence[T], st: Int, en: Int)(implicit m: Manifest[T]): InstanceSequence = {
    CrfInstance.diskCache match {
      case Some(filePath) =>
        val ofile = new java.io.File(filePath + "/" + icnt)
        icnt += 1
        if (m.toString equals "java.lang.String") { // should be a better way to set this up...
          val ssA = ss.asInstanceOf[SourceSequence[String]]
          val sgA = sg.asInstanceOf[TrainingSeqGen[String]]
          //sbinary.Operations.toFile[SourceSequence[String]](ssA)(ofile)
          serializeSourceSeqToFile(ssA, ofile)
          new RawInstanceSequenceStringObs(sgA, ofile, st, en, ss.length)
        } else throw new RuntimeException("Disk caching only allowed with SourceSequence[T] where T = String")
      case None => new FactoredCachedSourceSequence(sg, ss, st, en)
    }
  }
  def apply(s: Seq[AbstractInstance], st: Int, en: Int): InstanceSequence = {
    CrfInstance.diskCache match {
      case Some(filePath) =>
        val ofile = new java.io.File(filePath + "/" + icnt)
        icnt += 1
        s match {
          case s: Seq[CrfInstance] =>
            serializeCrfInstanceToFile(s, ofile)
            new CrfDiskInstanceSequence(ofile, st, en, s.length)
            /*
          case s: Seq[NonFactoredCrfInstance] =>
            serializeNonFactoredToFile(s, ofile)
            new NonFactoredCrfDiskInstanceSequence(ofile, st, en, s.length)
            * 
            */
        }
      case None => new MemoryInstanceSequence(s, st, en)
    }
  }
  def apply(s: Seq[AbstractInstance]): InstanceSequence = apply(s, -1, -1)
  def apply[T](sg: SeqGen[T], s: SourceSequence[T]) = {
    new NonFactoredCachedSourceSequence(sg, s, -1, -1)
  }
}

class SourceSequence[Obs](val seq: Seq[ObsSource[Obs]], val parentSeq: Option[SourceSequence[Obs]], val st: Int, val en: Int) {
  def this(s: Seq[ObsSource[Obs]], par: Option[SourceSequence[Obs]]) = this(s, par, -1, -1)
  def this(s: Seq[ObsSource[Obs]]) = this(s, None)
  def this(s: Seq[ObsSource[Obs]], st: Int, en: Int) = this(s, None, st, en)
  def this(s: Seq[ObsSource[Obs]], par: SourceSequence[Obs]) = this(s, Some(par))

  def getInfo(k: String): Option[String] = None // subclasses can store and retrieve info here
  def apply(i: Int) = seq(i)
  def length = seq.length
{'fraction_non_alphanumeric': 0.11967925659472423, 'fraction_numerical': 0.003147482014388489, 'mean_word_length': 3.6906854130052724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2651798', 'n_tokens_mistral': 4539, 'n_tokens_neox': 4436, 'n_words': 1121}