<commit_before>//--------------------------------------
//
// StructureEncoder.scala
// Since: 2013/01/17 10:36 AM
//
//--------------------------------------

package xerial.silk.index

import xerial.lens._
import xerial.core.log.Logger


trait FieldWriter {
  def write(index: OrdPath, value: Any): Unit
}

class SimpleFieldWriter(name: String) extends FieldWriter with Logger {
  private var prev : Option[OrdPath] = None
  def write(index: OrdPath, value: Any) {
    val diff = prev.map(index.incrementalDiff(_)) getOrElse (OrdPath.zero)
    val lmnz = diff.leftMostNonZeroPos
    val dl = if(lmnz == 0) 0 else diff(lmnz-1)
    debug("write %-10s (%-10s) %-10s [r:%d, d:%d] : %s".format(name, index, diff, diff.leftMostNonZeroPos, dl, value))
    prev = Some(index)
  }



}

case class ParamKey(name: String, valueType: ObjectType)

/**
 *
 *
 * @author <NAME>
 */
class StructureEncoder extends Logger {

  import TypeUtil._

  private val objectWriterTable = collection.mutable.Map[Int, FieldWriter]()
  private val writerTable = collection.mutable.Map[ParamKey, FieldWriter]()

  def objectWriter(level:Int) : FieldWriter = {
    objectWriterTable.getOrElseUpdate(level, new SimpleFieldWriter("<obj:L%d>".format(level)))
  }

  def fieldWriterOf(paramName: String, valueType: ObjectType): FieldWriter = {
    val k = ParamKey(paramName, valueType)
    writerTable.getOrElseUpdate(k, new SimpleFieldWriter(paramName))
  }

  private var current = OrdPath.one

  def encode(obj: Any) {
    current = encode(current, obj)
  }


  private def encode(path: OrdPath, obj: Any) : OrdPath = {
    val cl = obj.getClass
    if (TypeUtil.isSeq(cl)) {
      //objectWriter(path.length).write(path, "Seq")
      val seq = obj.asInstanceOf[Seq[_]]
      var next = path
      seq.foreach { e =>
        encode(next, e)
        next = next.sibling
      }
      next
    }
    else {
      val schema = ObjectSchema(cl)

      // write object type
      objectWriter(path.length).write(path, schema.name)

      var next = path
      for (param <- schema.constructor.params) {
        next = path.child
        encode(next, param.name, param.valueType, param.get(obj))
      }
      next
    }
  }


  private def encode(path: OrdPath, paramName: String, valueType: ObjectType, obj: Any) {

    def writeField {
      // TODO improve the value retrieval by using code generation
      val w = fieldWriterOf(paramName, valueType)
      w.write(path, obj)
    }

    valueType match {
      case p: Primitive => writeField
      case t: TextType => writeField
      case s: StandardType =>
        encode(path, obj)
      case s: SeqType =>
        val seq = obj.asInstanceOf[Seq[_]]
        var next = path.child
        seq.foreach { e =>
          encode(next, paramName, s.elementType, e)
          next = next.sibling
        }
      case o: OptionType =>
        val opt = obj.asInstanceOf[Option[_]]
        opt.foreach {
          encode(path, paramName, o.elementType, _)
        }
      case g: GenericType =>
        warn("TODO impl: %s", g)
    }

  }<commit_msg>Add a level-layer to FieldWriter<commit_after>//--------------------------------------
//
// StructureEncoder.scala
// Since: 2013/01/17 10:36 AM
//
//--------------------------------------

package xerial.silk.index

import xerial.lens._
import xerial.core.log.Logger


trait FieldWriter {
  def write(index: OrdPath, value: Any): Unit
}

class SimpleFieldWriter(level:Int, name: String) extends FieldWriter with Logger {
  private var prev : Option[OrdPath] = None
  def write(index: OrdPath, value: Any) {
    val diff = prev.map(index.incrementalDiff(_)) getOrElse (OrdPath.zero)
    val lmnz = diff.leftMostNonZeroPos
    val dl = if(lmnz == 0) 0 else diff(lmnz-1)
    debug("write %10s:L%d (%-10s) %-10s [level:%d, offset:%d] : %s".format(name, level, index, diff, diff.leftMostNonZeroPos, dl, value))
    prev = Some(index)
  }



}

case class ParamKey(level:Int, name: String, valueType: ObjectType)

/**
 *
 *
 * @author <NAME>
 */
class StructureEncoder extends Logger {

  import TypeUtil._

  private val objectWriterTable = collection.mutable.Map[Int, FieldWriter]()
  private val writerTable = collection.mutable.Map[ParamKey, FieldWriter]()

  def objectWriter(level:Int) : FieldWriter = {
    objectWriterTable.getOrElseUpdate(level, new SimpleFieldWriter(level, "<obj>"))
  }

  def fieldWriterOf(level:Int, paramName: String, valueType: ObjectType): FieldWriter = {
    val k = ParamKey(level, paramName, valueType)
    writerTable.getOrElseUpdate(k, new SimpleFieldWriter(level, paramName))
  }

  private var current = OrdPath.one

  def encode(obj: Any) {
    current = encode(current, obj)
  }


  private def encode(path: OrdPath, obj: Any) : OrdPath = {
    val cl = obj.getClass
    if (TypeUtil.isSeq(cl)) {
      //objectWriter(path.length).write(path, "Seq")
      val seq = obj.asInstanceOf[Seq[_]]
      var next = path
      seq.foreach { e =>
        encode(next, e)
        next = next.sibling
      }
      next
    }
    else {
      val schema = ObjectSchema(cl)

      // write object type
      objectWriter(path.length).write(path, schema.name)

      var next = path
      for (param <- schema.constructor.params) {
        next = path.child
        encode(next, param.name, param.valueType, param.get(obj))
      }
      next
    }
  }


  private def encode(path: OrdPath, paramName: String, valueType: ObjectType, obj: Any) {

    def writeField {
      // TODO improve the value retrieval by using code generation
      val w = fieldWriterOf(path.length, paramName, valueType)
      w.write(path, obj)
    }

    valueType match {
      case p: Primitive => writeField
      case t: TextType => writeField
      case s: StandardType =>
        encode(path, obj)
      case s: SeqType =>
        val seq = obj.asInstanceOf[Seq[_]]
        var next = path.child
        seq.foreach { e =>
          encode(next, paramName, s.elementType, e)
          next = next.sibling
        }
      case o: OptionType =>
        val opt = obj.asInstanceOf[Option[_]]
        opt.foreach {
          encode(path, paramName, o.elementType, _)
        }
      case g: GenericType =>
        warn("TODO impl: %s", g)
    }

  }
{'fraction_non_alphanumeric': 0.14193859508117665, 'fraction_numerical': 0.006751326153351551, 'mean_word_length': 2.501406865503658, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '13424', 'n_tokens_mistral': 2003, 'n_tokens_neox': 1934, 'n_words': 500}