<commit_before>package latis.reader.tsml

import latis.dm._
import scala.io.Source
import scala.collection._
import latis.data.Data
import latis.util.Util
import java.nio.ByteBuffer
import latis.data.IterableData
import latis.time.Time
import latis.reader.tsml.ml.Tsml

class IterativeAsciiAdapter(tsml: Tsml) extends IterativeAdapter(tsml) with AsciiAdapterHelper {
  
  def makeIterableData(sampleTemplate: Sample): Data = new IterableData {
    def recordSize = sampleTemplate.getSize
    val recordIterator = getRecordIterator
    
    //TODO: make PeekIterator? but don't access data source prematurely
    def iterator = new Iterator[Data] {
      override def hasNext = recordIterator.hasNext
      override def next = {
        val record = recordIterator.next
        val svals = parseRecord(record)
        makeDataFromRecord(sampleTemplate, svals)
      }
    }
  }
  
  //TODO: compare to Util dataToVariable... move this there?
  def makeDataFromRecord(sampleTemplate: Sample, svals: Map[String, String]): Data = {
    //build a ByteBuffer
    val size = sampleTemplate.getSize
    val bb = ByteBuffer.allocate(size)
    
    //assume every Scalar in the template has a value in the Map, e.g. not stored by Tuple name
    //get Seq of Scalars from template
    val vars = sampleTemplate.toSeq 
    
    for (v <- vars) {
      val s = svals(v.getName) //string value for the given scalar
      v match {
        case _: Real => bb.putDouble(s.toDouble)
        case _: Integer => bb.putLong(s.toLong)
        case t: Text => {
          val l = t.length<commit_msg>use PeekIterator and skip invalid records<commit_after>package latis.reader.tsml

import latis.dm._
import scala.io.Source
import scala.collection._
import latis.data.Data
import latis.util.Util
import java.nio.ByteBuffer
import latis.data.IterableData
import latis.time.Time
import latis.reader.tsml.ml.Tsml
import latis.util.PeekIterator

class IterativeAsciiAdapter(tsml: Tsml) extends IterativeAdapter(tsml) with AsciiAdapterHelper {
  
  def makeIterableData(sampleTemplate: Sample): Data = new IterableData {
    def recordSize = sampleTemplate.getSize
    
    def iterator = new PeekIterator[Data] {
      val it = getRecordIterator
      
      //keep going till we find a valid record or run out
      def getNext: Data = {
        if (it.hasNext) {
          val record = it.next
          val svals = parseRecord(record)
          if (svals.isEmpty) getNext
          else makeDataFromRecord(sampleTemplate, svals)
        } else null
      }
    }
  }
  
  //TODO: compare to Util dataToVariable... move this there?
  def makeDataFromRecord(sampleTemplate: Sample, svals: Map[String, String]): Data = {
    //build a ByteBuffer
    val size = sampleTemplate.getSize
    val bb = ByteBuffer.allocate(size)
    
    //assume every Scalar in the template has a value in the Map, e.g. not stored by Tuple name
    //get Seq of Scalars from template
    val vars = sampleTemplate.toSeq 
    
    for (v <- vars) {
      val s = svals(v.getName) //string value for the given scalar
      v match {
        case _: Real => bb.putDouble(s.toDouble)
        case _: Integer => bb.putLong(s.toLong)
        case t: Text => {
          val l = t.length
{'fraction_non_alphanumeric': 0.08292079207920793, 'fraction_numerical': 0.0, 'mean_word_length': 2.8351126927639383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1888227', 'n_tokens_mistral': 989, 'n_tokens_neox': 968, 'n_words': 300}