<commit_before>package latis.dm

import latis.data.SampleData
import latis.ops.Operation
import latis.util.DataUtils
import latis.util.MappingIterator
import latis.util.PeekIterator
import com.typesafe.scalalogging.slf4j.Logging

//TODO: SampleMappedFunction?
class WrappedFunction(function: Function, val operation: Operation) 
  extends SampledFunction(function.getDomain, function.getRange) with Logging {
  
  /**
   * Override iterator to apply the Operation to each sample as it iterates (lazy).
   */
  override def iterator: PeekIterator[Sample] = {
    logger.debug("Sample Iterating: " + this)
    new MappingIterator(function.iterator, (s: Sample) => operation.applyToSample(s))
  }
  
  /**
   * Get an Iterator of SampleData for this Function. This is really a step backwards
   * since it uses the wrapped Function's iterator which has already put the original Data into Samples.
   */
  //only needed for projection?
  override def getDataIterator: Iterator[SampleData] = {
    logger.debug("Data Iterating: " + this)
    iterator.map(DataUtils.sampleToData(_))
  }

  override def toString = super.toString + " + " + operation
}

object WrappedFunction {
  def apply(function: Function, operation: Operation) = new WrappedFunction(function, operation)
}

  
  <commit_msg>disable having WrappedFunction add Operation to its toString<commit_after>package latis.dm

import latis.data.SampleData
import latis.ops.Operation
import latis.util.DataUtils
import latis.util.MappingIterator
import latis.util.PeekIterator
import com.typesafe.scalalogging.slf4j.Logging

//TODO: SampleMappedFunction?
class WrappedFunction(function: Function, val operation: Operation) 
  extends SampledFunction(function.getDomain, function.getRange) with Logging {
  
  /**
   * Override iterator to apply the Operation to each sample as it iterates (lazy).
   */
  override def iterator: PeekIterator[Sample] = {
    logger.debug("Sample Iterating: " + this)
    new MappingIterator(function.iterator, (s: Sample) => operation.applyToSample(s))
  }
  
  /**
   * Get an Iterator of SampleData for this Function. This is really a step backwards
   * since it uses the wrapped Function's iterator which has already put the original Data into Samples.
   */
  //only needed for projection?
  override def getDataIterator: Iterator[SampleData] = {
    logger.debug("Data Iterating: " + this)
    iterator.map(DataUtils.sampleToData(_))
  }

  //override def toString = super.toString + " + " + operation
}

object WrappedFunction {
  def apply(function: Function, operation: Operation) = new WrappedFunction(function, operation)
}

  
  
{'fraction_non_alphanumeric': 0.09066666666666667, 'fraction_numerical': 0.0007619047619047619, 'mean_word_length': 4.337398373983739, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2131556', 'n_tokens_mistral': 757, 'n_tokens_neox': 749, 'n_words': 225}