<commit_before>package latis.dm

import scala.collection._
import latis.data._
import latis.metadata._
import java.nio.ByteBuffer
import latis.util.Util

trait Function extends Variable { //this: Variable =>
  def getDomain: Variable
  def getRange: Variable
  
  //TODO: reconsider when we add ContinuousFunction
  def getFirstSample: Sample
  def getLastSample: Sample
  
  //TODO: put in Variable?
  //def length: Int  getLength?
  def iterator: Iterator[Sample]
}

class SampledFunction(domain: Variable, range: Variable, _iterator: Iterator[Sample] = null,
    metadata: Metadata = EmptyMetadata, data: Data = EmptyData) 
    extends Variable2(metadata, data) with Function {
  
  /*
   * 2013-08-07
   * TODO: ContinuousFunction
   * now that Data is separable, how can we support continuous function (e.g. exp model)
   * ContinuousFunction:
   *   apply(domainVal: Variable) => range val
   *   apply(domainSet: Seq[Variable] or Var with SeqData) => SampledFunction
   *   length = -1?
   *   iterator => error
   */

  //expose domain and range via defs only so we can override (e.g. ProjectedFunction)
  def getDomain: Variable = domain
  def getRange: Variable = range
  
  //private var _iterator: Iterator[Sample] = null
  
  def iterator: Iterator[Sample] = {
    if (_iterator != null) _iterator
    else if (getData.isEmpty) iterateFromKids
    else getDataIterator.map(Util.dataToSample(_, Sample(domain, range)))
  }
  
  private def iterateFromKids: Iterator[Sample] = {
    val dit = domain.getDataIterator.map(data => Util.dataToVariable(data, domain))
    val rit = range.getDataIterator.map(data => Util.dataToVariable(data, range))
    (dit zip rit).map(pair => Sample(pair._1, pair._2))
  }

  
  //Support first and last filters
  //TODO: consider more optimal approaches
  def getFirstSample: Sample = iterator.next
  def getLastSample: Sample = {
    //iterator.drop(length-1).next  //dataIterator is giving Util.dataToSample null Data!?
    
    //TODO: only gets 190 of 570!?
    //trying to get new iterator with each ref? skipping by 3s
//    while(iterator.hasNext) {
//      sample = iterator.next
//      c = c+1
//      println(c +": "+ sample.domain.data)
//    }
    
    var sample: Sample = null
    for (s <- iterator) sample = s
    sample 
  }
  //TODO: return Function with single sample to preserve namespace...?

}

object Function {
  //TODO: used named args for data, md?
  
  
  def apply(domain: Variable, range: Variable): Function = new SampledFunction(domain, range)
  
  def apply(domain: Variable, range: Variable, md: Metadata): Function = new SampledFunction(domain, range, metadata = md)
  
  def apply(domain: Variable, range: Variable, data: Data): Function = new SampledFunction(domain, range, data = data)

  
  //build from Iterator[Sample]
  //TODO: could we set data to something?
  def apply(domain: Variable, range: Variable, sampleIterator: Iterator[Sample]): Function = {
    //Note, wouldn't need domain and range, but would have to trigger iterator, or use peek?
    new SampledFunction(domain, range, sampleIterator)
  }
  
  def apply(domain: Variable, range: Variable, md: Metadata, data: Data): Function = {
    new SampledFunction(domain, range, metadata = md, data = data)
  }
  
  def apply(vals: Seq[Seq[Double]]): Function = Function(vals.head, vals.tail: _*)
  
  def apply(dvals: Seq[Double], vals: Seq[Double]*): Function = {
    val domain = Real(dvals)
    //make Real if vals.length == 1
    val range = if (vals.length == 1) Real(vals(0)) else Tuple(vals)(0d) //hack to get around type erasure ambiguity
    Function(domain, range)
  }
  
  //def apply(samples: Seq[Sample]): Function = 
  <commit_msg>factor out SampledFunction, add constructors<commit_after>package latis.dm

import scala.collection._
import latis.data._
import latis.metadata._
import java.nio.ByteBuffer
import latis.util.Util

trait Function extends Variable { //this: Variable =>
  def getDomain: Variable
  def getRange: Variable
  
  //TODO: reconsider when we add ContinuousFunction
  def getFirstSample: Sample
  def getLastSample: Sample
  
  //TODO: put in Variable?
  //def length: Int  getLength?
  def iterator: Iterator[Sample]
}


object Function {
  //TODO: used named args for data, md?
  
  
  def apply(domain: Variable, range: Variable): Function = new SampledFunction(domain, range)
  
  def apply(domain: Variable, range: Variable, md: Metadata): Function = new SampledFunction(domain, range, metadata = md)
  
  def apply(domain: Variable, range: Variable, data: Data): Function = new SampledFunction(domain, range, data = data)

  
  //build from Iterator[Sample]
  //TODO: could we set data to something?
  def apply(domain: Variable, range: Variable, sampleIterator: Iterator[Sample], md: Metadata): Function = {
    //Note, wouldn't need domain and range, but would have to trigger iterator, or use peek?
    new SampledFunction(domain, range, sampleIterator, metadata = md)
  }
  
  def apply(domain: Variable, range: Variable, sampleIterator: Iterator[Sample]): Function = 
    Function(domain, range, sampleIterator, EmptyMetadata)
  
  def apply(domain: Variable, range: Variable, md: Metadata, data: Data): Function = {
    new SampledFunction(domain, range, metadata = md, data = data)
  }
  
  def apply(vs: Seq[Variable], metadata: Metadata = EmptyMetadata): Function = vs.head match {
    case sample: Sample => Function(sample.domain, sample.range, vs.asInstanceOf[Seq[Sample]].iterator, metadata)
    case _ => {
      //make Seq of samples where domain is index
      val samples = vs.zipWithIndex.map(s => Sample(Index(s._2), s._1))
      val sample = samples.head
      Function(sample.domain, sample.range, samples.iterator, metadata)
    }
  }
  
  def apply(ds: Seq[Variable], rs: Seq[Variable]): Function = {
    //TODO: assert same length?
    Function((ds zip rs).map(s => Sample(s._1, s._2)))
  }
  
  def fromValues(vals: Seq[Seq[Double]]): Function = Function.fromValues(vals.head, vals.tail: _*)
  
  def fromValues(dvals: Seq[Double], vals: Seq[Double]*): Function = {
    val domain = Real(dvals)
    //make Real if vals.length == 1
    val range = if (vals.length == 1) Real(vals(0)) else Tuple(vals)(0d) //hack to get around type erasure ambiguity
    Function(domain, range)
  }
  
  //def apply(samples: Seq[Sample]): Function = 
  
{'fraction_non_alphanumeric': 0.11442080378250591, 'fraction_numerical': 0.005200945626477541, 'mean_word_length': 3.61863173216885, 'pattern_counts': {'":': 1, '<': 4, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '396565', 'n_tokens_mistral': 2011, 'n_tokens_neox': 1922, 'n_words': 623}