<commit_before>package latis.reader.tsml.agg

import latis.dm.Dataset
import latis.dm.implicits._
import latis.reader.tsml.TsmlAdapter
import latis.reader.tsml.ml.Tsml
import scala.collection.mutable.ArrayBuffer
import latis.dm.Tuple
import latis.ops.Operation

/**
 * Base class for Adapters that aggregate (combine) Datasets.
 */
abstract class AggregationAdapter(tsml: Tsml) extends TsmlAdapter(tsml) {
  
  /**
   * Keep a list of adapters so we can close them.
   */
  private val adapters = ArrayBuffer[TsmlAdapter]()
  
  /**
   * Given a Dataset that contains other Datasets (now as a tuple)
   * apply the aggregation logic to make a single Dataset.
   */
  def aggregate(left: Dataset, right: Dataset): Dataset
  
  /**
   * Combine each aggregate Dataset into a single Dataset.
   */
  override protected def makeOrigDataset: Dataset = {
    //Get child dataset nodes
    val dsnodes = (tsml.xml \ "dataset")
    //Make a dataset for each
    val dss = for (node <- dsnodes) yield {
      val tsml = Tsml(node)
      val adapter = TsmlAdapter(tsml)
      adapters += adapter
      adapter.getOrigDataset
    }
    
    collect(dss) 
  }  
  
  override def getDataset(ops: Seq[Operation]) = {
    makeOrigDataset
    val dss = adapters.map(_.getDataset(ops))
    
    val ds = collect(dss)
    
    ops.foldLeft(ds)((dataset, op) => op(dataset)) //doesn't handle any Operations
  }
  
  override protected def makeDataset(ds: Dataset): Dataset = {
    getDataset(List())
  }
    
    
  /**
   * Make a single Dataset that contains the Datasets to be aggregated.
   */
  def collect(datasets: Seq[Dataset]): Dataset = {
      
    //Make metadata
    val md = makeMetadata(tsml.dataset) //TODO: provo<commit_msg>AggregationAdapter cleanup<commit_after>package latis.reader.tsml.agg

import latis.dm.Dataset
import latis.dm.implicits._
import latis.reader.tsml.TsmlAdapter
import latis.reader.tsml.ml.Tsml
import scala.collection.mutable.ArrayBuffer
import latis.dm.Tuple
import latis.ops.Operation

/**
 * Base class for Adapters that aggregate (combine) Datasets.
 */
abstract class AggregationAdapter(tsml: Tsml) extends TsmlAdapter(tsml) {
  
  /**
   * Keep a list of adapters so we can close them.
   */
  protected val adapters = (tsml.xml \ "dataset").map(n => TsmlAdapter(Tsml(n)))
  
  /**
   * Given a Dataset that contains other Datasets (now as a tuple)
   * apply the aggregation logic to make a single Dataset.
   */
  def aggregate(left: Dataset, right: Dataset): Dataset
  
  /**
   * Combine each aggregate Dataset into a single Dataset.
   */
  override protected def makeOrigDataset: Dataset = {
    //Make a dataset for each adapter
    val dss = adapters.map(_.getOrigDataset)
    
    collect(dss) 
  }  
  
  override def getDataset(ops: Seq[Operation]) = {
    val dss = adapters.map(_.getDataset(ops))
    
    val ds = collect(dss)
    
    ops.foldLeft(ds)((dataset, op) => op(dataset)) //doesn't handle any Operations
  }
  
  override protected def makeDataset(ds: Dataset): Dataset = {
    getDataset(List())
  }
    
    
  /**
   * Make a single Dataset that contains the Datasets to be aggregated.
   */
  def collect(datasets: Seq[Dataset]): Dataset = {
      
    //Make metadata
    val md = makeMetadata(tsml.dataset) //TODO: provo
{'fraction_non_alphanumeric': 0.1046050625190607, 'fraction_numerical': 0.0, 'mean_word_length': 3.034440344403444, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '373104', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1071, 'n_words': 300}