<commit_before>package latis.reader

import latis.dm._
import scala.collection.immutable._
import latis.writer.AsciiWriter

/**
 * Make a toy Dataset to test the LaTiS API against. 
 */
class ToyDatasetAccessor extends DatasetAccessor {

  /**
   * Make a Dataset.
   */
  def getDataset() = {
    //
    val v1 = new Real
    val v2 = new Real
    new Dataset(this, Seq(v1, v2))
  }
  
  def close() {}

}

object ToyDatasetAccessor extends App {
  
  //make a DatasetAccessor
  val da = new ToyDatasetAccessor()
  
  //get the Dataset
  val ds = da.getDataset()
  
  //write the Dataset
  val writer = new AsciiWriter(System.out)
  writer.write(ds)
  
}<commit_msg>generalize to easily make various Variables for the toy dataset<commit_after>package latis.reader

import latis.dm._
import scala.collection.immutable._
import latis.writer.AsciiWriter

/**
 * Make a toy Dataset to test the LaTiS API against. 
 */
class ToyDatasetAccessor(val variable: Variable) extends DatasetAccessor {

  /**
   * Make a Dataset.
   */
  def getDataset() = {
    Dataset(this, variable)
  }
  
  def close() {}

}

object ToyDatasetAccessor extends App {
  
  def real = {
    Real()
  }
  
  def realTuple = {
    Tuple(real, real, real)
  }
  
  //make Variable for toy dataset
  //val v = real
  val v = realTuple
  
  //make a DatasetAccessor for this variable
  val da = new ToyDatasetAccessor(v)
  
  //get the Dataset
  val ds = da.getDataset()
  
  //write the Dataset
  val writer = new AsciiWriter(System.out)
  writer.write(ds)
  
}
{'fraction_non_alphanumeric': 0.09797517962116264, 'fraction_numerical': 0.002612671456564337, 'mean_word_length': 2.792079207920792, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1687447', 'n_tokens_mistral': 522, 'n_tokens_neox': 520, 'n_words': 155}