<commit_before>package latis.ops.filter

import latis.dm.Function
import latis.metadata.Metadata
import latis.ops.OperationFactory


class TakeOperation(val n: Int) extends Filter {
  override def applyToFunction(function: Function) = {
    //Assume we can hold this all in memory.
    
    //get the first 'limit' samples, or all if we had less
    val samples = function.iterator.take(n).toList
    
    //change length of Function in metadata
    val md = Metadata(function.getMetadata.getProperties + ("length" -> samples.length.toString))
    
    //make the new function with the updated metadata
    samples.length match {
      case 0 => Some(Function(function.getDomain, function.getRange, Iterator.empty, md)) //empty Function with type of original
      case _ => Some(Function(samples, md))
    }
  }
}

object TakeOperation extends OperationFactory {
  
  override def apply(args: Seq[String]): TakeOperation = {
    if (args.length > 1) throw new UnsupportedOperationException("The TakeOperation accepts only one argument")
    try {
      TakeOperation(args.head.toInt)<commit_msg>updated getMetadata to use +<commit_after>package latis.ops.filter

import latis.dm.Function
import latis.metadata.Metadata
import latis.ops.OperationFactory


class TakeOperation(val n: Int) extends Filter {
  override def applyToFunction(function: Function) = {
    //Assume we can hold this all in memory.
    
    //get the first 'n' samples, or all if we had less
    val samples = function.iterator.take(n).toList
    
    //change length of Function in metadata
    val md = function.getMetadata + ("length" -> samples.length.toString)
    
    //make the new function with the updated metadata
    samples.length match {
      case 0 => Some(Function(function.getDomain, function.getRange, Iterator.empty, md)) //empty Function with type of original
      case _ => Some(Function(samples, md))
    }
  }
}

object TakeOperation extends OperationFactory {
  
  override def apply(args: Seq[String]): TakeOperation = {
    if (args.length > 1) throw new UnsupportedOperationException("The TakeOperation accepts only one argument")
    try {
      TakeOperation(args.head.toInt)
{'fraction_non_alphanumeric': 0.09086737035337311, 'fraction_numerical': 0.0018357044515832951, 'mean_word_length': 3.5798319327731094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '524858', 'n_tokens_mistral': 600, 'n_tokens_neox': 588, 'n_words': 199}