<commit_before>package latis.ops.filter

import latis.dm.Scalar
import latis.dm.Number
import latis.dm.Variable
import latis.ops.OperationFactory
import latis.dm.Tuple

class ExcludeMissing extends Filter {

  override def applyToScalar(scalar: Scalar): Option[Variable] = {
    if (scalar.isMissing) None
    else Some(scalar)
  }
  
  /**
   * Exclude a Tuple if any element in it is excluded.
   */
  override def applyToTuple(tuple: Tuple): Option[Variable] = {
    val vars = tuple.getVariables.flatMap(applyToVariable(_))
    if (vars.length == tuple.getElementCount) Some(tuple)
    else None
  }<commit_msg>return type should be Tuple for a Filter<commit_after>package latis.ops.filter

import latis.dm.Scalar
import latis.dm.Number
import latis.dm.Variable
import latis.ops.OperationFactory
import latis.dm.Tuple

class ExcludeMissing extends Filter {

  override def applyToScalar(scalar: Scalar): Option[Variable] = {
    if (scalar.isMissing) None
    else Some(scalar)
  }
  
  /**
   * Exclude a Tuple if any element in it is excluded.
   */
  override def applyToTuple(tuple: Tuple): Option[Tuple] = {
    val vars = tuple.getVariables.flatMap(applyToVariable(_))
    if (vars.length == tuple.getElementCount) Some(tuple)
    else None
  }
{'fraction_non_alphanumeric': 0.09728867623604466, 'fraction_numerical': 0.0, 'mean_word_length': 3.940944881889764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7553441', 'n_tokens_mistral': 391, 'n_tokens_neox': 387, 'n_words': 107}