<commit_before><commit_msg>placeholder<commit_after>package ws.vinta.albedo.transformers

import org.apache.spark.ml.UnaryTransformer
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable}
import org.apache.spark.sql.types._

class UDFTransformer(override val uid: String)
  extends UnaryTransformer[String, Seq[String], UDFTransformer] with DefaultParamsWritable {

  def this() = {
    this(Identifiable.randomUID("udfTransformer"))
  }

  override protected def createTransformFunc: String => Seq[String] = { originStr =>
    Seq("abc")
  }

  override protected def validateInputType(inputType: DataType): Unit = {
    require(inputType == StringType, s"Input type must be string type but got $inputType.")
  }

  override protected def outputDataType: DataType = {
    ArrayType(StringType)
  }

  override def copy(extra: ParamMap): this.type = {
    defaultCopy(extra)
  }
}

object UDFTransformer extends DefaultParamsReadable[UDFTransformer]
{'fraction_non_alphanumeric': 0.1, 'fraction_numerical': 0.0, 'mean_word_length': 5.364197530864198, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '828363', 'n_tokens_mistral': 315, 'n_tokens_neox': 306, 'n_words': 67}